From d0957409f1648ad5c7899202e055b032c25ca836 Mon Sep 17 00:00:00 2001 From: jinhaosong-source Date: Fri, 31 Jul 2026 17:07:21 +0800 Subject: [PATCH 01/31] feat(wizard): add OrcaRouter as an LLM provider (#4598) OrcaRouter is an OpenAI-compatible routing gateway. Mirror the existing OpenRouter entry in the setup wizard's LLM_PROVIDERS: reuse langchain_openai:ChatOpenAI pointed at api.orcarouter.ai/v1 with env var ORCAROUTER_API_KEY. Default model pins a tool-capable model; orcarouter/auto is also selectable. Disclosure: I'm an engineer on the OrcaRouter team. Co-authored-by: jinhaosong-source --- scripts/wizard/providers.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/wizard/providers.py b/scripts/wizard/providers.py index 5ad7be977..bb60fdf92 100644 --- a/scripts/wizard/providers.py +++ b/scripts/wizard/providers.py @@ -434,6 +434,23 @@ LLM_PROVIDERS: list[LLMProvider] = [ "temperature": 0.7, }, ), + LLMProvider( + name="orcarouter", + display_name="OrcaRouter", + description="OpenAI-compatible adaptive routing gateway", + use="langchain_openai:ChatOpenAI", + models=["openai/gpt-5.5", "anthropic/claude-opus-4.8", "google/gemini-3.5-flash", "orcarouter/auto"], + default_model="openai/gpt-5.5", + env_var="ORCAROUTER_API_KEY", + package="langchain-openai", + extra_config={ + "base_url": "https://api.orcarouter.ai/v1", + "request_timeout": 600.0, + "max_retries": 2, + "max_tokens": 8192, + "temperature": 0.7, + }, + ), LLMProvider( name="vllm", display_name="vLLM", From 0cc28d2c4225752bc4808bf160f0a635d0214e5e Mon Sep 17 00:00:00 2001 From: MiaoRuidx <965742329@qq.com> Date: Fri, 31 Jul 2026 17:13:12 +0800 Subject: [PATCH 02/31] fix(sandbox): enforce deployment-wide E2B capacity (#4575) * docs: design deployment-wide E2B capacity * fix(sandbox): enforce deployment-wide E2B capacity * fix(sandbox): address E2B capacity review findings * fix(sandbox): grace stale E2B capacity inventory --------- Co-authored-by: MiaoRuidx <12540796+MiaoRuidx@users.noreply.github.com> --- .github/workflows/backend-unit-tests.yml | 9 + README.md | 12 +- backend/AGENTS.md | 11 +- .../aio_sandbox/ownership/factory.py | 7 +- .../community/e2b_sandbox/__init__.py | 4 +- .../e2b_sandbox/capacity/__init__.py | 6 + .../community/e2b_sandbox/capacity/redis.py | 279 ++++++++++++++++ .../e2b_sandbox/e2b_sandbox_provider.py | 303 +++++++++++++++--- .../harness/deerflow/config/sandbox_config.py | 7 +- .../tests/test_e2b_capacity_store_redis.py | 178 ++++++++++ backend/tests/test_e2b_sandbox_provider.py | 184 ++++++++++- 11 files changed, 941 insertions(+), 59 deletions(-) create mode 100644 backend/packages/harness/deerflow/community/e2b_sandbox/capacity/__init__.py create mode 100644 backend/packages/harness/deerflow/community/e2b_sandbox/capacity/redis.py create mode 100644 backend/tests/test_e2b_capacity_store_redis.py diff --git a/.github/workflows/backend-unit-tests.yml b/.github/workflows/backend-unit-tests.yml index f6b48effd..29779d00a 100644 --- a/.github/workflows/backend-unit-tests.yml +++ b/.github/workflows/backend-unit-tests.yml @@ -61,6 +61,15 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 env: TEST_POSTGRES_URI: postgresql://deerflow:deerflow@localhost:5432/deerflow_test?sslmode=disable diff --git a/README.md b/README.md index 642eecea7..62b56e362 100644 --- a/README.md +++ b/README.md @@ -907,7 +907,17 @@ Use `burst` with `burst_limit` to permit bounded extra VMs. The `wait` and `reject` policies use only `replicas`. The `reject` policy can remove one warm VM before it returns an error. -`replicas` limits one Gateway process. It does not limit all Gateway processes. +With in-memory ownership, `replicas` limits one Gateway process. With Redis +ownership, E2B shares one capacity Hash between workers using the same +`sandbox.ownership.key_prefix`; `replicas` (plus a configured burst) is then a +deployment-wide hard limit. Use one unique prefix and the same effective limit +per deployment. To change the limit, stop its Gateways, delete the capacity +Hash, and restart; mismatched workers fail closed. + +The Hash counts remote VMs and in-flight creates, repairs interrupted creates +from E2B metadata, grace-protects stale inventory omissions, and blocks new +creates while Redis or initial inventory is unavailable. Run Redis with persistence, non-evicting memory, and HA. + E2B acquisition uses a bounded executor. Waiting acquisitions do not use the default asyncio executor. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 7b947ed96..2ce7a1e60 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -616,8 +616,13 @@ that cannot tell sibling branches apart. The `wait` policy fails the turn after `acquire_timeout`. The runtime does not retry the turn automatically. E2B acquisition uses a bounded executor. Waiting calls do not consume the default asyncio executor. The `reject` - policy can evict one warm VM before it returns an error. `replicas` limits - one Gateway process. It does not provide multi-process capacity control. + policy can evict one warm VM before it returns an error. With memory + ownership, `replicas` limits one Gateway process. Redis ownership shares one + `:e2b-capacity` Hash, making the limit (plus a bounded + burst) deployment-wide. Lua atomically manages VM and in-flight-create + entries; missing or unavailable state fails closed. E2B reservation metadata + repairs interrupted creates. Inventory replacement is revision-CAS guarded, + and incomplete inventories never remove entries; complete omissions get a grace period. Uncertain cleanup keeps a tombstone slot. Shutdown tracks owned remote operation IDs. Discovery can find a VM from another Gateway. Shutdown closes an unowned discovery client without destroying its VM. Release ends its @@ -656,7 +661,7 @@ that cannot tell sibling branches apart. - **Teardown join budget covers refresh plus release.** Redis bounds each ownership operation at five seconds, and context exit can catch the heartbeat in one final refresh before its `finally` performs the final release. `_TEARDOWN_JOIN_TIMEOUT_SECONDS` is therefore 12 seconds — greater than both sequential operation bounds — so a normal pair of socket timeouts does not emit the deferred-release warning; a still-running heartbeat continues to own the release safely. - **An absent lease means the same thing on both paths, and reconciliation must say so too.** The `LAPSED` rule above only covers an owner renewing its *own* lease; on its own it does not make state loss safe, because reconciliation reads the same absent key as "orphan, adopt". After a Redis flush (restart without persistence, or eviction under `maxmemory`) every owner is alive and merely pre-renewal-tick, so whichever instance reconciles first would adopt every live container, each real owner's next renewal would report `LOST`, and it would drop a sandbox mid-turn for the adopter to idle-destroy — #4206 through the back door. `_adoptable_after_grace` closes it: an untracked container must be seen unowned (`owner()`, a read-only peek — the atomic `claim()` is still what actually gates adoption) across a full lease TTL before it can be adopted, tracked per container in `_unowned_since`. That rebuilds the delay the flush erased — a live owner republishes within one renewal interval, shorter than the TTL by construction (`ttl_multiplier >= 2`) — while a genuinely crashed owner never republishes, so its containers are still adopted one grace later rather than leaking. A republished lease **resets** the grace; a pausing-only timer would still expire over a live owner's lease. The grace is skipped when `supports_cross_process` is `False`: no peer can hold a lease such a store would show us, so single-instance deployments keep instant orphan cleanup, and a grace could not help a multi-worker gateway on `memory` anyway (peers are invisible to each other's leases with or without it). - **The `memory` store is single-instance only** and says so via `supports_cross_process = False`; the provider logs a warning at startup when the configured store cannot see peers. A multi-worker gateway on `memory` has no cross-process coordination at all — same contract as `stream_bridge`'s memory backend. This is why the redis inference matters: it reads `app_config.stream_bridge` **and** the env var, in the same order the bridge's own resolver does, so any deployment already pointing the bridge at Redis (i.e. every multi-instance one) gets a redis ownership store without extra config. - - `get()` stays a pure in-memory lookup and must never call the store (that is blocking filesystem/network IO on the event loop); anchored by `tests/blocking_io/test_aio_sandbox_get.py`, which injects a deliberately-blocking probe store so the anchor keeps its teeth regardless of the configured backend. Tests: `tests/test_sandbox_ownership_store.py` (store contract, defined once for **both** backends — but the redis tier is `@pytest.mark.integration` + opt-in via `DEER_FLOW_TEST_REDIS_URL` and self-skips, and **CI provisions no redis**, so the merge gate runs the memory tier only and the Lua scripts never execute there; drift between the backends is caught only when the suite runs against a live redis. There is no fake-redis tier because the redis exclusion lives in Lua a fake would not execute) and `tests/test_sandbox_orphan_reconciliation.py` (provider behaviour, two providers sharing one store). + - `get()` stays a pure in-memory lookup and must never call the store (that is blocking filesystem/network IO on the event loop); anchored by `tests/blocking_io/test_aio_sandbox_get.py`, which injects a deliberately-blocking probe store so the anchor keeps its teeth regardless of the configured backend. Tests: `tests/test_sandbox_ownership_store.py` (store contract, defined once for **both** backends — the redis tier is `@pytest.mark.integration`, uses `DEER_FLOW_TEST_REDIS_URL` when set, and otherwise self-skips without a reachable Redis. Backend CI provisions Redis, so the merge gate executes the real Lua tier; there is no fake-redis tier because a fake would not execute the Lua exclusions) and `tests/test_sandbox_orphan_reconciliation.py` (provider behaviour, two providers sharing one store). - `BoxliteProvider` (`packages/harness/deerflow/community/boxlite/`) - BoxLite micro-VM isolation. The `boxlite` runtime is optional (`deerflow-harness[boxlite]`) and lazy-imported only when this provider is selected. The provider owns one private asyncio event loop on a daemon thread because BoxLite handles are loop-affine; sync `Sandbox` calls marshal onto that loop with `run_coroutine_threadsafe`. Boxes are named deterministically from `user_id:thread_id`, released into an in-process warm pool after each agent turn, and reclaimed only by the same user/thread. Warm-pool health checks use a short explicit timeout and forward that timeout through both BoxLite `exec(timeout=...)` and the private-loop `.result(timeout)` bridge so a hung VM cannot pin the per-thread acquire lock indefinitely. `sandbox.replicas` caps active + warm VMs per gateway process; if capacity is exhausted, only warm-pool VMs are evicted. `sandbox.idle_timeout` stops idle warm VMs after the configured seconds. `reset()` is intentionally a lightweight registry clear for `reset_sandbox_provider()` and does not close boxes, stop the idle reaper, or close the private loop; full teardown remains `shutdown()`. diff --git a/backend/packages/harness/deerflow/community/aio_sandbox/ownership/factory.py b/backend/packages/harness/deerflow/community/aio_sandbox/ownership/factory.py index 3070f8c21..0be236c88 100644 --- a/backend/packages/harness/deerflow/community/aio_sandbox/ownership/factory.py +++ b/backend/packages/harness/deerflow/community/aio_sandbox/ownership/factory.py @@ -61,7 +61,10 @@ def resolve_ownership_config(config: SandboxOwnershipConfig | None, *, stream_br return SandboxOwnershipConfig() -def _resolve_redis_url(config: SandboxOwnershipConfig) -> str: +def resolve_ownership_redis_url( + config: SandboxOwnershipConfig, +) -> str: + """Resolve the Redis endpoint shared by ownership-adjacent stores.""" return config.redis_url or os.getenv(_ENV_OWNERSHIP_REDIS_URL) or os.getenv(_ENV_STREAM_BRIDGE_REDIS_URL) or os.getenv("REDIS_URL") or "redis://localhost:6379/0" @@ -96,7 +99,7 @@ def make_sandbox_ownership_store(config: SandboxOwnershipConfig | None, *, owner if resolved.type == "redis": from .redis import RedisOwnershipStore - redis_url = _resolve_redis_url(resolved) + redis_url = resolve_ownership_redis_url(resolved) logger.info("Sandbox ownership store: redis (ttl=%.1fs, renewal=%.1fs)", ttl, resolved.renewal_interval_seconds) return RedisOwnershipStore( owner_id=effective_owner_id, diff --git a/backend/packages/harness/deerflow/community/e2b_sandbox/__init__.py b/backend/packages/harness/deerflow/community/e2b_sandbox/__init__.py index 289500a51..dd9e36ef7 100644 --- a/backend/packages/harness/deerflow/community/e2b_sandbox/__init__.py +++ b/backend/packages/harness/deerflow/community/e2b_sandbox/__init__.py @@ -12,8 +12,8 @@ Configuration example (``config.yaml``):: template: code-interpreter-v1 # e2b template id; defaults to e2b code-interpreter domain: e2b.dev # optional e2b domain (e.g. self-hosted) idle_timeout: 600 # forwarded to e2b ``set_timeout`` (seconds) - replicas: 3 # max concurrent sandboxes (LRU eviction beyond) - ownership: # required for safe multi-worker reconciliation + replicas: 3 # hard capacity shared when ownership is Redis + ownership: # multi-worker ownership + capacity coordination type: redis redis_url: $REDIS_URL reconciliation_interval_seconds: 60 diff --git a/backend/packages/harness/deerflow/community/e2b_sandbox/capacity/__init__.py b/backend/packages/harness/deerflow/community/e2b_sandbox/capacity/__init__.py new file mode 100644 index 000000000..e69750783 --- /dev/null +++ b/backend/packages/harness/deerflow/community/e2b_sandbox/capacity/__init__.py @@ -0,0 +1,6 @@ +"""Redis-backed deployment-wide E2B capacity.""" + +from .redis import CapacityBackendError as CapacityBackendError +from .redis import RedisE2BCapacityStore as RedisE2BCapacityStore +from .redis import ReserveStatus as ReserveStatus +from .redis import make_e2b_capacity_store as make_e2b_capacity_store diff --git a/backend/packages/harness/deerflow/community/e2b_sandbox/capacity/redis.py b/backend/packages/harness/deerflow/community/e2b_sandbox/capacity/redis.py new file mode 100644 index 000000000..22795639b --- /dev/null +++ b/backend/packages/harness/deerflow/community/e2b_sandbox/capacity/redis.py @@ -0,0 +1,279 @@ +"""Atomic Redis Hash ledger for deployment-wide E2B capacity.""" + +from __future__ import annotations + +import enum +import logging + +from deerflow.community.aio_sandbox.ownership.factory import resolve_ownership_redis_url +from deerflow.config.sandbox_config import SandboxOwnershipConfig + +logger = logging.getLogger(__name__) + +_SOCKET_TIMEOUT_SECONDS = 5.0 + +_LEDGER_SCRIPT = """ +local function now_ms() + local current = redis.call('TIME') + return tonumber(current[1]) * 1000 + math.floor(tonumber(current[2]) / 1000) +end + +local function initialize(hard_limit) + redis.call('HSET', KEYS[1], + 'meta:state', 'initializing', + 'meta:hard_limit', hard_limit, + 'meta:revision', '0') +end + +local function mark_present(sandbox_id) + local field = 's:' .. sandbox_id + if redis.call('HGET', KEYS[1], field) == '1' then + return false + end + redis.call('HSET', KEYS[1], field, '1') + return true +end + +local operation = ARGV[1] +local hard_limit = ARGV[2] +local state = redis.call('HGET', KEYS[1], 'meta:state') +local stored_limit = redis.call('HGET', KEYS[1], 'meta:hard_limit') +if state ~= false and stored_limit ~= hard_limit then + return redis.error_reply( + 'E2B capacity ledger configuration mismatch: configured hard_limit=' + .. hard_limit .. ', ledger hard_limit=' .. (stored_limit or '') + ) +end + +if operation == 'revision' then + return tonumber(redis.call('HGET', KEYS[1], 'meta:revision') or '0') +end + +if operation == 'reserve' then + if state ~= 'ready' then + return 'NOT_READY' + end + local field = 'r:' .. ARGV[3] + if redis.call('HEXISTS', KEYS[1], field) == 1 then + return 'GRANTED' + end + if redis.call('HLEN', KEYS[1]) - 3 >= tonumber(hard_limit) then + return 'FULL' + end + redis.call('HSET', KEYS[1], field, tostring(now_ms())) + redis.call('HINCRBY', KEYS[1], 'meta:revision', 1) + return 'GRANTED' +end + +if operation == 'release' then + if state ~= false and redis.call('HDEL', KEYS[1], 's:' .. ARGV[3]) == 1 then + redis.call('HINCRBY', KEYS[1], 'meta:revision', 1) + end + return 'OK' +end + +if state == false then + if operation == 'reconcile' and tonumber(ARGV[3]) ~= 0 then + return 'STALE' + end + initialize(hard_limit) + state = 'initializing' +end + +if operation == 'track' then + local changed = false + if ARGV[3] ~= '' and redis.call('HDEL', KEYS[1], 'r:' .. ARGV[3]) == 1 then + changed = true + end + if mark_present(ARGV[4]) then + changed = true + end + if changed then + redis.call('HINCRBY', KEYS[1], 'meta:revision', 1) + end + return 'OK' +end + +if operation ~= 'reconcile' then + return redis.error_reply('unknown E2B capacity operation: ' .. operation) +end + +local expected_revision = tonumber(ARGV[3]) +local revision = tonumber(redis.call('HGET', KEYS[1], 'meta:revision') or '0') +if revision ~= expected_revision then + return 'STALE' +end + +local complete = ARGV[4] == '1' +local remote_ids = {} +local changed = false +for index = 6, #ARGV, 2 do + local sandbox_id = ARGV[index] + local token = ARGV[index + 1] + remote_ids[sandbox_id] = true + if token ~= '' and redis.call('HDEL', KEYS[1], 'r:' .. token) == 1 then + changed = true + end + if mark_present(sandbox_id) then + changed = true + end +end + +if complete then + local stale_before_ms = now_ms() - tonumber(ARGV[5]) + for _, field in ipairs(redis.call('HKEYS', KEYS[1])) do + local prefix = string.sub(field, 1, 2) + local identifier = string.sub(field, 3) + if prefix == 's:' and remote_ids[identifier] ~= true then + local missing_since_ms = tonumber(string.match(redis.call('HGET', KEYS[1], field) or '', '^m:(%d+)$')) + if missing_since_ms == nil then + redis.call('HSET', KEYS[1], field, 'm:' .. now_ms()) + changed = true + elseif missing_since_ms <= stale_before_ms then + redis.call('HDEL', KEYS[1], field) + changed = true + end + elseif prefix == 'r:' then + local created_ms = tonumber(redis.call('HGET', KEYS[1], field)) + if created_ms ~= nil and created_ms <= stale_before_ms then + redis.call('HDEL', KEYS[1], field) + changed = true + end + end + end + if state ~= 'ready' then + redis.call('HSET', KEYS[1], 'meta:state', 'ready') + changed = true + end +end + +if changed then + redis.call('HINCRBY', KEYS[1], 'meta:revision', 1) +end +return 'APPLIED' +""" + + +class CapacityBackendError(RuntimeError): + """Redis could not return a definitive capacity decision.""" + + +class ReserveStatus(enum.StrEnum): + GRANTED = "GRANTED" + FULL = "FULL" + NOT_READY = "NOT_READY" + + +def _text(value: object) -> str: + return value.decode() if isinstance(value, bytes) else str(value) + + +class RedisE2BCapacityStore: + """One capacity scope stored in one Redis Hash.""" + + def __init__( + self, + *, + redis_url: str, + hard_limit: int, + key_prefix: str = "deerflow:sandbox:owner", + ) -> None: + if hard_limit < 1: + raise ValueError("hard_limit must be at least 1") + try: + from redis import Redis + from redis.exceptions import RedisError + except ImportError: # pragma: no cover - optional extra + raise ImportError("Redis E2B capacity requires: cd backend && uv sync --extra redis") from None + + self._hard_limit = hard_limit + self._key = f"{key_prefix.rstrip(':')}:e2b-capacity" + self._redis_error = RedisError + self._redis = Redis.from_url( + redis_url, + decode_responses=True, + socket_timeout=_SOCKET_TIMEOUT_SECONDS, + socket_connect_timeout=_SOCKET_TIMEOUT_SECONDS, + ) + self._script = self._redis.register_script(_LEDGER_SCRIPT) + + @property + def key(self) -> str: + return self._key + + def _run(self, operation: str, *args: object) -> object: + try: + return self._script(keys=[self._key], args=[operation, self._hard_limit, *args]) + except self._redis_error as error: + raise CapacityBackendError(f"failed to {operation} E2B capacity in Redis: {error}") from error + + def revision(self) -> int: + return int(self._run("revision")) + + def reserve(self, token: str) -> ReserveStatus: + if not token: + raise ValueError("token must not be empty") + try: + return ReserveStatus(_text(self._run("reserve", token))) + except ValueError as error: + raise CapacityBackendError("unexpected E2B capacity reserve result") from error + + def track( + self, + sandbox_id: str, + *, + reservation_token: str | None = None, + ) -> None: + if not sandbox_id: + raise ValueError("sandbox_id must not be empty") + self._run("track", reservation_token or "", sandbox_id) + + def release(self, sandbox_id: str) -> None: + if sandbox_id: + self._run("release", sandbox_id) + + def reconcile( + self, + *, + expected_revision: int, + remote_sandboxes: dict[str, str | None], + complete: bool, + reservation_max_age_ms: int, + ) -> bool: + remote_args = [item for sandbox_id, token in remote_sandboxes.items() for item in (sandbox_id, token or "")] + status = _text( + self._run( + "reconcile", + expected_revision, + "1" if complete else "0", + reservation_max_age_ms, + *remote_args, + ) + ) + if status not in {"APPLIED", "STALE"}: + raise CapacityBackendError(f"unexpected E2B capacity reconciliation result: {status}") + return status == "APPLIED" + + def close(self) -> None: + try: + self._redis.close() + except Exception as error: # pragma: no cover - teardown best effort + logger.warning("Error closing E2B capacity Redis client: %s", error) + + +def make_e2b_capacity_store( + ownership: SandboxOwnershipConfig, + *, + hard_limit: int, +) -> RedisE2BCapacityStore | None: + """Enable the shared ledger only with Redis ownership.""" + if ownership.type == "memory": + return None + if ownership.type != "redis": + raise ValueError(f"Unknown sandbox ownership type: {ownership.type!r}") + logger.info("E2B deployment capacity: redis (key_prefix=%s, hard_limit=%d)", ownership.key_prefix, hard_limit) + return RedisE2BCapacityStore( + redis_url=resolve_ownership_redis_url(ownership), + hard_limit=hard_limit, + key_prefix=ownership.key_prefix, + ) diff --git a/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox_provider.py b/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox_provider.py index d4124b652..e0cfd4d91 100644 --- a/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox_provider.py +++ b/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox_provider.py @@ -15,6 +15,9 @@ provider fields during startup. overflow_policy: wait # wait | reject | burst (default: wait) acquire_timeout: 30 # seconds for ``wait`` policy (default: 30) burst_limit: 2 # extra slots for ``burst`` policy (default: 0) + ownership: + type: redis # shares ownership and capacity across Gateways + redis_url: redis://redis:6379/0 mounts: # one-shot uploads on sandbox start - host_path: /data/skills container_path: /home/user/skills @@ -44,6 +47,7 @@ from functools import partial from pathlib import Path from typing import Any +from e2b import SandboxQuery from e2b_code_interpreter import Sandbox as E2BClientSandbox from deerflow.config import get_app_config @@ -56,10 +60,16 @@ from ..aio_sandbox.ownership import ( OwnershipBackendError, RenewOutcome, SandboxOwnershipStore, + compute_lease_ttl, generate_owner_id, make_sandbox_ownership_store, resolve_ownership_config, ) +from .capacity import ( + CapacityBackendError, + ReserveStatus, + make_e2b_capacity_store, +) from .e2b_sandbox import DEFAULT_E2B_HOME_DIR, E2BSandbox, _is_sandbox_gone_error logger = logging.getLogger(__name__) @@ -77,6 +87,9 @@ DEFAULT_RECONCILIATION_ORPHAN_TTL_SECONDS = 3600.0 DEFAULT_RECONCILIATION_MAX_PAGES = 10 DEFAULT_RECONCILIATION_MAX_ITEMS = 200 DEFAULT_RECONCILIATION_MAX_SECONDS = 15.0 +# Twice the E2B SDK's 60-second create request timeout. Short ownership lease +# settings must not make an in-flight create look abandoned. +MIN_CAPACITY_RESERVATION_SECONDS = 120.0 # Hard upper bound for ``set_timeout`` (e2b currently caps at 24h on the # free plan; passing an excessive value is rejected by the control-plane). MAX_E2B_TIMEOUT = 24 * 60 * 60 @@ -88,6 +101,8 @@ META_KEY_THREAD = "deer_flow_thread" META_KEY_PROVIDER = "deer_flow_provider" META_KEY_GATEWAY = "deer_flow_gateway" META_KEY_CREATED_AT = "deer_flow_created_at" +META_KEY_CAPACITY_LEDGER = "deer_flow_capacity_ledger" +META_KEY_CAPACITY_RESERVATION = "deer_flow_capacity_reservation" META_VAL_PROVIDER = "e2b_sandbox_provider" E2B_EXTRA_CONFIG_KEYS = frozenset({"api_key", "domain", "home_dir", "template"}) @@ -168,6 +183,10 @@ class E2BSandboxProvider(SandboxProvider): self._ownership_config, owner_id=self._owner_id, ) + self._deployment_capacity = make_e2b_capacity_store( + self._ownership_config, + hard_limit=self._capacity_limit(), + ) if not self._ownership.supports_cross_process: logger.warning("E2B sandbox ownership is process-local. Multi-worker gateways must configure sandbox.ownership.type: redis for safe reconciliation.") @@ -285,6 +304,24 @@ class E2BSandboxProvider(SandboxProvider): def _stable_seed(thread_id: str, user_id: str) -> str: return hashlib.sha256(f"{user_id}:{thread_id}".encode()).hexdigest()[:16] + def _metadata_matches_capacity_ledger( + self, + metadata: dict[str, Any], + ) -> bool: + """Include this ledger and legacy sandboxes that predate the tag.""" + store = self._deployment_capacity + if store is None: + return True + remote_ledger = metadata.get(META_KEY_CAPACITY_LEDGER) + return remote_ledger in (None, "", store.key) + + @staticmethod + def _capacity_reservation_from_metadata( + metadata: dict[str, Any], + ) -> str | None: + token = metadata.get(META_KEY_CAPACITY_RESERVATION) + return token if isinstance(token, str) and token else None + # ── Signal / shutdown handling ─────────────────────────────────────── def _register_signal_handlers(self) -> None: @@ -390,6 +427,7 @@ class E2BSandboxProvider(SandboxProvider): sandbox.close() except Exception: pass + self._release_deployment_sandbox(sid) return None try: @@ -496,7 +534,7 @@ class E2BSandboxProvider(SandboxProvider): """ sandbox_cls = self._get_sandbox_cls() seed = self._stable_seed(thread_id, user_id) - entries, _ = self._list_remote_entries( + entries, _, _ = self._list_remote_entries( { META_KEY_PROVIDER: META_VAL_PROVIDER, META_KEY_USER: user_id, @@ -504,16 +542,21 @@ class E2BSandboxProvider(SandboxProvider): } ) candidates = sorted( - ((sandbox_id, metadata) for entry in entries if (sandbox_id := self._entry_id(entry)) and (metadata := self._entry_metadata(entry)).get(META_KEY_USER) == user_id and metadata.get(META_KEY_THREAD) == thread_id), + ( + (sandbox_id, metadata) + for entry in entries + if (sandbox_id := self._entry_id(entry)) and (metadata := self._entry_metadata(entry)).get(META_KEY_USER) == user_id and metadata.get(META_KEY_THREAD) == thread_id and self._metadata_matches_capacity_ledger(metadata) + ), key=lambda item: (item[1].get(META_KEY_CREATED_AT, ""), item[0]), ) - for target_id, _metadata in candidates: + for target_id, metadata in candidates: adopted = self._adopt_remote_candidate( sandbox_cls, target_id, thread_id=thread_id, user_id=user_id, seed=seed, + capacity_reservation=(self._capacity_reservation_from_metadata(metadata)), ) if adopted is not None: return adopted @@ -527,6 +570,7 @@ class E2BSandboxProvider(SandboxProvider): thread_id: str, user_id: str, seed: str, + capacity_reservation: str | None = None, ) -> str | None: """Try to adopt one discovered candidate without harming peer-owned VMs.""" @@ -548,6 +592,10 @@ class E2BSandboxProvider(SandboxProvider): return None try: + self._track_deployment_sandbox( + target_id, + reservation_token=capacity_reservation, + ) self._reserve_capacity( thread_id, user_id, @@ -664,6 +712,66 @@ class E2BSandboxProvider(SandboxProvider): with self._lock: self._end_transition_locked() + def _capacity_reservation_max_age_ms(self) -> int: + configured = compute_lease_ttl(self._ownership_config) + float(self._config["reconciliation_grace_seconds"]) + return int(max(configured, MIN_CAPACITY_RESERVATION_SECONDS) * 1_000) + + def _capacity_error( + self, + message: str, + *, + reason: str = "capacity", + ) -> SandboxCapacityExceededError: + with self._lock: + return SandboxCapacityExceededError( + message, + active=len(self._sandboxes), + warm=len(self._warm_pool), + reserved=self._reserved_slots, + replicas=int(self._config["replicas"]), + reason=reason, + ) + + def _track_deployment_sandbox( + self, + sandbox_id: str, + *, + reservation_token: str | None = None, + required: bool = True, + ) -> None: + store = self._deployment_capacity + if store is None: + return + try: + store.track( + sandbox_id, + reservation_token=reservation_token, + ) + except CapacityBackendError as error: + if required: + raise self._capacity_error( + f"Deployment-wide E2B capacity is unavailable; cannot safely track sandbox {sandbox_id}", + reason="capacity_backend", + ) from error + logger.warning( + "Could not track E2B sandbox %s in deployment capacity; reconciliation will retry: %s", + sandbox_id, + error, + ) + + def _release_deployment_sandbox(self, sandbox_id: str) -> None: + store = self._deployment_capacity + if store is None: + return + try: + store.release(sandbox_id) + except CapacityBackendError as error: + logger.warning( + "Could not release deployment capacity for destroyed E2B sandbox %s; reconciliation will retry: %s", + sandbox_id, + error, + ) + def _reserve_capacity( self, thread_id: str | None, @@ -671,23 +779,71 @@ class E2BSandboxProvider(SandboxProvider): *, remote_id: str | None = None, remote_owned: bool = True, - ) -> None: - """Acquire a capacity slot, blocking or raising as configured. - - Must be called before ``Sandbox.create()``. The caller MUST call - ``_commit_capacity()`` on success or ``_release_capacity()`` on - failure — otherwise the reserved slot is leaked until shutdown. - - Raises: - SandboxCapacityExceededError: when the overflow policy is - ``reject`` or the wait timeout expires. - """ + ) -> str | None: + """Reserve local capacity, then deployment capacity for a new VM.""" + store = self._deployment_capacity policy = self._config["overflow_policy"] timeout = float(self._config["acquire_timeout"]) deadline = time.monotonic() + timeout + token = uuid.uuid4().hex if store is not None and remote_id is None else None + + while True: + self._reserve_local_capacity( + thread_id, + user_id, + remote_id=remote_id, + remote_owned=remote_owned, + deadline=deadline, + ) + if store is None or remote_id is not None: + return token + + assert token is not None + backend_error = None + try: + status = store.reserve(token) + except CapacityBackendError as error: + backend_error = error + status = None + if status is ReserveStatus.GRANTED: + return token + self._release_capacity() + if status is ReserveStatus.FULL and self._evict_oldest_warm() is not None: + continue + + if policy != "wait": + if backend_error is not None: + raise self._capacity_error( + "Deployment-wide E2B capacity is unavailable", + reason="capacity_backend", + ) from backend_error + if status is ReserveStatus.NOT_READY: + raise self._capacity_error( + "Deployment-wide E2B capacity is initializing", + reason="capacity_initializing", + ) + raise self._capacity_error("Deployment-wide E2B capacity is full") + remaining = deadline - time.monotonic() + if remaining <= 0: + raise self._capacity_error(f"Timed out after {timeout}s waiting for deployment-wide E2B capacity") + with self._capacity_cond: + self._capacity_cond.wait(timeout=min(remaining, 1.0)) + + def _reserve_local_capacity( + self, + thread_id: str | None, + user_id: str, + *, + remote_id: str | None = None, + remote_owned: bool = True, + deadline: float | None = None, + ) -> None: + """Acquire the existing process-local lifecycle slot.""" + policy = self._config["overflow_policy"] + timeout = float(self._config["acquire_timeout"]) + deadline = deadline or time.monotonic() + timeout while True: - # Reject immediately if the provider is shutting down. with self._lock: if self._shutdown_called: raise SandboxCapacityExceededError( @@ -697,7 +853,6 @@ class E2BSandboxProvider(SandboxProvider): reason="shutdown", ) - # 1. Try immediate atomic reservation. with self._lock: if self._shutdown_called: raise SandboxCapacityExceededError( @@ -714,7 +869,6 @@ class E2BSandboxProvider(SandboxProvider): remote_ops.add(remote_id) return - # 2. Try evicting a warm entry to free a slot. evicted = self._evict_oldest_warm() if evicted is not None: with self._lock: @@ -731,9 +885,7 @@ class E2BSandboxProvider(SandboxProvider): remote_ops = self._remote_ops_in_progress if remote_owned else self._unowned_remote_ops_in_progress remote_ops.add(remote_id) return - # Slot was stolen; fall through to policy / wait. - # 3. Apply overflow policy. with self._lock: if self._shutdown_called: raise SandboxCapacityExceededError( @@ -764,7 +916,6 @@ class E2BSandboxProvider(SandboxProvider): replicas=int(self._config["replicas"]), ) - # policy == "wait": block until a slot frees or timeout. remaining = deadline - time.monotonic() if remaining <= 0: raise SandboxCapacityExceededError( @@ -839,7 +990,7 @@ class E2BSandboxProvider(SandboxProvider): Capacity is enforced atomically via :meth:`_reserve_capacity`. """ - self._reserve_capacity(thread_id, user_id) + reservation_token = self._reserve_capacity(thread_id, user_id) sandbox_cls = self._get_sandbox_cls() metadata: dict[str, str] = { @@ -847,6 +998,10 @@ class E2BSandboxProvider(SandboxProvider): META_KEY_GATEWAY: self._owner_id, META_KEY_CREATED_AT: str(time.time()), } + if self._deployment_capacity is not None: + metadata[META_KEY_CAPACITY_LEDGER] = self._deployment_capacity.key + if reservation_token is not None: + metadata[META_KEY_CAPACITY_RESERVATION] = reservation_token if thread_id: metadata[META_KEY_USER] = user_id metadata[META_KEY_THREAD] = thread_id @@ -869,6 +1024,11 @@ class E2BSandboxProvider(SandboxProvider): raise sandbox_id: str = getattr(client, "sandbox_id", None) or str(uuid.uuid4())[:8] + self._track_deployment_sandbox( + sandbox_id, + reservation_token=reservation_token, + required=False, + ) if not self._track_reserved_remote_op(sandbox_id): kill_error = self._kill_client(client) cleanup_confirmed = kill_error is None @@ -998,26 +1158,25 @@ class E2BSandboxProvider(SandboxProvider): value = entry.get("metadata") return value if isinstance(value, dict) else {} - def _list_remote_entries(self, metadata: dict[str, str]) -> tuple[list[Any], bool]: - """List matching E2B entries within configured page/item/time budgets.""" + def _list_remote_entries( + self, + metadata: dict[str, str], + ) -> tuple[list[Any], bool, bool]: + """List E2B entries and report budget exhaustion and completeness.""" sandbox_cls = self._get_sandbox_cls() try: - result = sandbox_cls.list(query={"metadata": metadata}, **self._common_kwargs()) # type: ignore[attr-defined] - except TypeError: - try: - result = sandbox_cls.list(metadata=metadata, **self._common_kwargs()) # type: ignore[attr-defined] - except Exception as e: - logger.warning("E2B reconciliation list failed: %s", e) - return [], False + query = SandboxQuery(metadata=metadata) + result = sandbox_cls.list(query=query, **self._common_kwargs()) # type: ignore[attr-defined] except Exception as e: logger.warning("E2B reconciliation list failed: %s", e) - return [], False + return [], False, False max_pages = int(self._config["reconciliation_max_pages"]) max_items = int(self._config["reconciliation_max_items"]) deadline = time.monotonic() + float(self._config["reconciliation_max_seconds"]) entries: list[Any] = [] exhausted = False + complete = True if hasattr(result, "next_items") and hasattr(result, "has_next"): for page_number in range(max_pages): @@ -1028,6 +1187,7 @@ class E2BSandboxProvider(SandboxProvider): page = result.next_items() except Exception as e: logger.warning("E2B reconciliation paginator failed: %s", e) + complete = False break if not page: break @@ -1045,13 +1205,15 @@ class E2BSandboxProvider(SandboxProvider): all_entries = list(result or []) except TypeError: logger.warning("E2B Sandbox.list returned non-iterable %s", type(result).__name__) - return [], False + return [], False, False entries = all_entries[:max_items] exhausted = len(all_entries) > max_items if time.monotonic() >= deadline: exhausted = True - return entries, exhausted + if exhausted: + complete = False + return entries, exhausted, complete def _publish_ownership(self, sandbox_id: str) -> None: """Publish acquire-side ownership before exposing a sandbox locally.""" @@ -1147,8 +1309,21 @@ class E2BSandboxProvider(SandboxProvider): self._lease_thread.start() self._reconcile_thread.start() - def _reserve_reconciliation_capacity(self, sandbox_id: str) -> bool: + def _reserve_reconciliation_capacity( + self, + sandbox_id: str, + *, + reservation_token: str | None = None, + ) -> bool: """Reserve one local slot for adoption without blocking maintenance.""" + try: + self._track_deployment_sandbox( + sandbox_id, + reservation_token=reservation_token, + ) + except SandboxCapacityExceededError as error: + logger.warning("Could not track discovered E2B sandbox %s: %s", sandbox_id, error) + return False with self._lock: if self._shutdown_called or self._total_capacity_used_locked() >= self._capacity_limit(): return False @@ -1162,8 +1337,38 @@ class E2BSandboxProvider(SandboxProvider): observed_at = time.monotonic() if now is None else now deadline = time.monotonic() + float(self._config["reconciliation_max_seconds"]) stats = ReconciliationStats() - entries, stats.budget_exhausted = self._list_remote_entries({META_KEY_PROVIDER: META_VAL_PROVIDER}) + capacity_revision = None + capacity_store = self._deployment_capacity + if capacity_store is not None: + try: + capacity_revision = capacity_store.revision() + except CapacityBackendError as error: + logger.warning( + "Could not read E2B capacity before reconciliation: %s", + error, + ) + + entries, stats.budget_exhausted, inventory_complete = self._list_remote_entries({META_KEY_PROVIDER: META_VAL_PROVIDER}) + entries = [entry for entry in entries if self._metadata_matches_capacity_ledger(self._entry_metadata(entry))] stats.discovered = len(entries) + + if capacity_store is not None and capacity_revision is not None: + records = {sandbox_id: self._capacity_reservation_from_metadata(self._entry_metadata(entry)) for entry in entries if (sandbox_id := self._entry_id(entry))} + try: + applied = capacity_store.reconcile( + expected_revision=capacity_revision, + remote_sandboxes=records, + complete=inventory_complete, + reservation_max_age_ms=self._capacity_reservation_max_age_ms(), + ) + if not applied: + logger.debug("E2B capacity inventory became stale during reconciliation; retrying on the next pass") + except CapacityBackendError as error: + logger.warning( + "Could not apply E2B capacity inventory: %s", + error, + ) + groups: dict[tuple[str, str], list[tuple[str, dict[str, Any]]]] = {} orphans: list[tuple[str, dict[str, Any]]] = [] present_ids: set[str] = set() @@ -1206,12 +1411,15 @@ class E2BSandboxProvider(SandboxProvider): if not live: continue stats.duplicates += max(0, len(live) - 1) - canonical_id, _metadata, canonical_client = live[0] + canonical_id, canonical_metadata, canonical_client = live[0] with self._lock: already_local = canonical_id in self._sandboxes if already_local: self._safe_close_client(canonical_client) - elif not self._reserve_reconciliation_capacity(canonical_id): + elif not self._reserve_reconciliation_capacity( + canonical_id, + reservation_token=self._capacity_reservation_from_metadata(canonical_metadata), + ): self._safe_close_client(canonical_client) stats.deferred += 1 elif not self._claim_ownership(canonical_id): @@ -1332,10 +1540,16 @@ class E2BSandboxProvider(SandboxProvider): reaped the VM. Closing that host-side client before returning ``None`` keeps both acquire paths from leaking a connection. """ - client = self._reconnect_client(sandbox_cls, sandbox_id) + try: + client = self._reconnect_client(sandbox_cls, sandbox_id) + except Exception as error: + if _is_sandbox_gone_error(error): + self._release_deployment_sandbox(sandbox_id) + raise if self._client_alive(client): return client self._safe_close_client(client) + self._release_deployment_sandbox(sandbox_id) return None def _register_connected_sandbox( @@ -2073,13 +2287,14 @@ class E2BSandboxProvider(SandboxProvider): except Exception: pass - @staticmethod def _kill_client( + self, client: E2BClientSandbox | None, ) -> Exception | None: """Kill a remote VM and return an exception for the caller to log.""" if client is None: return RuntimeError("Cannot confirm remote VM destruction without a client") + sandbox_id = getattr(client, "sandbox_id", None) try: kill = getattr(client, "kill", None) if not callable(kill): @@ -2087,6 +2302,8 @@ class E2BSandboxProvider(SandboxProvider): kill() except Exception as e: return e + if sandbox_id is not None: + self._release_deployment_sandbox(sandbox_id) return None def reset(self) -> None: @@ -2178,3 +2395,11 @@ class E2BSandboxProvider(SandboxProvider): self._ownership.close() except Exception as e: logger.warning("Failed to close E2B ownership store: %s", e) + if self._deployment_capacity is not None: + try: + self._deployment_capacity.close() + except Exception as e: + logger.warning( + "Failed to close E2B deployment capacity store: %s", + e, + ) diff --git a/backend/packages/harness/deerflow/config/sandbox_config.py b/backend/packages/harness/deerflow/config/sandbox_config.py index 732c47621..f6c8f10f8 100644 --- a/backend/packages/harness/deerflow/config/sandbox_config.py +++ b/backend/packages/harness/deerflow/config/sandbox_config.py @@ -76,8 +76,9 @@ class SandboxConfig(BaseModel): AioSandboxProvider, BoxliteProvider, and E2BSandboxProvider shared options: image: Sandbox image to use (Docker/AIO image or BoxLite OCI image) - replicas: Positive provider capacity per gateway process. Each provider - defines which lifecycle states count toward this limit. + replicas: Positive provider capacity. E2B shares it across Gateway + workers when ownership uses Redis; other modes/providers keep + process-local accounting. idle_timeout: Idle timeout in seconds before released warm sandboxes/VMs are stopped (default: 600 = 10 minutes). Set to 0 to disable. environment: Environment variables to inject into the sandbox (values starting with $ are resolved from host env) @@ -115,7 +116,7 @@ class SandboxConfig(BaseModel): replicas: int | None = Field( default=None, gt=0, - description="Positive provider capacity per gateway process. Each provider defines which lifecycle states count toward this limit.", + description=("Positive provider capacity. E2B enforces it deployment-wide when sandbox ownership uses Redis; otherwise accounting is per Gateway process. Each provider defines which lifecycle states count."), ) overflow_policy: SandboxOverflowPolicy = Field( default="wait", diff --git a/backend/tests/test_e2b_capacity_store_redis.py b/backend/tests/test_e2b_capacity_store_redis.py new file mode 100644 index 000000000..65e59ae06 --- /dev/null +++ b/backend/tests/test_e2b_capacity_store_redis.py @@ -0,0 +1,178 @@ +"""Integration tests for deployment-wide E2B admission.""" + +from __future__ import annotations + +import os +import threading +import uuid +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from deerflow.community.e2b_sandbox.capacity import ( + CapacityBackendError, + RedisE2BCapacityStore, + ReserveStatus, + make_e2b_capacity_store, +) +from deerflow.config.sandbox_config import SandboxOwnershipConfig + +REDIS_URL = os.environ.get("DEER_FLOW_TEST_REDIS_URL", "redis://localhost:6379/15") +pytestmark = pytest.mark.integration + + +@pytest.fixture +def make_store(): + redis = pytest.importorskip("redis") + probe = redis.Redis.from_url(REDIS_URL, decode_responses=True, socket_connect_timeout=0.5) + try: + probe.ping() + except Exception: + probe.close() + pytest.skip(f"Redis not reachable at {REDIS_URL}") + + prefix = f"deerflow:test:{uuid.uuid4().hex}" + stores = [] + + def make(hard_limit: int = 1): + store = RedisE2BCapacityStore( + redis_url=REDIS_URL, + hard_limit=hard_limit, + key_prefix=prefix, + ) + stores.append(store) + return store + + try: + yield make + finally: + probe.delete(f"{prefix}:e2b-capacity") + probe.close() + for store in stores: + store.close() + + +def _initialize(store) -> None: + assert store.reconcile( + expected_revision=store.revision(), + remote_sandboxes={}, + complete=True, + reservation_max_age_ms=0, + ) + + +def _counts(store) -> tuple[int, int]: + fields = store._redis.hkeys(store.key) + return ( + sum(field.startswith("s:") for field in fields), + sum(field.startswith("r:") for field in fields), + ) + + +def test_factory_is_lazy_and_backend_errors_fail_closed() -> None: + assert make_e2b_capacity_store(SandboxOwnershipConfig(type="memory"), hard_limit=3) is None + store = make_e2b_capacity_store( + SandboxOwnershipConfig(type="redis", redis_url="redis://127.0.0.1:1/0", key_prefix="test"), + hard_limit=3, + ) + assert store is not None and store.key == "test:e2b-capacity" + try: + with pytest.raises(CapacityBackendError): + store.reserve("reservation") + finally: + store.close() + + +def test_two_gateways_atomically_share_one_hash(make_store) -> None: + gateway_a, gateway_b = make_store(), make_store() + assert gateway_a.reserve("not-ready") is ReserveStatus.NOT_READY + _initialize(gateway_a) + barrier = threading.Barrier(2) + + def reserve(args): + store, token = args + barrier.wait() + return store.reserve(token) + + tokens = ["reservation-a", "reservation-b"] + with ThreadPoolExecutor(max_workers=2) as executor: + results = list(executor.map(reserve, zip((gateway_a, gateway_b), tokens))) + + assert results.count(ReserveStatus.GRANTED) == 1 + assert results.count(ReserveStatus.FULL) == 1 + assert list(gateway_a._redis.scan_iter(f"{gateway_a.key}")) == [gateway_a.key] + winner = tokens[results.index(ReserveStatus.GRANTED)] + gateway_a.track("sandbox-a", reservation_token=winner) + gateway_a.track("sandbox-a", reservation_token=winner) + assert _counts(gateway_a) == (1, 0) + # A successful but stale list must not release a just-tracked slot. + assert gateway_b.reconcile( + expected_revision=gateway_b.revision(), + remote_sandboxes={}, + complete=True, + reservation_max_age_ms=120_000, + ) + assert gateway_b.reserve("stale-inventory") is ReserveStatus.FULL + gateway_b.release("sandbox-a") + gateway_b.release("sandbox-a") + assert _counts(gateway_a) == (0, 0) + + +def test_reconcile_repairs_crashes_without_erasing_concurrent_changes(make_store) -> None: + gateway_a, gateway_b = make_store(2), make_store(2) + _initialize(gateway_a) + assert gateway_a.reserve("crashed-create") is ReserveStatus.GRANTED + assert gateway_b.reconcile( + expected_revision=gateway_b.revision(), + remote_sandboxes={"sandbox-a": "crashed-create"}, + complete=True, + reservation_max_age_ms=0, + ) + stale_revision = gateway_a.revision() + assert gateway_b.reserve("concurrent") is ReserveStatus.GRANTED + assert not gateway_a.reconcile( + expected_revision=stale_revision, + remote_sandboxes={}, + complete=True, + reservation_max_age_ms=0, + ) + assert _counts(gateway_a) == (1, 1) + + +def test_reconcile_keeps_incomplete_inventory_and_fresh_reservations(make_store) -> None: + store = make_store(2) + _initialize(store) + store.track("sandbox-a") + assert store.reserve("creating") is ReserveStatus.GRANTED + + assert store.reconcile( + expected_revision=store.revision(), + remote_sandboxes={}, + complete=False, + reservation_max_age_ms=0, + ) + assert _counts(store) == (1, 1) + assert store.reconcile( + expected_revision=store.revision(), + remote_sandboxes={"sandbox-a": None}, + complete=True, + reservation_max_age_ms=120_000, + ) + assert _counts(store) == (1, 1) + store._redis.hset(store.key, "s:sandbox-a", "m:0") + assert store.reconcile( + expected_revision=store.revision(), + remote_sandboxes={}, + complete=True, + reservation_max_age_ms=0, + ) + assert _counts(store) == (0, 0) + + +def test_mismatched_hard_limits_fail_closed(make_store) -> None: + gateway_a, gateway_b = make_store(), make_store(2) + _initialize(gateway_a) + with pytest.raises(CapacityBackendError, match="configuration mismatch"): + gateway_b.revision() + with pytest.raises(CapacityBackendError, match="configuration mismatch"): + gateway_b.reserve("reservation") diff --git a/backend/tests/test_e2b_sandbox_provider.py b/backend/tests/test_e2b_sandbox_provider.py index cd3aa865f..e49a5f435 100644 --- a/backend/tests/test_e2b_sandbox_provider.py +++ b/backend/tests/test_e2b_sandbox_provider.py @@ -17,6 +17,10 @@ from unittest.mock import MagicMock import pytest from pydantic import ValidationError +from deerflow.community.e2b_sandbox.capacity import ( + CapacityBackendError, + ReserveStatus, +) from deerflow.config.paths import Paths from deerflow.config.sandbox_config import SandboxConfig from deerflow.sandbox.exceptions import SandboxCapacityExceededError @@ -253,7 +257,12 @@ def _make_provider(*, replicas: int = 3, idle_timeout: int = 1800, overflow_poli provider._shutdown_called = False provider._owner_id = "owner-a" provider._ownership = FakeOwnershipStore({}, owner_id=provider._owner_id) - provider._ownership_config = SimpleNamespace(renewal_interval_seconds=60.0) + provider._ownership_config = SimpleNamespace( + renewal_interval_seconds=60.0, + ttl_multiplier=4.0, + key_prefix="deerflow:test", + ) + provider._deployment_capacity = None provider._owned_sandbox_ids = set() provider._acquire_inflight = set() provider._orphan_first_seen = {} @@ -282,6 +291,22 @@ def _make_provider(*, replicas: int = 3, idle_timeout: int = 1800, overflow_poli return provider +def _install_shared_deployment_capacity( + *providers, + reserve_results: list[ReserveStatus] | None = None, +) -> MagicMock: + store = MagicMock() + store.key = "deerflow:test:e2b-capacity" + store.revision.return_value = 0 + store.reserve.return_value = ReserveStatus.GRANTED + store.reconcile.return_value = True + if reserve_results is not None: + store.reserve.side_effect = reserve_results + for provider in providers: + provider._deployment_capacity = store + return store + + def _install_fake_sdk(monkeypatch, provider) -> FakeSandboxClass: fake_cls = FakeSandboxClass() monkeypatch.setattr(provider, "_get_sandbox_cls", lambda: fake_cls) @@ -1081,11 +1106,17 @@ def test_bootstrap_failure_does_not_kill_without_destroy_lease(monkeypatch): def test_kill_client_returns_exception_without_raising(): p = _make_provider() - client = FakeClient() + store = _install_shared_deployment_capacity(p) + failed_client = FakeClient() error = RuntimeError("already gone") - client.kill = MagicMock(side_effect=error) + failed_client.kill = MagicMock(side_effect=error) - assert p._kill_client(client) is error + assert p._kill_client(failed_client) is error + store.release.assert_not_called() + + client = FakeClient() + assert p._kill_client(client) is None + store.release.assert_called_once_with(client.sandbox_id) def test_kill_client_reports_uncertain_cleanup_without_callable_kill(): @@ -2107,9 +2138,132 @@ def test_grep_single_file_path_with_matching_glob(): assert truncated is False -# ────────────────────────────────────────────────────────────────────────────── # Capacity enforcement tests (#4339) -# ────────────────────────────────────────────────────────────────────────────── + + +def test_deployment_capacity_reserves_commits_and_rejects_globally(monkeypatch) -> None: + gateway_a = _make_provider(replicas=1, overflow_policy="reject") + gateway_b = _make_provider(replicas=1, overflow_policy="reject") + store = _install_shared_deployment_capacity( + gateway_a, + gateway_b, + reserve_results=[ReserveStatus.GRANTED, ReserveStatus.FULL], + ) + sdk_a = _install_fake_sdk(monkeypatch, gateway_a) + sdk_b = FakeSandboxClass() + monkeypatch.setattr(gateway_b, "_get_sandbox_cls", lambda: sdk_b) + + sandbox_id = gateway_a.acquire("thread-a", user_id="user-a") + with pytest.raises(SandboxCapacityExceededError): + gateway_b.acquire("thread-b", user_id="user-b") + + metadata = sdk_a.create_calls[0]["metadata"] + assert metadata["deer_flow_capacity_ledger"] == store.key + assert metadata["deer_flow_capacity_reservation"] + store.track.assert_called_once_with( + sandbox_id, + reservation_token=metadata["deer_flow_capacity_reservation"], + ) + assert len(sdk_a.create_calls) == 1 + assert sdk_b.create_calls == [] + assert store.reserve.call_count == 2 + + +def test_ambiguous_create_failure_retains_deployment_reservation(monkeypatch) -> None: + provider = _make_provider(replicas=1, overflow_policy="reject") + store = _install_shared_deployment_capacity(provider) + sdk = _install_fake_sdk(monkeypatch, provider) + sdk.create_factory = lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("control-plane timeout")) + + with pytest.raises(RuntimeError, match="control-plane timeout"): + provider.acquire("thread-a", user_id="user-a") + + assert provider._reserved_slots == 0 + store.reserve.assert_called_once() + store.track.assert_not_called() + store.release.assert_not_called() + + +def test_discovery_uses_sdk_query_and_tracks_without_reserving(monkeypatch) -> None: + provider = _make_provider(replicas=1, overflow_policy="reject") + store = _install_shared_deployment_capacity(provider) + sdk = _install_fake_sdk(monkeypatch, provider) + entry = SimpleNamespace( + sandbox_id="sandbox-existing", + metadata={ + "deer_flow_provider": "e2b_sandbox_provider", + "deer_flow_user": "user-a", + "deer_flow_thread": "thread-a", + "deer_flow_capacity_ledger": store.key, + }, + ) + expected_query = {key: entry.metadata[key] for key in ("deer_flow_provider", "deer_flow_user", "deer_flow_thread")} + sdk.list_return = SimpleNamespace( + has_next=False, + next_items=lambda: [entry] if sdk.list_calls[-1]["query"].metadata == expected_query else [], + ) + assert provider.acquire("thread-a", user_id="user-a") == entry.sandbox_id + assert sdk.create_calls == [] + store.reserve.assert_not_called() + store.track.assert_called_once_with(entry.sandbox_id, reservation_token=None) + + +def test_reconciliation_repairs_crash_and_uses_safe_reservation_age(monkeypatch) -> None: + provider = _make_provider(replicas=1, overflow_policy="reject") + provider._ownership_config.renewal_interval_seconds = 1.0 + provider._ownership_config.ttl_multiplier = 2.0 + provider._config["reconciliation_grace_seconds"] = 0.0 + store = _install_shared_deployment_capacity(provider) + sdk = _install_fake_sdk(monkeypatch, provider) + sdk.list_return = [ + { + "sandbox_id": "sandbox-existing", + "metadata": { + "deer_flow_provider": "e2b_sandbox_provider", + "deer_flow_capacity_ledger": store.key, + "deer_flow_capacity_reservation": "reservation-crashed", + }, + }, + { + "sandbox_id": "sandbox-other-deployment", + "metadata": { + "deer_flow_provider": "e2b_sandbox_provider", + "deer_flow_capacity_ledger": "deerflow:other:e2b-capacity", + }, + }, + ] + + stats = provider._reconcile_remote_sandboxes(now=100.0) + args = store.reconcile.call_args.kwargs + assert stats.discovered == 1 + assert args["remote_sandboxes"] == {"sandbox-existing": "reservation-crashed"} + assert args["complete"] is True + assert args["reservation_max_age_ms"] == 120_000 + + +def test_failed_inventory_and_redis_error_both_prevent_create(monkeypatch) -> None: + provider = _make_provider(replicas=1, overflow_policy="reject") + store = _install_shared_deployment_capacity( + provider, + reserve_results=[ReserveStatus.NOT_READY], + ) + sdk = _install_fake_sdk(monkeypatch, provider) + sdk.list = MagicMock(side_effect=RuntimeError("E2B unavailable")) + + provider._reconcile_remote_sandboxes(now=100.0) + + reconcile_args = store.reconcile.call_args.kwargs + assert reconcile_args["complete"] is False + assert reconcile_args["remote_sandboxes"] == {} + with pytest.raises(SandboxCapacityExceededError): + provider._create_sandbox("thread-a", user_id="user-a") + store.reserve.side_effect = CapacityBackendError("Redis unavailable") + with pytest.raises(SandboxCapacityExceededError) as error: + provider._create_sandbox("thread-a", user_id="user-a") + + assert error.value.reason == "capacity_backend" + assert sdk.create_calls == [] + assert provider._reserved_slots == 0 def test_capacity_reject_policy_raises_when_full(monkeypatch): @@ -2146,7 +2300,11 @@ def test_capacity_reject_frees_slot_on_release(monkeypatch): def test_capacity_reject_evicts_other_thread_warm_entry_before_create(monkeypatch): """Reject policy can evict one warm VM before it rejects new capacity.""" - p = _make_provider(replicas=1, overflow_policy="reject") + p = _make_provider(replicas=3, overflow_policy="reject") + store = _install_shared_deployment_capacity( + p, + reserve_results=[ReserveStatus.GRANTED, ReserveStatus.FULL, ReserveStatus.GRANTED], + ) fake_cls = _install_fake_sdk(monkeypatch, p) sid1 = p.acquire("t1", user_id="u1") @@ -2158,6 +2316,7 @@ def test_capacity_reject_evicts_other_thread_warm_entry_before_create(monkeypatc assert sid2 != sid1 assert len(p._warm_pool) == 0 assert len(fake_cls.create_calls) == 2 + store.release.assert_called_once_with(sid1) def test_capacity_wait_policy_times_out(monkeypatch): @@ -3330,8 +3489,14 @@ def test_shutdown_during_discovery_does_not_kill_unowned_vm(monkeypatch): allow_commit = threading.Event() reserve_capacity = p._reserve_capacity - def pause_after_reserve(thread_id, user_id, *, remote_id=None, remote_owned=True): - reserve_capacity( + def pause_after_reserve( + thread_id, + user_id, + *, + remote_id=None, + remote_owned=True, + ): + reservation = reserve_capacity( thread_id, user_id, remote_id=remote_id, @@ -3339,6 +3504,7 @@ def test_shutdown_during_discovery_does_not_kill_unowned_vm(monkeypatch): ) reserved.set() assert allow_commit.wait(timeout=2) + return reservation monkeypatch.setattr(p, "_reserve_capacity", pause_after_reserve) result: list[str | None] = [] From 80848837b7d059610dc24055684f9753ae452192 Mon Sep 17 00:00:00 2001 From: Vanzeren <53075619+Vanzeren@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:25:07 +0800 Subject: [PATCH 03/31] feat(gateway): seed checkpoint history (#4590) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(runtime): seed empty run-event feed from checkpoint history Threads created before the journaled run-event model hold their history only in the LangGraph checkpoint. Before the first journaled run, backfill an empty run-event message feed from the existing checkpoint head so legacy history receives earlier thread-global seq numbers and remains visible in the unified feed. Threads with no checkpoint or an already populated feed skip the path. The seed guard resolves the user explicitly instead of relying on the store's AUTO default, which raises without a user contextvar (scheduler launch path on the DB event store). * docs: document checkpoint history seeding in thread runs Before the first journaled run, an empty run-event message feed is seeded from an existing checkpoint head so legacy checkpoint-only history stays visible with earlier thread-global sequence numbers. * fix(gateway): make checkpoint-history seed guard thread-scoped The emptiness guard filtered by the current user whenever one was in context, answering "does this user have any messages?" rather than "has this thread's feed ever been journaled?". Seed rows stamped with a different principal (NULL for ownerless seeds, or another user on a shared NULL-owner thread) were invisible to the guard, so each new principal re-seeded a duplicate history. Pass user_id=None unconditionally; None also opts out of AUTO resolution, so the ownerless scheduler path still cannot raise. Adds a DbRunEventStore-backed regression test (the MemoryRunEventStore tests cannot catch this — the memory store ignores user_id) proving the ownerless-seed -> authenticated-run sequence seeds exactly once. --- backend/AGENTS.md | 2 +- backend/app/gateway/services.py | 63 ++++ .../harness/deerflow/runtime/journal.py | 62 +++- backend/tests/test_gateway_services.py | 287 +++++++++++++++++- .../test_stateless_runs_owner_isolation.py | 5 + 5 files changed, 395 insertions(+), 24 deletions(-) diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 2ce7a1e60..1f303d537 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -495,7 +495,7 @@ Localhost persistence deliberately reads the direct request `Host` and ignores ` | **Artifacts** (`/api/threads/{id}/artifacts`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types | | **Suggestions** (`/api/suggestions`) | `GET /config` - returns global suggestions config boolean; `POST /threads/{id}/suggestions` - generate follow-up questions; rich list/block model content is normalized and inline reasoning (`...`, including unclosed/truncated blocks from reasoning models like MiniMax-M3) is stripped before JSON parsing | | **Input Polish** (`/api/input-polish`) | `POST /` - rewrite a composer draft before it is sent. This is a short authenticated `runs:create` LLM request using `input_polish` config; it does not create a LangGraph run, persist a message, or modify thread state. Shares the non-graph one-shot LLM path (`deerflow.utils.oneshot_llm.run_oneshot_llm`) with the suggestions route so model build + Langfuse metadata + invoke stay in one place; validates the same stripped view of the draft it sends to the model, and preserves literal `` substrings in the rewrite (`strip_think_blocks(truncate_unclosed=False)`) | -| **Thread Runs** (`/api/threads/{id}/runs`) | `POST /` - create background run; `POST /stream` - create + SSE stream; `POST /wait` - create + block; `POST /regenerate/prepare` - prepare clean input + checkpoint metadata for regenerating the latest completed or interrupted assistant answer, carrying the latest non-empty thread title in graph input so resuming an older checkpoint cannot roll back a later manual rename (#4457); `POST /edit-regenerate/prepare` - prepare a checkpoint replay from the latest editable human turn with a replacement user message and edit replay metadata; it carries the current thread title the same way, but only when the replay base already has one — an untitled base belongs to a thread the title middleware has not named yet, so pinning the current title there would keep a name generated from the prompt the edit just replaced; `GET /` - list runs; `GET /{rid}` - run details; `POST /{rid}/cancel` - cancel; `GET /{rid}/join` - join SSE; `GET /{rid}/messages` - paginated per-run messages `{data, has_more}`; `GET /{rid}/events` - full event stream; `GET /{rid}/workspace-changes` - workspace/output file change summary and optional diffs; `GET /../messages` - legacy thread message array; `GET /../messages/page` - backward thread-global `seq` history page with middleware/subagent-AI/successful-regenerate/edit-replay filtering and page-run-scoped feedback enrichment; subagent AI callbacks remain available through run events while parent `task` ToolMessages stay visible for card restoration; `GET /../token-usage` - aggregate tokens | +| **Thread Runs** (`/api/threads/{id}/runs`) | `POST /` - create background run; `POST /stream` - create + SSE stream; `POST /wait` - create + block. Before the first journaled run, an empty run-event message feed is seeded from an existing checkpoint head so legacy checkpoint-only history receives earlier thread-global sequence numbers and remains visible after the new run; a thread with no checkpoint or an already-populated feed skips this compatibility path. `POST /regenerate/prepare` - prepare clean input + checkpoint metadata for regenerating the latest completed or interrupted assistant answer, carrying the latest non-empty thread title in graph input so resuming an older checkpoint cannot roll back a later manual rename (#4457); `POST /edit-regenerate/prepare` - prepare a checkpoint replay from the latest editable human turn with a replacement user message and edit replay metadata; it carries the current thread title the same way, but only when the replay base already has one — an untitled base belongs to a thread the title middleware has not named yet, so pinning the current title there would keep a name generated from the prompt the edit just replaced; `GET /` - list runs; `GET /{rid}` - run details; `POST /{rid}/cancel` - cancel; `GET /{rid}/join` - join SSE; `GET /{rid}/messages` - paginated per-run messages `{data, has_more}`; `GET /{rid}/events` - full event stream; `GET /{rid}/workspace-changes` - workspace/output file change summary and optional diffs; `GET /../messages` - legacy thread message array; `GET /../messages/page` - backward thread-global `seq` history page with middleware/subagent-AI/successful-regenerate/edit-replay filtering and page-run-scoped feedback enrichment; subagent AI callbacks remain available through run events while parent `task` ToolMessages stay visible for card restoration; `GET /../token-usage` - aggregate tokens | | **Feedback** (`/api/threads/{id}/runs/{rid}/feedback`) | `PUT /` - upsert feedback; `DELETE /` - delete user feedback; `POST /` - create feedback; `GET /` - list feedback; `GET /stats` - aggregate stats; `DELETE /{fid}` - delete specific | | **Runs** (`/api/runs`) | `POST /stream` - stateless run + SSE; `POST /wait` - stateless run + block; `GET /{rid}/messages` - paginated messages by run_id `{data, has_more}` (cursor: `after_seq`/`before_seq`); `GET /{rid}/feedback` - list feedback by run_id | | **GitHub Webhooks** (`/api/webhooks/github`) | `POST /` - receive GitHub App / repo webhook deliveries. Verifies `X-Hub-Signature-256` against `GITHUB_WEBHOOK_SECRET`; exempt from auth + CSRF because authenticity is enforced by HMAC. The route is fail-closed: mounted only when `GITHUB_WEBHOOK_SECRET` is set, or when explicit dev opt-in `DEER_FLOW_ALLOW_UNVERIFIED_GITHUB_WEBHOOKS=1` is set. Recognized events include `ping`, `issues`, `issue_comment`, `pull_request`, `pull_request_review`, and `pull_request_review_comment`; unknown events return 200 with `handled=false`. Fan-out runtime failures return 503, keeping the delivery recorded as failed for manual/API/scripted redelivery (GitHub does not automatically retry any failed delivery, 5xx included); permanent/non-retryable conditions such as `channels.github.enabled: false`, unknown events, malformed payloads, or unavailable channel service return 200 with a skipped/handled response. | diff --git a/backend/app/gateway/services.py b/backend/app/gateway/services.py index 44c39f2a2..6d30f494e 100644 --- a/backend/app/gateway/services.py +++ b/backend/app/gateway/services.py @@ -61,6 +61,7 @@ from deerflow.runtime.checkpoint_mode import ( ) from deerflow.runtime.checkpoint_state import graph_state_schema from deerflow.runtime.goal import goal_thread_lock +from deerflow.runtime.journal import build_checkpoint_history_seed_events from deerflow.runtime.runs.naming import resolve_root_run_name from deerflow.runtime.secret_context import ( LegacyRunMetadataSecretError, @@ -983,6 +984,63 @@ async def apply_checkpoint_to_run_config( configurable["checkpoint_map"] = checkpoint_map +async def ensure_checkpoint_history_seeded( + request: Request, + *, + thread_id: str, + assistant_id: str | None, +) -> None: + """Backfill an empty run-event feed from an existing checkpoint head. + + No-op unless the feed is empty AND a checkpoint head with messages + exists — i.e. a legacy checkpoint-only thread facing its first journaled + run. This is a migration shim: remove it once pre-journal threads are no + longer a supported upgrade source. The info log on a successful seed is + the observability hook for that decision — when it stops appearing, the + shim is dead. + """ + event_store = request.app.state.run_event_store + # The emptiness check is deliberately thread-scoped, never user-scoped: + # seed rows may be stamped with a different principal (NULL for ownerless + # seeds, or another user on a shared NULL-owner thread), so a user-scoped + # query would miss them and re-seed a duplicate history per principal. + # Passing user_id=None also opts out of AUTO resolution explicitly, which + # would raise when no user contextvar is set (e.g. the scheduler launch + # path for ownerless internal tasks). + if await event_store.list_messages(thread_id, limit=1, user_id=None): + return + + checkpoint_config = { + "configurable": { + "thread_id": thread_id, + "checkpoint_ns": "", + } + } + if await get_checkpointer(request).aget_tuple(checkpoint_config) is None: + return + + accessor, config = build_checkpoint_state_accessor( + request, + thread_id=thread_id, + assistant_id=assistant_id, + ) + snapshot = await accessor.aget(config) + values = getattr(snapshot, "values", None) + messages = values.get("messages") if isinstance(values, dict) else None + if not isinstance(messages, list) or not messages: + return + + events = build_checkpoint_history_seed_events( + messages, + thread_id=thread_id, + run_id_prefix=f"checkpoint-seed-{thread_id}", + ) + if not events: + return + await event_store.put_batch(events) + logger.info("Seeded %d checkpoint-history events for thread %s", len(events), thread_id) + + # --------------------------------------------------------------------------- # Run lifecycle # --------------------------------------------------------------------------- @@ -1156,6 +1214,11 @@ async def start_run( try: async with goal_thread_lock(thread_id): + await ensure_checkpoint_history_seeded( + request, + thread_id=thread_id, + assistant_id=body.assistant_id, + ) record = await run_mgr.create_or_reject( thread_id, body.assistant_id, diff --git a/backend/packages/harness/deerflow/runtime/journal.py b/backend/packages/harness/deerflow/runtime/journal.py index 7583d07d0..6f3007fa1 100644 --- a/backend/packages/harness/deerflow/runtime/journal.py +++ b/backend/packages/harness/deerflow/runtime/journal.py @@ -87,24 +87,16 @@ def _coerce_seed_message(message: Any) -> Any: return message -def build_branch_history_seed_events( +def _build_history_seed_events( messages: Sequence[Any], *, thread_id: str, run_id_prefix: str, - parent_thread_id: str, + seed_metadata: Mapping[str, Any], ) -> list[dict]: - """Serialize a branch checkpoint's messages into run-event message rows. + """Serialize checkpoint messages into run-event rows. - Thread branching copies checkpoint state, but the thread feed - (``list_messages`` / ``GET /threads/{id}/messages/page``) reads the - run-event store — which a fresh branch has no rows in, so the inherited - history vanishes from the UI as soon as the branch's first run refreshes - the feed (#4380). Seeding the branch's run_events from the same - checkpoint snapshot the branch was created from keeps the feed - consistent with what the branch actually contains. - - Rows are grouped into one synthetic run per inherited turn + Rows are grouped into one synthetic run per checkpoint turn (``{run_id_prefix}-{n}``), a new turn starting at every persisted human message — the same boundary a real run has, since a run begins with a human input (including the allowlisted hidden ``ask_clarification`` @@ -117,9 +109,9 @@ def build_branch_history_seed_events( id per turn confines the drop to the turn actually regenerated. Mirrors RunJournal's message-event contract so seeded rows are - indistinguishable from journaled ones except by the ``branch_seed`` - marker: same event types, ``category="message"``, ``content= - message.model_dump()``, the human-input persistence rule + indistinguishable from journaled ones except by the supplied seed metadata: + same event types, ``category="message"``, ``content=message.model_dump()``, + the human-input persistence rule (``_should_persist_human_input_message``), the original-user-text restoration, and the same treatment of ``hide_from_ui`` AI/tool rows — RunJournal persists them (``on_llm_end`` / ``_persist_tool_result_message`` @@ -136,7 +128,6 @@ def build_branch_history_seed_events( """ events: list[dict] = [] created_at = datetime.now(UTC).isoformat() - seed_metadata = {"branch_seed": True, "branch_parent_thread_id": parent_thread_id} # Messages ahead of the first human turn (none in practice) stay in turn 0. turn_index = 0 for raw_message in messages: @@ -175,6 +166,45 @@ def build_branch_history_seed_events( return events +def build_branch_history_seed_events( + messages: Sequence[Any], + *, + thread_id: str, + run_id_prefix: str, + parent_thread_id: str, +) -> list[dict]: + """Serialize inherited branch history into the branch's empty event feed.""" + return _build_history_seed_events( + messages, + thread_id=thread_id, + run_id_prefix=run_id_prefix, + seed_metadata={ + "branch_seed": True, + "branch_parent_thread_id": parent_thread_id, + }, + ) + + +def build_checkpoint_history_seed_events( + messages: Sequence[Any], + *, + thread_id: str, + run_id_prefix: str, +) -> list[dict]: + """Serialize legacy checkpoint history for a thread's empty event feed. + + Reuse the branch seed's message normalization and per-turn synthetic run + grouping, but stamp migration-specific metadata so these rows are not + misidentified as history inherited from another thread. + """ + return _build_history_seed_events( + messages, + thread_id=thread_id, + run_id_prefix=run_id_prefix, + seed_metadata={"checkpoint_history_seed": True}, + ) + + class RunJournal(BaseCallbackHandler): """LangChain callback handler that captures events to RunEventStore.""" diff --git a/backend/tests/test_gateway_services.py b/backend/tests/test_gateway_services.py index 842546991..9d283a3b6 100644 --- a/backend/tests/test_gateway_services.py +++ b/backend/tests/test_gateway_services.py @@ -12,6 +12,7 @@ import pytest from app.gateway.auth_disabled import AUTH_SOURCE_INTERNAL from deerflow.config.app_config import AppConfig, reset_app_config, set_app_config +from deerflow.runtime.events.store.memory import MemoryRunEventStore @pytest.fixture @@ -38,7 +39,7 @@ def _make_start_run_request(run_manager, *, thread_store=None, auth_source=None) run_manager=run_manager, checkpointer=InMemorySaver(), store=store, - run_event_store=SimpleNamespace(), + run_event_store=MemoryRunEventStore(), run_events_config=None, thread_store=thread_store or MemoryThreadMetaStore(store), ) @@ -918,7 +919,6 @@ def test_apply_checkpoint_to_run_config_writes_checkpoint_fields(): config = {"configurable": {"thread_id": "thread-1"}} asyncio.run(apply_checkpoint_to_run_config(config, body=body, thread_id="thread-1", request=request)) - assert checkpointer.seen_config == { "configurable": { "thread_id": "thread-1", @@ -932,6 +932,279 @@ def test_apply_checkpoint_to_run_config_writes_checkpoint_fields(): assert config["configurable"]["checkpoint_map"] == {"": "ckpt-1"} +@pytest.mark.anyio +async def test_seeded_checkpoint_messages_precede_the_first_new_run_messages(): + from unittest.mock import AsyncMock, patch + + from langchain_core.messages import AIMessage, HumanMessage + + from app.gateway.services import ensure_checkpoint_history_seeded + + event_store = MemoryRunEventStore() + checkpointer = SimpleNamespace( + aget_tuple=AsyncMock(return_value=SimpleNamespace(checkpoint={})), + ) + snapshot = SimpleNamespace( + values={ + "messages": [ + HumanMessage(id="legacy-human", content="old question"), + AIMessage(id="legacy-ai", content="old answer"), + ] + } + ) + accessor = SimpleNamespace(aget=AsyncMock(return_value=snapshot)) + request = SimpleNamespace( + app=SimpleNamespace( + state=SimpleNamespace( + checkpointer=checkpointer, + run_event_store=event_store, + ) + ) + ) + + with patch( + "app.gateway.services.build_checkpoint_state_accessor", + return_value=(accessor, {"configurable": {"thread_id": "thread-1"}}), + ): + await ensure_checkpoint_history_seeded( + request, + thread_id="thread-1", + assistant_id="lead_agent", + ) + + for message_type, message_id in ( + ("human", "new-human"), + ("ai", "new-ai"), + ): + await event_store.put( + thread_id="thread-1", + run_id="new-run", + event_type=("llm.human.input" if message_type == "human" else "llm.ai.response"), + category="message", + content={ + "type": message_type, + "id": message_id, + "content": message_id, + "additional_kwargs": {}, + }, + metadata={"caller": "lead_agent"}, + ) + + rows = await event_store.list_messages("thread-1", limit=10) + assert [row["content"]["id"] for row in rows] == [ + "legacy-human", + "legacy-ai", + "new-human", + "new-ai", + ] + assert [row["seq"] for row in rows] == [1, 2, 3, 4] + assert {row["run_id"] for row in rows[:2]} == {"checkpoint-seed-thread-1-1"} + assert all(row["metadata"].get("checkpoint_history_seed") is True for row in rows[:2]) + + +@pytest.mark.anyio +async def test_checkpoint_history_seed_skips_new_thread_without_checkpoint(): + from unittest.mock import AsyncMock, patch + + from app.gateway.services import ensure_checkpoint_history_seeded + + event_store = SimpleNamespace( + list_messages=AsyncMock(return_value=[]), + put_batch=AsyncMock(), + ) + checkpointer = SimpleNamespace(aget_tuple=AsyncMock(return_value=None)) + request = SimpleNamespace( + app=SimpleNamespace( + state=SimpleNamespace( + checkpointer=checkpointer, + run_event_store=event_store, + ) + ) + ) + + with patch( + "app.gateway.services.build_checkpoint_state_accessor", + side_effect=AssertionError("new threads should not build an accessor"), + ): + await ensure_checkpoint_history_seeded( + request, + thread_id="thread-1", + assistant_id="lead_agent", + ) + + event_store.put_batch.assert_not_awaited() + + +@pytest.mark.anyio +async def test_checkpoint_history_seed_is_skipped_when_journal_already_has_messages(): + from unittest.mock import AsyncMock, patch + + from app.gateway.services import ensure_checkpoint_history_seeded + + event_store = SimpleNamespace( + list_messages=AsyncMock(return_value=[{"seq": 1}]), + put_batch=AsyncMock(), + ) + request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(run_event_store=event_store))) + + with patch( + "app.gateway.services.build_checkpoint_state_accessor", + side_effect=AssertionError("checkpoint state should not be loaded"), + ): + await ensure_checkpoint_history_seeded( + request, + thread_id="thread-1", + assistant_id="lead_agent", + ) + + event_store.put_batch.assert_not_awaited() + + +@pytest.mark.anyio +@pytest.mark.no_auto_user +async def test_checkpoint_history_seed_guard_tolerates_missing_user_context(): + """Scheduler/internal launch paths can run without a user contextvar. + DbRunEventStore resolves user_id=AUTO strictly and raises in that case; + the seed guard must pass user_id=None explicitly instead of aborting the + run.""" + from unittest.mock import AsyncMock + + from app.gateway.services import ensure_checkpoint_history_seeded + from deerflow.runtime.user_context import AUTO, _AutoSentinel + + captured: dict[str, object] = {} + + async def list_messages(thread_id, *, limit=50, before_seq=None, after_seq=None, user_id=AUTO): + # Mirror DbRunEventStore: AUTO with no user context raises. + if isinstance(user_id, _AutoSentinel): + raise RuntimeError("list_messages called with user_id=AUTO but no user context is set") + captured["user_id"] = user_id + return [] + + event_store = SimpleNamespace(list_messages=list_messages, put_batch=AsyncMock()) + checkpointer = SimpleNamespace(aget_tuple=AsyncMock(return_value=None)) + request = SimpleNamespace( + app=SimpleNamespace( + state=SimpleNamespace( + checkpointer=checkpointer, + run_event_store=event_store, + ) + ) + ) + + await ensure_checkpoint_history_seeded( + request, + thread_id="thread-1", + assistant_id="lead_agent", + ) + + assert captured["user_id"] is None + event_store.put_batch.assert_not_awaited() + + +@pytest.mark.anyio +async def test_checkpoint_history_seed_guard_is_thread_scoped_under_user_context(): + """Regression: the emptiness guard must stay thread-scoped (user_id=None) + even when a user is authenticated. Seed rows stamped by another principal + (or NULL) are invisible to a user-scoped query, which would re-seed a + duplicate history per principal.""" + from unittest.mock import AsyncMock, patch + + from app.gateway.services import ensure_checkpoint_history_seeded + from deerflow.runtime.user_context import AUTO + + captured: dict[str, object] = {} + + async def list_messages(thread_id, *, limit=50, before_seq=None, after_seq=None, user_id=AUTO): + captured["user_id"] = user_id + return [{"seq": 1}] + + event_store = SimpleNamespace(list_messages=list_messages, put_batch=AsyncMock()) + request = SimpleNamespace(app=SimpleNamespace(state=SimpleNamespace(run_event_store=event_store))) + + with patch( + "app.gateway.services.build_checkpoint_state_accessor", + side_effect=AssertionError("checkpoint state should not be loaded"), + ): + await ensure_checkpoint_history_seeded( + request, + thread_id="thread-1", + assistant_id="lead_agent", + ) + + assert captured["user_id"] is None + event_store.put_batch.assert_not_awaited() + + +@pytest.mark.anyio +@pytest.mark.no_auto_user +async def test_checkpoint_history_seed_runs_exactly_once_across_principals(tmp_path): + """DbRunEventStore regression: an ownerless seed stamps rows with + user_id=NULL; a later authenticated run on the same thread must still + see them and skip re-seeding (the MemoryRunEventStore-based tests above + cannot catch this because the memory store ignores user_id).""" + from unittest.mock import AsyncMock, patch + + from langchain_core.messages import AIMessage, HumanMessage + + from app.gateway.services import ensure_checkpoint_history_seeded + from deerflow.persistence.engine import close_engine, get_session_factory, init_engine + from deerflow.runtime.events.store.db import DbRunEventStore + from deerflow.runtime.user_context import reset_current_user, set_current_user + + url = f"sqlite+aiosqlite:///{tmp_path / 'events.db'}" + await init_engine("sqlite", url=url, sqlite_dir=str(tmp_path)) + try: + event_store = DbRunEventStore(get_session_factory()) + checkpointer = SimpleNamespace( + aget_tuple=AsyncMock(return_value=SimpleNamespace(checkpoint={})), + ) + snapshot = SimpleNamespace( + values={ + "messages": [ + HumanMessage(id="legacy-human", content="old question"), + AIMessage(id="legacy-ai", content="old answer"), + ] + } + ) + accessor = SimpleNamespace(aget=AsyncMock(return_value=snapshot)) + request = SimpleNamespace( + app=SimpleNamespace( + state=SimpleNamespace( + checkpointer=checkpointer, + run_event_store=event_store, + ) + ) + ) + + with patch( + "app.gateway.services.build_checkpoint_state_accessor", + return_value=(accessor, {"configurable": {"thread_id": "thread-1"}}), + ): + # First seed: ownerless (no user contextvar) — rows stamped NULL. + await ensure_checkpoint_history_seeded( + request, + thread_id="thread-1", + assistant_id="lead_agent", + ) + # Second attempt: authenticated user on the same thread — must + # see the NULL-stamped rows and skip. + token = set_current_user(SimpleNamespace(id="user-a")) + try: + await ensure_checkpoint_history_seeded( + request, + thread_id="thread-1", + assistant_id="lead_agent", + ) + finally: + reset_current_user(token) + + rows = await event_store.list_messages("thread-1", limit=100, user_id=None) + assert [row["content"]["id"] for row in rows] == ["legacy-human", "legacy-ai"] + finally: + await close_engine() + + def test_apply_checkpoint_to_run_config_rejects_missing_checkpoint(): import asyncio from types import SimpleNamespace @@ -1362,7 +1635,7 @@ async def _capture_start_run_graph_input(body, *, auth_source=None): run_manager=run_manager, checkpointer=InMemorySaver(), store=InMemoryStore(), - run_event_store=SimpleNamespace(), + run_event_store=MemoryRunEventStore(), run_events_config=None, thread_store=MemoryThreadMetaStore(InMemoryStore()), ) @@ -1403,7 +1676,7 @@ def _make_start_run_persistence_context(): run_manager=RunManager(store=run_store), checkpointer=InMemorySaver(), store=InMemoryStore(), - run_event_store=SimpleNamespace(), + run_event_store=MemoryRunEventStore(), run_events_config=None, thread_store=thread_store, checkpoint_channel_mode="full", @@ -1656,7 +1929,7 @@ def test_start_run_uses_internal_owner_header_for_persistence(_stub_app_config): run_manager=run_manager, checkpointer=InMemorySaver(), store=InMemoryStore(), - run_event_store=SimpleNamespace(), + run_event_store=MemoryRunEventStore(), run_events_config=None, thread_store=thread_store, ) @@ -1745,7 +2018,7 @@ def test_start_run_stamps_internal_owner_guardrail_attribution(_stub_app_config) run_manager=run_manager, checkpointer=InMemorySaver(), store=InMemoryStore(), - run_event_store=SimpleNamespace(), + run_event_store=MemoryRunEventStore(), run_events_config=None, thread_store=thread_store, ) @@ -1827,7 +2100,7 @@ def test_start_run_session_caller_anti_forgery(_stub_app_config): run_manager=run_manager, checkpointer=InMemorySaver(), store=InMemoryStore(), - run_event_store=SimpleNamespace(), + run_event_store=MemoryRunEventStore(), run_events_config=None, thread_store=thread_store, ) diff --git a/backend/tests/test_stateless_runs_owner_isolation.py b/backend/tests/test_stateless_runs_owner_isolation.py index 6d6521238..b5ab980cf 100644 --- a/backend/tests/test_stateless_runs_owner_isolation.py +++ b/backend/tests/test_stateless_runs_owner_isolation.py @@ -81,9 +81,14 @@ def _client(user): app.state.thread_store = _make_thread_store() app.state.stream_bridge = MagicMock() app.state.checkpointer = MagicMock() + # start_run's checkpoint-history seeding runs before admission: give the + # store/checkpointer async stubs so the seed path sees an empty feed and + # no checkpoint head, then skips. + app.state.checkpointer.aget_tuple = AsyncMock(return_value=None) app.state.store = MagicMock() app.state.run_events_config = None app.state.run_event_store = MagicMock() + app.state.run_event_store.list_messages = AsyncMock(return_value=[]) run_manager = MagicMock() run_manager.create_or_reject = AsyncMock(side_effect=ConflictError("sentinel: owner check passed")) app.state.run_manager = run_manager From 486b51eb51364979e192030f7443fa331e2ec783 Mon Sep 17 00:00:00 2001 From: Baldwinzc <56501736+Baldwinzc@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:27:51 +0800 Subject: [PATCH 04/31] fix(scheduler): normalize once-schedule next_run_at to UTC (#4607) The once branch of next_run_at returned the run time with the task's local timezone offset attached, while the cron branch normalizes to UTC. The value is persisted into scheduled_tasks.next_run_at, and SQLAlchemy's SQLite dialect discards tzinfo on bind, so a once task declared in a non-UTC timezone fires shifted by the whole offset (e.g. 8 hours late for Asia/Shanghai, hours early for negative offsets - also bypassing the min_once_delay_seconds guard). Postgres timestamptz normalizes on write, which is why only SQLite deployments are affected. Align the once branch with the cron branch by converting to UTC before returning. --- .../harness/deerflow/scheduler/schedules.py | 4 +++ .../tests/test_scheduled_task_schedules.py | 26 ++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/backend/packages/harness/deerflow/scheduler/schedules.py b/backend/packages/harness/deerflow/scheduler/schedules.py index 8509c3523..a7783e1a3 100644 --- a/backend/packages/harness/deerflow/scheduler/schedules.py +++ b/backend/packages/harness/deerflow/scheduler/schedules.py @@ -41,6 +41,10 @@ def next_run_at( # A naive run_at means "wall-clock time in the task's declared # timezone", matching how cron schedules interpret it. run_at = run_at.replace(tzinfo=ZoneInfo(timezone_name)) + # Normalize to UTC like the cron branch: next_run_at is persisted to + # timezone-discarding columns (SQLite), where a non-UTC offset shifts + # the effective fire time by the whole offset. + run_at = run_at.astimezone(UTC) return run_at if run_at > now else None if schedule_type == "cron": diff --git a/backend/tests/test_scheduled_task_schedules.py b/backend/tests/test_scheduled_task_schedules.py index fcaac8676..48644a9df 100644 --- a/backend/tests/test_scheduled_task_schedules.py +++ b/backend/tests/test_scheduled_task_schedules.py @@ -1,4 +1,4 @@ -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta import pytest @@ -38,6 +38,30 @@ def test_next_run_at_for_once_returns_none_after_fire_time(): assert result is None +def test_next_run_at_for_once_normalizes_naive_run_at_to_utc(): + now = datetime(2026, 7, 31, 0, 0, tzinfo=UTC) + result = next_run_at( + "once", + {"run_at": "2026-08-01T09:00:00"}, + "Asia/Shanghai", + now=now, + ) + assert result == datetime(2026, 8, 1, 1, 0, tzinfo=UTC) + assert result.utcoffset() == timedelta(0) + + +def test_next_run_at_for_once_normalizes_aware_run_at_to_utc(): + now = datetime(2026, 7, 31, 0, 0, tzinfo=UTC) + result = next_run_at( + "once", + {"run_at": "2026-08-01T09:00:00+08:00"}, + "UTC", + now=now, + ) + assert result == datetime(2026, 8, 1, 1, 0, tzinfo=UTC) + assert result.utcoffset() == timedelta(0) + + def test_next_run_at_for_cron_uses_timezone(): now = datetime(2026, 7, 1, 0, 30, tzinfo=UTC) result = next_run_at( From f2e832330e6717c3fa660253b0ba0990cc6b7344 Mon Sep 17 00:00:00 2001 From: Xinmin Zeng <135568692+fancyboi999@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:55:24 +0800 Subject: [PATCH 05/31] fix(sandbox): enforce disabled skills in filesystem views (#4178) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(sandbox): project enabled skills into sandbox views * fix(skills): keep projection mutations consistent * fix(skills): fail closed on projection errors * fix(skills): isolate per-scope failures during boot projection rebuild rebuild_all_skill_projections() propagated any exception from the public rebuild or from a single user's rebuild straight out of the gateway lifespan startup, uncaught. A single broken user directory (bad permissions, corrupted _skill_states.json, unreadable content) would therefore abort gateway boot for every user, not just that one - _rebuild_*_locked already fails closed internally (clears the view and re-raises), so the boot loop only needed to stop treating that re-raise as fatal. Each scope's rebuild now fails closed independently and boot continues; a scope left empty by a boot failure self-heals on the next sandbox acquire via ensure_skill_projections(). Also patches deerflow.skills.projection.rebuild_all_skill_projections in the memory-flush lifespan test fixture, matching the two sibling fixtures in the same file — this call is now on the lifespan startup path and the fixture's minimal SimpleNamespace config predates it. * test(skills): update authz test for the projection-aware public toggle _persist_shared_skill_state (introduced earlier in this branch) reads the shared extensions_config.json fresh from disk under the projection lock instead of through the cached get_extensions_config() singleton - that's the whole point of the fix (stale worker caches must not clobber another worker's concurrent update). The name no longer exists on the skills router module, so the test's monkeypatch of it started raising AttributeError instead of exercising the endpoint. The mock storage in this test isn't a real LocalSkillStorage instance, so _persist_shared_skill_state's projection-mutation branch is already skipped (nullcontext) and it falls back to a fresh ExtensionsConfig() for the nonexistent tmp config_path - no replacement monkeypatch needed. * fix(sandbox): make skill projection ensure best-effort in acquire acquire() called _ensure_skills_projection() directly, outside any try/except, in both LocalSandboxProvider and AioSandboxProvider. Every other skill-mount setup path in these providers has always caught exceptions and logged a warning rather than failing sandbox acquire outright (e.g. when config.yaml can't be resolved) - these two new call sites broke that contract, so any projection failure (including simply not having a config.yaml, as in CI's test environment) now failed acquire() itself instead of just leaving skill mounts off. _ensure_skills_projection now catches its own exceptions and returns None; both providers' callers already tolerate that (a None projection skips the skill-specific mounts, matching the existing degrade path) after making _append_public_skill_mapping and the custom/legacy mount block in LocalSandboxProvider explicitly None-safe. Caught by running the full suite with config.yaml removed, matching CI's environment - not caught locally because a real config.yaml was present, masking the failure. * fix(sandbox): make E2B skill projection mounts best-effort _skill_projection_mounts called ensure_skill_projections with no guard, unlike Local/AIO's _ensure_skills_projection. A raise propagated out of _apply_mounts before the configured-mounts loop ran, so a skills projection failure dropped the operator's own configured mounts too - only caught by create()'s outer warning, with nothing applied at all. Swallow here and return an empty mount list on failure, matching the Local/AIO pattern: still fail-closed for skills, but no longer widens the blast radius to unrelated configured mounts. Review feedback from PR #4178. * docs(skills): document projection trade-offs flagged in review - _update_tree_digest: note the metadata-only (not content) hashing trade-off and why runtime writes through this codebase are still covered regardless (rebuild-under-lock + rename always changes inode). - LocalSandboxProvider.acquire: note the acquire-time self-heal cost (cheap on a fresh manifest, ~400ms rebuild under lock on stale/drift). - skill_projection_mutation: drop the no-op except-Exception-then-raise; a raise from the mutation already propagates past the yield with the view left cleared, no explicit re-raise needed. - provisioner README: spell out that hostPath skills volumes require the gateway and K8s node to share DEER_FLOW_HOST_BASE_DIR (single-node or shared storage), and that the custom/legacy volumes' hostPath type Directory (not DirectoryOrCreate) makes a violation of that assumption a visible Pod-creation failure instead of a silent empty mount. Review feedback from PR #4178. * fix(skills): lazily repair user projections * fix(skills): close projection review gaps * fix(skills): refresh user projection enable state * fix(skills): close projection review follow-ups * fix(skills): preserve state across projection writes --------- Co-authored-by: Willem Jiang --- CHANGELOG.md | 13 + README.md | 2 + backend/AGENTS.md | 10 +- backend/app/gateway/app.py | 6 + backend/app/gateway/routers/skills.py | 90 ++- backend/packages/harness/deerflow/client.py | 16 +- .../aio_sandbox/aio_sandbox_provider.py | 100 +-- .../e2b_sandbox/e2b_sandbox_provider.py | 49 +- .../packages/harness/deerflow/config/paths.py | 26 + .../sandbox/local/local_sandbox_provider.py | 143 ++-- .../harness/deerflow/skills/projection.py | 552 ++++++++++++++++ .../skills/storage/local_skill_storage.py | 44 +- .../deerflow/skills/storage/skill_storage.py | 9 + .../storage/user_scoped_skill_storage.py | 26 +- .../deerflow/tools/skill_manage_tool.py | 6 +- .../blocking_io/test_skills_update_router.py | 16 + backend/tests/test_aio_sandbox_provider.py | 16 +- backend/tests/test_client.py | 31 +- backend/tests/test_e2b_sandbox_provider.py | 95 +++ .../tests/test_gateway_lifespan_shutdown.py | 3 + .../test_local_sandbox_provider_mounts.py | 9 +- backend/tests/test_monocle_tracing.py | 2 + backend/tests/test_provisioner_pvc_volumes.py | 56 +- .../test_provisioner_request_threading.py | 2 +- backend/tests/test_skill_manage_tool.py | 42 ++ backend/tests/test_skill_projection.py | 612 ++++++++++++++++++ backend/tests/test_skills_custom_router.py | 105 ++- backend/tests/test_skills_router_authz.py | 28 +- .../tests/test_three_way_skills_mount_e2e.py | 100 ++- .../tests/test_user_scoped_skill_storage.py | 8 +- .../templates/gateway-deployment.yaml | 2 - docker/docker-compose-dev.yaml | 2 - docker/docker-compose.yaml | 2 - docker/provisioner/README.md | 12 +- docker/provisioner/app.py | 39 +- 35 files changed, 2010 insertions(+), 264 deletions(-) create mode 100644 backend/packages/harness/deerflow/skills/projection.py create mode 100644 backend/tests/test_skill_projection.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e79d4d75..d80022249 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,18 @@ This section accumulates work toward the **2.1.0** milestone ### ⚠ Breaking changes +- **skills:** Sandboxes now reserve `/mnt/skills` for managed enabled-only + projections. `DEER_FLOW_HOST_SKILLS_PATH` and `SKILLS_HOST_PATH` are no longer + used; Docker/AIO and hostPath deployments derive projection paths from + `DEER_FLOW_HOST_BASE_DIR`. E2B operator mounts targeting `/mnt/skills` or any + child path are skipped with a warning so they cannot shadow the managed + projection; move extra E2B content to a different container path. User + projections re-read global enable state from disk so toggles propagate across + Gateway workers on the next sandbox acquire. Existing E2B sandboxes retain + their creation-time snapshot until they are recreated. PVC-backed provisioner + deployments still mount the operator-supplied PVC snapshot directly, so + disabled-skill filesystem isolation does not apply in PVC mode until dynamic + PVC materialization is implemented. ([#4178]) - **sandbox:** E2B now enforces `sandbox.replicas` as a process-local capacity limit. The default `wait` policy waits for `acquire_timeout`, then fails the agent turn. DeerFlow does not retry the turn automatically. Use `burst` with @@ -1264,6 +1276,7 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#4170]: https://github.com/bytedance/deer-flow/pull/4170 [#4171]: https://github.com/bytedance/deer-flow/pull/4171 [#4174]: https://github.com/bytedance/deer-flow/pull/4174 +[#4178]: https://github.com/bytedance/deer-flow/pull/4178 [#4181]: https://github.com/bytedance/deer-flow/pull/4181 [#4187]: https://github.com/bytedance/deer-flow/pull/4187 [#4188]: https://github.com/bytedance/deer-flow/pull/4188 diff --git a/README.md b/README.md index 62b56e362..7064e36e3 100644 --- a/README.md +++ b/README.md @@ -718,6 +718,8 @@ An enabled skill's `allowed-tools` policy applies only after that skill is expli When you install `.skill` archives through the Gateway, DeerFlow accepts standard optional frontmatter metadata such as `version`, `author`, and `compatibility` instead of rejecting otherwise valid external skills. +Disabling a skill also removes it from the sandbox filesystem view, so shell commands and structured file tools follow the same enabled state. Local, Docker/AIO, hostPath provisioner, and newly created E2B sandboxes source `/mnt/skills` from enabled-only projections that update when public, custom, legacy, or managed integration skills are toggled, edited, created, deleted, or installed. Managed integration packages remain shared, while their projected filesystem visibility follows each user's enabled state. Multi-worker Gateways re-read on-disk enable state while rebuilding user projections, so a toggle handled by one worker is honored by another worker's next sandbox acquire. Existing E2B sandboxes retain their creation-time snapshot until they are recreated. PVC-backed provisioner skills keep their configured PVC snapshot/layout for now; dynamic PVC materialization is tracked separately. + Managed integrations install shared read-only skill packs without mixing them into custom skills. The Lark/Feishu CLI integration is available under `Settings → Integrations → Lark / Feishu CLI`; an administrator installs or diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 1f303d537..9c3f185e0 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -608,9 +608,12 @@ that cannot tell sibling branches apart. **Provider Pattern**: `SandboxProvider` with `acquire`, `acquire_async`, `get`, `release` lifecycle. Async agent/tool paths call async sandbox lifecycle hooks so Docker sandbox creation, discovery, cross-process locking, readiness polling, and release stay off the event loop. **Environment policy** (`sandbox/env_policy.py`): `execute_command` no longer inherits the full `os.environ`. `build_sandbox_env()` scrubs secret-looking names (`*KEY*`/`*SECRET*`/`*TOKEN*`/`*PASS*`/`*CREDENTIAL*`) from the inherited environment before layering injected request secrets on top, so platform credentials (e.g. `OPENAI_API_KEY`) never leak into skill subprocesses. Benign vars (`PATH`, `HOME`, `LANG`, `VIRTUAL_ENV`, ...) are preserved. **Implementations**: -- `LocalSandboxProvider` - Local filesystem execution. `acquire(thread_id)` returns a per-thread `LocalSandbox` (id `local:{thread_id}`) whose `path_mappings` resolve `/mnt/user-data/{workspace,uploads,outputs}` and `/mnt/acp-workspace` to that thread's host directories, so the public `Sandbox` API honours the `/mnt/user-data` contract uniformly with AIO. `acquire()` / `acquire(None)` keeps the legacy generic singleton (id `local`) for callers without a thread context. Per-thread sandboxes are held in an LRU cache (default 256 entries) guarded by a `threading.Lock`. Legacy global-custom mounts are gated by the same user-scoped skill discovery rule used for prompt/list visibility; providers must not infer visibility from raw directory presence alone. -- `AioSandboxProvider` (`packages/harness/deerflow/community/`) - Docker-based isolation. Active-cache and warm-pool entries are checked with the backend during acquire/reuse; definitively dead containers are dropped from all in-process maps so the thread can discover or create a fresh sandbox instead of reusing a stale client. Backend health-check failures are treated as unknown, not dead; local discovery likewise treats an unverifiable container as not adoptable and falls through to create rather than failing acquire. `get()` remains an in-memory lookup for event-loop-safe tool paths — it never touches the ownership store (that would be blocking IO on the event loop); ownership is published on acquire/reclaim and refreshed off the event loop by the dedicated renewal thread (`_renew_owned_leases`). `uses_thread_data_mounts` defaults to backend detection (`LocalContainerBackend=True`, remote/provisioner backends=False), while the optional `sandbox.thread_data_mounts` boolean takes precedence for deployments that guarantee the Gateway and sandbox share the same thread user-data directories. Setting it `true` skips upload-time sandbox acquire/sync; a false positive leaves uploads unavailable to the sandbox. Legacy global-custom mounts follow the same shared visibility helper as local and remote providers. Readiness probes and `agent_sandbox` clients classify loopback/private IPs, single-label cluster hosts, and Docker/Podman internal hostnames as direct control-plane destinations and set `trust_env=False`; external FQDNs and public IPs retain environment proxy support. +- `LocalSandboxProvider` - Local filesystem execution. `acquire(thread_id)` returns a per-thread `LocalSandbox` (id `local:{thread_id}`) whose `path_mappings` resolve `/mnt/user-data/{workspace,uploads,outputs}` and `/mnt/acp-workspace` to that thread's host directories, so the public `Sandbox` API honours the `/mnt/user-data` contract uniformly with AIO. `acquire()` / `acquire(None)` keeps the legacy generic singleton (id `local`) for callers without a thread context. Per-thread sandboxes are held in an LRU cache (default 256 entries) guarded by a `threading.Lock`. Public, custom, legacy, and managed integration skill mappings point at stable enabled-only projection roots rather than raw skill directories. +- `AioSandboxProvider` (`packages/harness/deerflow/community/`) - Docker-based isolation. Active-cache and warm-pool entries are checked with the backend during acquire/reuse; definitively dead containers are dropped from all in-process maps so the thread can discover or create a fresh sandbox instead of reusing a stale client. Backend health-check failures are treated as unknown, not dead; local discovery likewise treats an unverifiable container as not adoptable and falls through to create rather than failing acquire. `get()` remains an in-memory lookup for event-loop-safe tool paths — it never touches the ownership store (that would be blocking IO on the event loop); ownership is published on acquire/reclaim and refreshed off the event loop by the dedicated renewal thread (`_renew_owned_leases`). `uses_thread_data_mounts` defaults to backend detection (`LocalContainerBackend=True`, remote/provisioner backends=False), while the optional `sandbox.thread_data_mounts` boolean takes precedence for deployments that guarantee the Gateway and sandbox share the same thread user-data directories. Setting it `true` skips upload-time sandbox acquire/sync; a false positive leaves uploads unavailable to the sandbox. Local-container and hostPath-provisioner mounts use the same stable skill projection roots; PVC-backed skills remain governed by the operator-supplied PVC layout until PVC materialization is implemented. Readiness probes and `agent_sandbox` clients classify loopback/private IPs, single-label cluster hosts, and Docker/Podman internal hostnames as direct control-plane destinations and set `trust_env=False`; external FQDNs and public IPs retain environment proxy support. - `E2BSandboxProvider` (`packages/harness/deerflow/community/e2b_sandbox/`) provides E2B remote isolation. + New sandboxes receive a one-shot upload from the enabled-only public, custom, + legacy, and managed integration projections. Existing E2B VMs keep their + creation-time snapshot because E2B has no shared host mount. Acquire and release share a per-user and thread lock. The provider lock does not cover remote IO. `burst_limit` adds capacity only for the `burst` policy. The `wait` policy fails the turn after `acquire_timeout`. The runtime does not @@ -672,7 +675,7 @@ that cannot tell sibling branches apart. **Virtual Path System**: - Agent sees: `/mnt/user-data/{workspace,uploads,outputs}`, `/mnt/skills` -- Physical: `backend/.deer-flow/users/{user_id}/threads/{thread_id}/user-data/...`, `deer-flow/skills/` +- Physical: `backend/.deer-flow/users/{user_id}/threads/{thread_id}/user-data/...`; raw skills stay under `deer-flow/skills/` and managed integration storage, while sandboxes read `backend/.deer-flow/skills_view/public/` and `backend/.deer-flow/users/{user_id}/skills_view/{custom,legacy,integrations}/` - Translation: `LocalSandboxProvider` builds per-thread `PathMapping`s for the user-data prefixes at acquire time; `tools.py` keeps `replace_virtual_path()` / `replace_virtual_paths_in_command()` as a defense-in-depth layer (and for path validation). AIO has the directories volume-mounted at the same virtual paths inside its container, so both implementations accept `/mnt/user-data/...` natively. - Detection: `is_local_sandbox()` accepts both `sandbox_id == "local"` (legacy / no-thread) and `sandbox_id.startswith("local:")` (per-thread) @@ -766,6 +769,7 @@ E2B output sync records remote file versions and actual host file metadata in a - **Loading**: `load_skills()` recursively scans public, per-user custom, global integration, and legacy custom locations for `SKILL.md`, parses metadata, and reads enabled state from extensions_config.json plus per-user skill state for non-public categories; that directory is a package boundary, so no nested `SKILL.md` is registered as a runtime skill. SkillScan has a deliberately narrower packaging rule: known eval fixtures are permitted as support data, while other nested `SKILL.md` files are reported as package defects. It parses runtime metadata and reads enabled state from extensions_config.json. - **External reload**: `POST /api/skills/reload` is an admin-only, process-local invalidation hook for trusted MinIO/NFS/CSI writes. `SkillStorage` instances do not cache a catalog — `load_skills()` scans on every call — so the route clears all `(app_config, user_id)` entries and the rendered prompt-section LRU, then waits up to the shared refresh timeout for the existing off-loop single-flight refresh. Each invalidation receives a generation-bound result handle; a successful scan atomically replaces the global enabled-skills cache, while a loader-level failure propagates to the HTTP waiter and preserves the last-known-good global cache. Per-user/config scans capture the refresh version and cannot repopulate shared caches if invalidation occurs while they are loading. A timed-out HTTP wait fails generically while the daemon refresh worker continues. Subsequent runs rescan after a successful reload; active runs keep their existing snapshot. Each Uvicorn worker/Kubernetes Pod must be targeted separately. Direct mount writes bypass install/edit validation, SkillScan, and history, so mounted roots are an operator-controlled trust boundary. - **Tool policy**: Agent `allowed-tools` declarations apply dynamically only to slash-activated skills and skills captured in `ThreadState.skill_context` through configured `read_file` loads; passive enabled skills and custom-agent/subagent skill allowlists remain discoverable without clamping the baseline toolset. Subagents render only skill discovery metadata at startup and reuse the same adjacent `SkillActivationMiddleware` + `SkillToolPolicyMiddleware` pair as the lead; their configured `skills` field limits discovery and activation instead of eagerly loading bodies or unioning policies. Slash policy is dominant for its run, preventing subsequently read skills from widening explicit authority; autonomous captured skills use the existing union only when no slash source exists. `tool_search` and `describe_skill` stay available as framework discovery infrastructure, while every discovered or promoted business tool still requires active-policy permission for schema visibility and execution; `task` likewise requires an explicit declaration. Each active model call intentionally reloads the full live registry so enable/disable changes, frontmatter edits, and custom/public name-shadow winners take effect without a stale TTL or unsafe direct-path cache; all tool calls produced by that model step reuse the resulting source-and-path-signed decision. Registry failures and all-invalid active sets fail closed, while stale individual paths are skipped when another valid skill remains. This is best-effort behavioral scoping, not a hard security boundary: alternate loading paths are not captured and bounded autonomous context may evict entries. +- **Sandbox projection**: `skills/projection.py` materializes enabled-only trees at `{base_dir}/skills_view/public` and `{base_dir}/users/{user_id}/skills_view/{custom,legacy,integrations}`. It hardlinks files when possible and falls back to copies across filesystems. Storage writes, archive installs, deletes, and toggles rebuild under a cross-process lock; Gateway boot ensures only the shared public view, while each user view is repaired lazily on first sandbox acquire. Managed integration packages are global, but their projected category is per-user because enabled state is isolated. Rebuilds stage a complete tree and reconcile it with per-file atomic replacement, so unrelated enabled skills remain continuously visible; disable/delete paths remove only the affected package before mutating to preserve fail-closed behavior. User projection rebuilds re-read global enable state from disk instead of the process singleton, so a toggle handled by another Gateway worker is reflected on the next acquire. Gateway public-skill toggles take the public projection lock before the shared `extensions_config_write_lock`, re-read an existing config from disk, persist the full model shape, and rebuild before responding; keep this as one worker-owned critical section so MCP writes cannot interleave and request cancellation cannot release either lock while the worker still runs. The shared public steady-state signature check runs without the global projection lock; stale/error paths take the lock and re-check before rebuilding or clearing. User-scope checks remain serialized per user. Category root inodes remain stable so live bind mounts observe content changes without sandbox recreation. Projection failures clear the affected view before raising. - **Injection (legacy / default)**: Enabled skills are listed in the agent system prompt with full metadata and container paths (`` block). Controlled by `skills.deferred_discovery: false` (default). - **Deferred discovery** (`skills.deferred_discovery: true`): Skills are listed by name only in a compact `` block, keeping the system prompt prefix-cache friendly. The agent calls the `describe_skill` tool at runtime to fetch full metadata for skills it wants to use, then loads the SKILL.md via `read_file`. Two new modules support this path: - `skills/catalog.py` — `SkillCatalog` (immutable, searchable; query forms: `select:a,b`, `+prefix`, free-text regex); `select:` returns all requested skills without a result cap; other modes cap at `MAX_RESULTS=5`. diff --git a/backend/app/gateway/app.py b/backend/app/gateway/app.py index 705f9be7d..3ee88a29b 100644 --- a/backend/app/gateway/app.py +++ b/backend/app/gateway/app.py @@ -210,6 +210,12 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: config = get_gateway_config() logger.info(f"Starting API Gateway on {config.host}:{config.port}") + from deerflow.skills.projection import ensure_public_skill_projection + + public_projection_ready = await asyncio.to_thread(ensure_public_skill_projection, app_config=startup_config) + if public_projection_ready: + logger.info("Ensured the public skill projection; user projections repair lazily on sandbox acquire") + # Agent observability (Monocle). Off by default; enabled with # MONOCLE_TRACING. Initialized here at startup — not at import time — so a # plain `import deerflow.agents` never installs a process-global tracer. diff --git a/backend/app/gateway/routers/skills.py b/backend/app/gateway/routers/skills.py index 68f7da7b5..d1f315182 100644 --- a/backend/app/gateway/routers/skills.py +++ b/backend/app/gateway/routers/skills.py @@ -273,8 +273,9 @@ async def update_custom_skill(skill_name: str, body: CustomSkillUpdateRequest, r if scan.decision == "block": raise HTTPException(status_code=400, detail=f"Security scan blocked the edit: {scan.reason}") prev_content = storage.read_custom_skill(skill_name) - storage.write_custom_skill(skill_name, SKILL_MD_FILE, body.content) - storage.append_history( + await asyncio.to_thread(storage.write_custom_skill, skill_name, SKILL_MD_FILE, body.content) + await asyncio.to_thread( + storage.append_history, skill_name, { "action": "human_edit", @@ -305,7 +306,8 @@ async def delete_custom_skill(skill_name: str, request: Request, config: AppConf try: skill_name = skill_name.replace("\r\n", "").replace("\n", "") storage = _get_user_skill_storage(config) - storage.delete_custom_skill( + await asyncio.to_thread( + storage.delete_custom_skill, skill_name, history_meta={ "action": "human_delete", @@ -384,10 +386,10 @@ async def rollback_custom_skill(skill_name: str, body: SkillRollbackRequest, req "scanner": {"decision": scan.decision, "reason": scan.reason, "static_findings": static_findings}, } if scan.decision == "block": - storage.append_history(skill_name, history_entry) + await asyncio.to_thread(storage.append_history, skill_name, history_entry) raise HTTPException(status_code=400, detail=f"Rollback blocked by security scanner: {scan.reason}") - storage.write_custom_skill(skill_name, SKILL_MD_FILE, target_content) - storage.append_history(skill_name, history_entry) + await asyncio.to_thread(storage.write_custom_skill, skill_name, SKILL_MD_FILE, target_content) + await asyncio.to_thread(storage.append_history, skill_name, history_entry) await refresh_user_skills_system_prompt_cache_async(get_effective_user_id()) return await _read_custom_skill_response(skill_name, config) except HTTPException: @@ -426,35 +428,47 @@ async def get_skill(skill_name: str, config: AppConfig = Depends(get_config)) -> raise HTTPException(status_code=500, detail=f"Failed to get skill: {str(e)}") -def _write_extensions_skill_state(skill_name: str, enabled: bool) -> None: +def _write_extensions_skill_state( + storage: SkillStorage, + skill_name: str, + enabled: bool, + *, + rebuild_public_projection: bool, +) -> None: """Read-modify-write a skill's enabled state in the shared extensions_config.json. Blocking filesystem IO: always call this via ``asyncio.to_thread``. It takes - ``extensions_config_write_lock`` itself, so that this router and the MCP - router (which performs the same RMW on the same file) cannot interleave and - drop each other's change. The lock is held by the worker rather than by the - awaiting task, so cancelling the request cannot release it mid-write. + the public projection lock before ``extensions_config_write_lock``. The first + keeps the enabled-only view synchronized across workers; the second prevents + this router and the MCP router from interleaving writes to the shared file. + Both locks are held by the worker, so request cancellation cannot release + either lock while the write or projection rebuild is still running. """ - with extensions_config_write_lock: - config_path = ExtensionsConfig.resolve_config_path() - if config_path is None: - config_path = Path.cwd().parent / "extensions_config.json" - logger.info(f"No existing extensions config found. Creating new config at: {config_path}") + from contextlib import nullcontext - # Work on a deep copy rather than the cached singleton: mutating the - # singleton in place would publish the new state to readers before it is - # durable on disk, and leave it applied even if the write below fails. - # to_file_dict() serializes the full extensions_config.json shape (all - # top-level keys), so no field is dropped from the file. - extensions_config = get_extensions_config().model_copy(deep=True) - extensions_config.skills[skill_name] = SkillStateConfig(enabled=enabled) + from deerflow.skills.projection import skill_projection_mutation + from deerflow.skills.storage.local_skill_storage import LocalSkillStorage - config_data = extensions_config.to_file_dict() + removal_names = (skill_name,) if not enabled else () + projection_update = skill_projection_mutation(storage, "public", remove_names=removal_names) if rebuild_public_projection and isinstance(storage, LocalSkillStorage) else nullcontext() + with projection_update: + with extensions_config_write_lock: + config_path = ExtensionsConfig.resolve_config_path() + if config_path is None: + config_path = Path.cwd().parent / "extensions_config.json" + logger.info(f"No existing extensions config found. Creating new config at: {config_path}") - atomic_write_extensions_config(config_path, config_data) + # The projection lock is cross-process, but the singleton cache is + # not. Existing files are therefore re-read under the lock; a new + # file starts from a deep snapshot of the cached defaults. + extensions_config = ExtensionsConfig.from_file(config_path) if config_path.exists() else get_extensions_config().model_copy(deep=True) + extensions_config.skills[skill_name] = SkillStateConfig(enabled=enabled) - logger.info(f"Skills configuration updated and saved to: {config_path}") - reload_extensions_config() + config_data = extensions_config.to_file_dict() + atomic_write_extensions_config(config_path, config_data) + + logger.info(f"Skills configuration updated and saved to: {config_path}") + reload_extensions_config() @router.put( @@ -488,10 +502,13 @@ async def update_skill(skill_name: str, body: SkillUpdateRequest, request: Reque # CUSTOM / LEGACY skills → per-user _skill_states.json (isolated state) # so that two users with same-named custom skills can toggle independently. if skill.category == SkillCategory.PUBLIC: - # Shared-file RMW. The worker takes extensions_config_write_lock for - # the whole read→write window, so it stays serialized against the MCP - # router even if this request is cancelled mid-write. - await asyncio.to_thread(_write_extensions_skill_state, skill_name, body.enabled) + await asyncio.to_thread( + _write_extensions_skill_state, + storage, + skill_name, + body.enabled, + rebuild_public_projection=True, + ) else: # CUSTOM / LEGACY: write per-user state from deerflow.skills.storage.user_scoped_skill_storage import UserScopedSkillStorage @@ -500,8 +517,15 @@ async def update_skill(skill_name: str, body: SkillUpdateRequest, request: Reque await asyncio.to_thread(storage.set_skill_enabled_state, skill_name, body.enabled) else: # Fallback for non-user-scoped storage (unlikely in practice): - # same shared-file RMW as the PUBLIC branch, same lock. - await asyncio.to_thread(_write_extensions_skill_state, skill_name, body.enabled) + # same shared-file RMW as the PUBLIC branch, without a public + # projection rebuild for this non-public skill. + await asyncio.to_thread( + _write_extensions_skill_state, + storage, + skill_name, + body.enabled, + rebuild_public_projection=False, + ) # PUBLIC skill enabled state lives in the global extensions_config.json # and affects every user, so the prompt cache for ALL users must be diff --git a/backend/packages/harness/deerflow/client.py b/backend/packages/harness/deerflow/client.py index 1b8a687bc..cf667a359 100644 --- a/backend/packages/harness/deerflow/client.py +++ b/backend/packages/harness/deerflow/client.py @@ -1281,13 +1281,19 @@ class DeerFlowClient: if config_path is None: raise FileNotFoundError("Cannot locate extensions_config.json. Set DEER_FLOW_EXTENSIONS_CONFIG_PATH or ensure it exists in the project root.") - extensions_config = get_extensions_config() - extensions_config.skills[name] = SkillStateConfig(enabled=enabled) + from deerflow.skills.projection import skill_projection_mutation - config_data = extensions_config.to_file_dict() + removal_names = (name,) if not enabled else () + with skill_projection_mutation(storage, "public", remove_names=removal_names): + # The projection lock is cross-process, but the singleton cache + # is not. Reload from disk under the lock before this RMW. + extensions_config = ExtensionsConfig.from_file(config_path) + extensions_config.skills[name] = SkillStateConfig(enabled=enabled) - self._atomic_write_json(config_path, config_data) - reload_extensions_config() + config_data = extensions_config.to_file_dict() + + self._atomic_write_json(config_path, config_data) + reload_extensions_config() else: # CUSTOM / LEGACY: write per-user state from deerflow.skills.storage.user_scoped_skill_storage import UserScopedSkillStorage diff --git a/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py b/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py index a66abf682..cd7d4ab97 100644 --- a/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py +++ b/backend/packages/harness/deerflow/community/aio_sandbox/aio_sandbox_provider.py @@ -44,7 +44,6 @@ from deerflow.integrations.lark_cli import LARK_CLI_SANDBOX_CONFIG_DIR, LARK_CLI from deerflow.runtime.user_context import get_effective_user_id from deerflow.sandbox.sandbox import Sandbox from deerflow.sandbox.sandbox_provider import SandboxProvider -from deerflow.skills.storage import user_should_see_legacy_skills from .aio_sandbox import AioSandbox from .backend import SandboxBackend, wait_for_sandbox_ready, wait_for_sandbox_ready_async @@ -868,41 +867,34 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): that ``Skill.get_container_path()`` category-aware paths resolve correctly inside the sandbox. - Mount sources use ``DEER_FLOW_HOST_SKILLS_PATH`` and - ``DEER_FLOW_HOST_BASE_DIR`` when running inside Docker (DooD) so the - host Docker daemon can resolve the paths. + Mount sources use ``DEER_FLOW_HOST_BASE_DIR`` when running inside + Docker (DooD) so the host Docker daemon can resolve the projection + paths. """ mounts: list[tuple[str, str, bool]] = [] try: config = get_app_config() - skills_path = config.skills.get_skills_path() container_path = config.skills.container_path - - # When running inside Docker with DooD, use host-side skills path. - host_skills_root = os.environ.get("DEER_FLOW_HOST_SKILLS_PATH") or str(skills_path) + effective_user_id = AioSandboxProvider._effective_acquire_user_id(user_id) + AioSandboxProvider._ensure_skills_projection(effective_user_id) + paths = get_paths() + host_base_dir = str(paths.host_base_dir) # 1. Public skills: global, read-only — static, shared by all threads - public_skills_path = skills_path / "public" - if public_skills_path.exists(): - mounts.append( - ( - join_host_path(host_skills_root, "public"), - f"{container_path}/public", - True, - ) + mounts.append( + ( + join_host_path(host_base_dir, "skills_view", "public"), + f"{container_path}/public", + True, ) + ) # 2. Per-user custom skills: read-only, per-thread/per-user - effective_user_id = AioSandboxProvider._effective_acquire_user_id(user_id) - paths = get_paths() - user_custom_path = paths.user_custom_skills_dir(effective_user_id) - user_custom_path.mkdir(parents=True, exist_ok=True) - host_user_custom = join_host_path( - str(paths.host_base_dir), + host_base_dir, "users", effective_user_id, - "skills", + "skills_view", "custom", ) mounts.append( @@ -913,38 +905,66 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): ) ) - # 3. Legacy (pre-migration global-custom) skills: only mount for - # users who have no per-user custom skills yet, mirroring - # ``UserScopedSkillStorage._iter_skill_files`` visibility rule. - legacy_skills_path = skills_path / "custom" - if user_should_see_legacy_skills(effective_user_id, host_path=str(skills_path)) and legacy_skills_path.exists(): - mounts.append( - ( - join_host_path(host_skills_root, "custom"), - f"{container_path}/legacy", - True, - ) + # 3. Legacy visibility is encoded by projection contents. Keep the + # mount stable even when the directory is empty so a later state + # change is visible without recreating the sandbox. + mounts.append( + ( + join_host_path(host_base_dir, "users", effective_user_id, "skills_view", "legacy"), + f"{container_path}/legacy", + True, ) + ) except Exception as e: logger.warning("Could not setup skills mounts: %s", e) return mounts + @staticmethod + def _ensure_skills_projection(user_id: str): + """Best-effort: a projection failure must not fail sandbox acquire. + + Called directly (for its side effect) from ``_acquire_internal`` / + ``_acquire_internal_async`` outside any try/except, as well as from + within ``_get_skills_mounts``'s own guarded block — swallowing here + keeps both call sites safe without duplicating the guard. + """ + from deerflow.skills.projection import ensure_skill_projections + from deerflow.skills.storage import get_or_new_user_skill_storage + + try: + storage = get_or_new_user_skill_storage(user_id, app_config=get_app_config()) + return ensure_skill_projections(storage) + except Exception as exc: + logger.warning("Could not ensure skills projection for user %s: %s", user_id, exc, exc_info=True) + return None + @staticmethod def _get_user_skill_mounts(*, user_id: str | None = None) -> list[tuple[str, str, bool]]: - """Mount managed integration skills into AIO sandboxes. + """Mount enabled managed integration skills into AIO sandboxes. Per-user custom skills are already mounted by ``_get_skills_mounts``. - This helper adds the shared integration skill root so sandbox paths match - the skill registry without duplicating ``/mnt/skills/custom``. + Integration packages are shared, but their enabled state is per-user, so + this helper mounts the user's projection instead of the raw shared root. """ try: config = get_app_config() paths = get_paths() skills_container_path = config.skills.container_path - paths.integration_skills_dir().mkdir(parents=True, exist_ok=True) + effective_user_id = AioSandboxProvider._effective_acquire_user_id(user_id) + AioSandboxProvider._ensure_skills_projection(effective_user_id) return [ - (paths.host_integration_skills_dir(), f"{skills_container_path}/integrations", True), + ( + join_host_path( + str(paths.host_base_dir), + "users", + effective_user_id, + "skills_view", + "integrations", + ), + f"{skills_container_path}/integrations", + True, + ), ] except Exception as e: logger.warning(f"Could not setup user skill mounts: {e}") @@ -1810,6 +1830,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): sandbox_id is deterministic from thread_id so no shared state file is needed — any process can derive the same container name) """ + self._ensure_skills_projection(user_id) cached_id = self._reuse_in_process_sandbox(thread_id, user_id=user_id) if cached_id is not None: return cached_id @@ -1837,6 +1858,7 @@ class AioSandboxProvider(WarmPoolLifecycleMixin[SandboxInfo], SandboxProvider): async def _acquire_internal_async(self, thread_id: str | None, *, user_id: str) -> str: """Async counterpart to ``_acquire_internal``.""" + await asyncio.to_thread(self._ensure_skills_projection, user_id) cached_id = await asyncio.to_thread(self._reuse_in_process_sandbox, thread_id, user_id=user_id) if cached_id is not None: return cached_id diff --git a/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox_provider.py b/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox_provider.py index e0cfd4d91..a0b69056f 100644 --- a/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox_provider.py +++ b/backend/packages/harness/deerflow/community/e2b_sandbox/e2b_sandbox_provider.py @@ -1092,7 +1092,7 @@ class E2BSandboxProvider(SandboxProvider): # One-shot mount uploads. e2b has no host bind-mount, so we copy # files from ``host_path`` into ``container_path`` at sandbox start. try: - self._apply_mounts(client) + self._apply_mounts(client, user_id=user_id) except Exception as e: logger.warning("Failed to apply some mounts to e2b sandbox %s: %s", sandbox_id, e) @@ -1717,11 +1717,42 @@ class E2BSandboxProvider(SandboxProvider): if exit_code not in (0, None) or "BOOTSTRAP_OK" not in stdout: raise RuntimeError(f"e2b bootstrap script failed with exit code {exit_code}; stderr={stderr.strip()}") - def _apply_mounts(self, client: E2BClientSandbox) -> None: - mounts = self._config.get("mounts") or [] - if not mounts: - return - for mount in mounts: + def _skill_projection_mounts(self, user_id: str) -> list[tuple[Path, str, bool]]: + """Best-effort: a projection failure must not drop configured mounts too. + + Unlike Local/AIO's ``_ensure_skills_projection``, this used to raise + straight out of ``_apply_mounts`` before the configured-mounts loop + ran, so a projection hiccup dropped the operator's own mounts as + collateral damage (only caught by ``create()``'s outer warning, with + no mounts applied at all). Swallowing here keeps the two mount + sources independent, matching the other two providers. + """ + from deerflow.skills.projection import ensure_skill_projections + from deerflow.skills.storage import get_or_new_user_skill_storage + + try: + config = get_app_config() + storage = get_or_new_user_skill_storage(user_id, app_config=config) + projection = ensure_skill_projections(storage) + container_root = config.skills.container_path.rstrip("/") + return [ + (projection.public, f"{container_root}/public", True), + (projection.custom, f"{container_root}/custom", True), + (projection.legacy, f"{container_root}/legacy", True), + (projection.integrations, f"{container_root}/integrations", True), + ] + except Exception as exc: + logger.warning("Could not ensure skills projection for user %s: %s", user_id, exc, exc_info=True) + return [] + + def _apply_mounts(self, client: E2BClientSandbox, *, user_id: str | None = None) -> None: + effective_user_id = user_id or get_effective_user_id() + projection_mounts = self._skill_projection_mounts(effective_user_id) + configured_mounts = self._config.get("mounts") or [] + skills_root = get_app_config().skills.container_path.rstrip("/") + + mounts: list[tuple[Path, str, bool]] = list(projection_mounts) + for mount in configured_mounts: try: host_path = Path(getattr(mount, "host_path", "") or "") container_path = (getattr(mount, "container_path", "") or "").rstrip("/") @@ -1731,6 +1762,12 @@ class E2BSandboxProvider(SandboxProvider): container_path = (mount.get("container_path", "") or "").rstrip("/") read_only = bool(mount.get("read_only", False)) + if container_path == skills_root or container_path.startswith(skills_root + "/"): + logger.warning("Skipping e2b mount that conflicts with managed skills projection: %s", container_path) + continue + mounts.append((host_path, container_path, read_only)) + + for host_path, container_path, read_only in mounts: if not host_path.exists(): logger.warning("Skipping e2b mount: host_path %s does not exist", host_path) continue diff --git a/backend/packages/harness/deerflow/config/paths.py b/backend/packages/harness/deerflow/config/paths.py index 1d846f7e4..495ecafed 100644 --- a/backend/packages/harness/deerflow/config/paths.py +++ b/backend/packages/harness/deerflow/config/paths.py @@ -258,6 +258,32 @@ class Paths: """ return self.base_dir / "integrations" / "skills" + @property + def skills_view_dir(self) -> Path: + """Global sandbox-visible skills projection: ``{base_dir}/skills_view/``.""" + return self.base_dir / "skills_view" + + @property + def public_skills_view_dir(self) -> Path: + """Enabled public skills exposed to sandboxes.""" + return self.skills_view_dir / "public" + + def user_skills_view_dir(self, user_id: str) -> Path: + """Per-user sandbox-visible skills projection root.""" + return self.user_dir(user_id) / "skills_view" + + def user_custom_skills_view_dir(self, user_id: str) -> Path: + """Enabled custom skills exposed to one user's sandboxes.""" + return self.user_skills_view_dir(user_id) / "custom" + + def user_legacy_skills_view_dir(self, user_id: str) -> Path: + """Enabled legacy skills exposed to one user's sandboxes.""" + return self.user_skills_view_dir(user_id) / "legacy" + + def user_integration_skills_view_dir(self, user_id: str) -> Path: + """Enabled managed integration skills exposed to one user's sandboxes.""" + return self.user_skills_view_dir(user_id) / "integrations" + def thread_dir(self, thread_id: str, *, user_id: str | None = None) -> Path: """ Host path for a thread's data. diff --git a/backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py b/backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py index 470f690bd..36ef8d7f6 100644 --- a/backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py +++ b/backend/packages/harness/deerflow/sandbox/local/local_sandbox_provider.py @@ -6,7 +6,6 @@ from pathlib import Path from deerflow.sandbox.local.local_sandbox import LocalSandbox, PathMapping from deerflow.sandbox.sandbox import Sandbox from deerflow.sandbox.sandbox_provider import SandboxProvider -from deerflow.skills.storage import user_should_see_legacy_skills logger = logging.getLogger(__name__) @@ -101,11 +100,11 @@ class LocalSandboxProvider(SandboxProvider): from deerflow.config import get_app_config config = get_app_config() - skills_path = config.skills.get_skills_path() container_path = config.skills.container_path + projection = self._ensure_skills_projection() # Public skills: global, read-only — static, shared by all threads - public_skills_path = skills_path / "public" + public_skills_path = projection.public if public_skills_path.exists(): mappings.append( PathMapping( @@ -217,6 +216,52 @@ class LocalSandboxProvider(SandboxProvider): def _thread_key(thread_id: str, user_id: str) -> tuple[str, str]: return (user_id, thread_id) + @staticmethod + def _ensure_skills_projection(user_id: str | None = None): + """Best-effort: a projection failure must not fail sandbox acquire. + + Mirrors the surrounding skill-mount setup, which has always logged + and continued rather than failing the whole acquire (e.g. missing + config.yaml in a test double). Callers see ``None`` and skip the + skill mounts for this acquire; the projection self-heals on a later + acquire once the underlying condition clears. + """ + from deerflow.config import get_app_config + from deerflow.skills.projection import ensure_skill_projections + from deerflow.skills.storage import get_or_new_skill_storage, get_or_new_user_skill_storage + + try: + config = get_app_config() + if user_id is None: + storage = get_or_new_skill_storage(app_config=config) + else: + storage = get_or_new_user_skill_storage(user_id, app_config=config) + return ensure_skill_projections(storage) + except Exception as exc: + logger.warning("Could not ensure skills projection for user %s: %s", user_id, exc, exc_info=True) + return None + + @staticmethod + def _append_public_skill_mapping(mappings: list[PathMapping], projection) -> None: + if projection is None: + return + try: + from deerflow.config import get_app_config + + container_path = get_app_config().skills.container_path.rstrip("/") + public_container_path = f"{container_path}/public" + if any(mapping.container_path.rstrip("/") == public_container_path for mapping in mappings): + return + mappings.append( + PathMapping( + container_path=public_container_path, + local_path=str(projection.public), + read_only=True, + ) + ) + except Exception as exc: + logger.warning("Could not append public skill mapping: %s", exc, exc_info=True) + @staticmethod def _sandbox_id_for_thread(thread_id: str, user_id: str) -> str: return f"local:{user_id}:{thread_id}" @@ -232,7 +277,7 @@ class LocalSandboxProvider(SandboxProvider): return (user_id, thread_id) @staticmethod - def _build_thread_path_mappings(thread_id: str, *, user_id: str | None = None) -> list[PathMapping]: + def _build_thread_path_mappings(thread_id: str, *, user_id: str | None = None, skill_projection=None) -> list[PathMapping]: """Build per-thread path mappings for /mnt/user-data, /mnt/acp-workspace, and /mnt/skills/custom. @@ -281,56 +326,35 @@ class LocalSandboxProvider(SandboxProvider): ), ] - # Per-user custom skills mount (read-only). This must be per-thread - # because ``/mnt/skills/custom`` resolves to different host directories - # for different users. + # Per-user category mounts stay present for the sandbox lifetime. Their + # enabled-only contents change beneath these stable roots. try: config = get_app_config() skills_container_path = config.skills.container_path - user_custom_path = paths.user_custom_skills_dir(effective_user_id) - integrations_path = paths.integration_skills_dir() - user_custom_path.mkdir(parents=True, exist_ok=True) - integrations_path.mkdir(parents=True, exist_ok=True) + projection = skill_projection if skill_projection is not None else LocalSandboxProvider._ensure_skills_projection(effective_user_id) - mappings.append( - PathMapping( - container_path=f"{skills_container_path}/custom", - local_path=str(user_custom_path), - read_only=True, - ) - ) - mappings.append( - PathMapping( - container_path=f"{skills_container_path}/integrations", - local_path=str(integrations_path), - read_only=True, - ) - ) - except Exception as exc: - logger.warning("Could not setup per-thread custom skills mount: %s", exc, exc_info=True) - - # Legacy (pre-migration global-custom) skills: only mount for users - # who have no per-user custom skills yet, mirroring the - # ``UserScopedSkillStorage._iter_skill_files`` visibility rule. Users - # with their own per-user custom skills cannot see LEGACY in the - # listing/prompt and must not be able to read it via the sandbox - # either — otherwise the listing layer and the sandbox layer disagree - # about visibility, and the sandbox layer is the more permissive one. - try: - config = get_app_config() - skills_container_path = config.skills.container_path - user_custom_path = paths.user_custom_skills_dir(effective_user_id) - legacy_skills_path = config.skills.get_skills_path() / "custom" - if user_should_see_legacy_skills(effective_user_id, host_path=str(config.skills.get_skills_path())) and legacy_skills_path.exists(): - mappings.append( - PathMapping( - container_path=f"{skills_container_path}/legacy", - local_path=str(legacy_skills_path), - read_only=True, - ) + if projection is not None: + mappings.extend( + [ + PathMapping( + container_path=f"{skills_container_path}/custom", + local_path=str(projection.custom), + read_only=True, + ), + PathMapping( + container_path=f"{skills_container_path}/legacy", + local_path=str(projection.legacy), + read_only=True, + ), + PathMapping( + container_path=f"{skills_container_path}/integrations", + local_path=str(projection.integrations), + read_only=True, + ), + ] ) except Exception as exc: - logger.warning("Could not setup per-thread legacy skills mount: %s", exc, exc_info=True) + logger.warning("Could not setup per-thread skills projection mounts: %s", exc, exc_info=True) return mappings @@ -350,13 +374,23 @@ class LocalSandboxProvider(SandboxProvider): global _singleton if thread_id is None: + skill_projection = self._ensure_skills_projection() with self._lock: if self._generic_sandbox is None: - self._generic_sandbox = LocalSandbox("local", path_mappings=list(self._path_mappings)) + mappings = list(self._path_mappings) + self._append_public_skill_mapping(mappings, skill_projection) + self._generic_sandbox = LocalSandbox("local", path_mappings=mappings) _singleton = self._generic_sandbox return self._generic_sandbox.id effective_user_id = self._effective_acquire_user_id(user_id) + # Runs on every acquire, including cache hits, to self-heal drift — + # cheap (~3-4 ms metadata walk) when the manifest is fresh. If another + # worker mutated this user's skills since the last check, this + # triggers a full rebuild (~400 ms measured locally) under the + # cross-process projection lock, serializing concurrent acquires and + # mutations for that user. Acceptable for an editing-frequency event. + skill_projection = self._ensure_skills_projection(effective_user_id) key = self._thread_key(thread_id, effective_user_id) # Fast path under lock. @@ -366,11 +400,18 @@ class LocalSandboxProvider(SandboxProvider): # Mark as most-recently used so frequently-touched threads # survive eviction. self._thread_sandboxes.move_to_end(key) - return cached.id + if cached is not None: + return cached.id # ``_build_thread_path_mappings`` touches the filesystem # (``ensure_thread_dirs``); release the lock during I/O. - new_mappings = list(self._path_mappings) + self._build_thread_path_mappings(thread_id, user_id=effective_user_id) + new_mappings = list(self._path_mappings) + self._append_public_skill_mapping(new_mappings, skill_projection) + new_mappings += self._build_thread_path_mappings( + thread_id, + user_id=effective_user_id, + skill_projection=skill_projection, + ) with self._lock: # Re-check after the lock-free I/O: another caller may have diff --git a/backend/packages/harness/deerflow/skills/projection.py b/backend/packages/harness/deerflow/skills/projection.py new file mode 100644 index 000000000..ec9a3bf9a --- /dev/null +++ b/backend/packages/harness/deerflow/skills/projection.py @@ -0,0 +1,552 @@ +"""Materialize enabled-only skill trees for sandbox filesystem exposure.""" + +from __future__ import annotations + +import errno +import hashlib +import json +import logging +import os +import shutil +import tempfile +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from deerflow.skills.parser import parse_skill_file +from deerflow.skills.types import SKILL_MD_FILE, Skill, SkillCategory + +if TYPE_CHECKING: + from deerflow.skills.storage.skill_storage import SkillStorage + +logger = logging.getLogger(__name__) + +try: + import fcntl +except ImportError: # pragma: no cover - Windows + fcntl = None # type: ignore[assignment] + import msvcrt + +_locks_guard = threading.Lock() +_process_locks: dict[Path, threading.RLock] = {} +_MANIFEST_VERSION = 1 +_MAX_REBUILD_ATTEMPTS = 2 + + +@dataclass(frozen=True) +class SkillProjectionPaths: + """Stable category roots mounted or uploaded by sandbox providers.""" + + public: Path + custom: Path + legacy: Path + integrations: Path + + +def get_skill_projection_paths(storage: SkillStorage) -> SkillProjectionPaths: + from deerflow.config.paths import get_paths + + paths = getattr(storage, "_paths", None) or get_paths() + user_id = getattr(storage, "user_id", None) + if user_id is None: + return SkillProjectionPaths( + public=paths.public_skills_view_dir, + custom=paths.skills_view_dir / "custom", + legacy=paths.skills_view_dir / "legacy", + integrations=paths.skills_view_dir / "integrations", + ) + return SkillProjectionPaths( + public=paths.public_skills_view_dir, + custom=paths.user_custom_skills_view_dir(user_id), + legacy=paths.user_legacy_skills_view_dir(user_id), + integrations=paths.user_integration_skills_view_dir(user_id), + ) + + +def _lock_for(path: Path) -> threading.RLock: + resolved = path.resolve() + with _locks_guard: + return _process_locks.setdefault(resolved, threading.RLock()) + + +@contextmanager +def _projection_lock(root: Path) -> Iterator[None]: + """Serialize projection replacement in-process and across POSIX workers.""" + lock_path = root.parent / f".{root.name}.projection.lock" + lock_path.parent.mkdir(parents=True, exist_ok=True) + process_lock = _lock_for(lock_path) + with process_lock, lock_path.open("a", encoding="utf-8") as lock_file: + if fcntl is not None: + fcntl.flock(lock_file, fcntl.LOCK_EX) + else: # pragma: no cover - Windows + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_LOCK, 1) + try: + yield + finally: + if fcntl is not None: + fcntl.flock(lock_file, fcntl.LOCK_UN) + else: # pragma: no cover - Windows + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) + + +def _link_or_copy(source: str, target: str, *, follow_symlinks: bool = True) -> str: + # Hardlinks share the source inode and provide no write isolation. Any + # read-only guarantee must come from the consuming sandbox or mount. + try: + os.link(source, target, follow_symlinks=follow_symlinks) + except OSError as exc: + if exc.errno not in {errno.EXDEV, errno.EPERM, errno.EACCES, errno.ENOTSUP}: + raise + shutil.copy2(source, target, follow_symlinks=follow_symlinks) + return target + + +def _stage_skill(source: Path, target: Path, nested_skill_roots: set[Path]) -> None: + def _exclude_nested_skills(current: str, names: list[str]) -> list[str]: + relative_root = Path(current).relative_to(source) + return [name for name in names if relative_root / name in nested_skill_roots] + + shutil.copytree( + source, + target, + copy_function=_link_or_copy, + symlinks=True, + ignore=_exclude_nested_skills, + dirs_exist_ok=True, + ) + + +def _path_kind(path: Path) -> str: + if path.is_symlink(): + return "symlink" + if path.is_dir(): + return "directory" + return "file" + + +def _tree_entries(root: Path) -> dict[Path, str]: + entries: dict[Path, str] = {} + for current_root, dir_names, file_names in os.walk(root, followlinks=False): + current = Path(current_root) + for name in dir_names: + path = current / name + entries[path.relative_to(root)] = _path_kind(path) + dir_names[:] = [name for name in dir_names if not (current / name).is_symlink()] + for name in file_names: + path = current / name + entries[path.relative_to(root)] = _path_kind(path) + return entries + + +def _remove_projection_entry(path: Path) -> None: + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink(missing_ok=True) + + +def _validate_projection_relative_path(relative_path: Path) -> None: + if relative_path.is_absolute() or not relative_path.parts or any(part in {"", ".", ".."} for part in relative_path.parts): + raise ValueError("Projection removal path must identify a package within its category root") + + +def _remove_projection_relative(root: Path, relative_path: Path) -> None: + """Remove a projected package without following a drifted namespace symlink.""" + current = root + for part in relative_path.parts: + current /= part + if current.is_symlink(): + current.unlink() + return + _remove_projection_entry(current) + + +def _sync_staged_category(root: Path, staging: Path) -> None: + desired = _tree_entries(staging) + live = _tree_entries(root) + + for relative_path, live_kind in sorted(live.items(), key=lambda item: len(item[0].parts), reverse=True): + if desired.get(relative_path) != live_kind: + _remove_projection_entry(root / relative_path) + + for relative_path, kind in sorted(desired.items(), key=lambda item: len(item[0].parts)): + if kind == "directory": + (root / relative_path).mkdir(parents=True, exist_ok=True) + + for relative_path, kind in desired.items(): + if kind == "directory": + continue + target = root / relative_path + target.parent.mkdir(parents=True, exist_ok=True) + (staging / relative_path).replace(target) + + +def _replace_category(root: Path, desired: dict[Path, Skill], skill_boundaries: set[Path]) -> None: + """Reconcile entries beneath a stable category root without blanking it.""" + root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f".{root.name}.projection-", dir=root.parent) as staging_dir: + staging = Path(staging_dir) + for relative_path, skill in desired.items(): + nested_roots = {boundary.relative_to(relative_path) for boundary in skill_boundaries if boundary != relative_path and boundary.is_relative_to(relative_path)} + _stage_skill(skill.skill_dir, staging / relative_path, nested_roots) + _sync_staged_category(root, staging) + + +def _clear_category(root: Path) -> None: + root.mkdir(parents=True, exist_ok=True) + for path in root.iterdir(): + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink() + + +def _clear_projection_scope(scope_root: Path, *category_roots: Path) -> None: + for category_root in category_roots: + _clear_category(category_root) + _manifest_path(scope_root).unlink(missing_ok=True) + + +def _update_tree_digest(digest, root: Path, label: str) -> None: + """Hash directory metadata (inode/mode/size/mtime), not file contents. + + Trade-off: fast enough to run on every sandbox acquire (O(files), no + reads), but an external edit that preserves inode+size+mtime — unlikely, + not zero-probability — is invisible to this signature and leaves the + projection stale until the next explicit rebuild. Runtime writes through + this codebase are covered regardless: the mutation path rebuilds under + lock, and atomic-rename always changes the inode. + """ + digest.update(f"root:{label}\0".encode()) + if not root.exists(): + digest.update(b"absent\0") + return + + stack = [(root, Path("."))] + while stack: + current, relative_root = stack.pop() + with os.scandir(current) as entries: + ordered = sorted(entries, key=lambda entry: entry.name) + child_dirs: list[tuple[Path, Path]] = [] + for entry in ordered: + relative = relative_root / entry.name + metadata = entry.stat(follow_symlinks=False) + if entry.is_symlink(): + kind = "link" + elif entry.is_dir(follow_symlinks=False): + kind = "dir" + child_dirs.append((Path(entry.path), relative)) + else: + kind = "file" + digest.update((f"{label}:{relative.as_posix()}:{kind}:{metadata.st_ino}:{metadata.st_mode}:{metadata.st_size}:{metadata.st_mtime_ns}\0").encode()) + stack.extend(reversed(child_dirs)) + + +def _extensions_state() -> dict: + from deerflow.config.extensions_config import ExtensionsConfig + + config = ExtensionsConfig.from_file() + return {name: state.model_dump(mode="json") for name, state in config.skills.items()} + + +def _source_signature(storage: SkillStorage, scope: str) -> str: + digest = hashlib.sha256() + host_root = storage.get_skills_root_path() + if scope == "public": + _update_tree_digest(digest, host_root / SkillCategory.PUBLIC.value, "public") + state = {"extensions": _extensions_state()} + elif scope == "user": + user_custom_root = storage.get_user_custom_root() + integration_root = storage.get_user_integrations_root() + _update_tree_digest(digest, user_custom_root, "custom") + _update_tree_digest(digest, host_root / SkillCategory.CUSTOM.value, "legacy") + _update_tree_digest(digest, integration_root, "integrations") + # CUSTOM/LEGACY/INTEGRATION visibility is the intersection of the + # per-user state and the global extensions default, so both belong in + # this signature. + state = { + "extensions": _extensions_state(), + "user": storage._read_skill_states(), + } + else: # pragma: no cover - internal invariant + raise ValueError(f"Unknown skill projection scope: {scope}") + digest.update(json.dumps(state, sort_keys=True, separators=(",", ":")).encode()) + return digest.hexdigest() + + +def _manifest_path(scope_root: Path) -> Path: + return scope_root / ".projection-manifest.json" + + +def _read_manifest(scope_root: Path) -> dict | None: + try: + value = json.loads(_manifest_path(scope_root).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _write_manifest(scope_root: Path, source_signature: str) -> None: + scope_root.mkdir(parents=True, exist_ok=True) + target = _manifest_path(scope_root) + fd, temporary_name = tempfile.mkstemp(prefix=".projection-manifest-", suffix=".tmp", dir=scope_root) + temporary = Path(temporary_name) + try: + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump({"version": _MANIFEST_VERSION, "source_signature": source_signature}, stream, sort_keys=True) + temporary.replace(target) + except Exception: + temporary.unlink(missing_ok=True) + raise + + +def _load_public_skills(storage: SkillStorage, *, enabled_only: bool) -> list[Skill]: + from deerflow.config.extensions_config import ExtensionsConfig + + public_root = storage.get_skills_root_path() / SkillCategory.PUBLIC.value + if not public_root.is_dir(): + return [] + extensions = ExtensionsConfig.from_file() + skills: list[Skill] = [] + for current_root, dir_names, file_names in os.walk(public_root, followlinks=True): + dir_names[:] = sorted(name for name in dir_names if not name.startswith(".")) + if SKILL_MD_FILE not in file_names: + continue + # Match the runtime loader: nested SKILL.md files inside a package are + # support data, not independently configurable skills. + dir_names.clear() + skill_file = Path(current_root) / SKILL_MD_FILE + skill = parse_skill_file( + skill_file, + category=SkillCategory.PUBLIC, + relative_path=skill_file.parent.relative_to(public_root), + ) + if skill is None: + continue + enabled = extensions.is_skill_enabled(skill.name, SkillCategory.PUBLIC.value) + if not enabled_only or enabled: + skills.append(skill) + return skills + + +def _by_relative_path(skills: list[Skill], category: SkillCategory) -> dict[Path, Skill]: + return {skill.relative_path: skill for skill in skills if skill.category == category} + + +def _category_boundaries(skills: list[Skill], category: SkillCategory) -> set[Path]: + return {skill.relative_path for skill in skills if skill.category == category} + + +def _rebuild_public_locked(storage: SkillStorage, paths: SkillProjectionPaths) -> None: + scope_root = paths.public.parent + try: + for _attempt in range(_MAX_REBUILD_ATTEMPTS): + before = _source_signature(storage, "public") + all_public_skills = _load_public_skills(storage, enabled_only=False) + enabled_public_skills = _load_public_skills(storage, enabled_only=True) + _replace_category( + paths.public, + _by_relative_path(enabled_public_skills, SkillCategory.PUBLIC), + _category_boundaries(all_public_skills, SkillCategory.PUBLIC), + ) + after = _source_signature(storage, "public") + if before == after: + _write_manifest(scope_root, after) + return + raise RuntimeError("Public skills changed repeatedly while rebuilding the sandbox projection") + except Exception: + _clear_projection_scope(scope_root, paths.public) + raise + + +def _rebuild_user_locked(storage: SkillStorage, paths: SkillProjectionPaths) -> None: + scope_root = paths.custom.parent + try: + for _attempt in range(_MAX_REBUILD_ATTEMPTS): + before = _source_signature(storage, "user") + all_user_skills = storage.load_skills(enabled_only=False) + enabled_user_skills = [skill for skill in all_user_skills if skill.enabled] + _replace_category( + paths.custom, + _by_relative_path(enabled_user_skills, SkillCategory.CUSTOM), + _category_boundaries(all_user_skills, SkillCategory.CUSTOM), + ) + _replace_category( + paths.legacy, + _by_relative_path(enabled_user_skills, SkillCategory.LEGACY), + _category_boundaries(all_user_skills, SkillCategory.LEGACY), + ) + _replace_category( + paths.integrations, + _by_relative_path(enabled_user_skills, SkillCategory.INTEGRATION), + _category_boundaries(all_user_skills, SkillCategory.INTEGRATION), + ) + after = _source_signature(storage, "user") + if before == after: + _write_manifest(scope_root, after) + return + raise RuntimeError("User skills changed repeatedly while rebuilding the sandbox projection") + except Exception: + _clear_projection_scope(scope_root, paths.custom, paths.legacy, paths.integrations) + raise + + +def rebuild_skill_projections( + storage: SkillStorage, + *, + include_public: bool = True, + include_user: bool = True, +) -> SkillProjectionPaths: + """Rebuild enabled-only projection scopes visible through ``storage``.""" + paths = get_skill_projection_paths(storage) + user_id = getattr(storage, "user_id", None) + if include_public: + with _projection_lock(paths.public.parent): + _rebuild_public_locked(storage, paths) + if include_user and user_id is not None: + with _projection_lock(paths.custom.parent): + _rebuild_user_locked(storage, paths) + + return paths + + +def _public_projection_is_fresh(storage: SkillStorage, paths: SkillProjectionPaths) -> bool: + if not paths.public.is_dir(): + return False + manifest_before = _read_manifest(paths.public.parent) + if manifest_before is None or manifest_before.get("version") != _MANIFEST_VERSION: + return False + signature = _source_signature(storage, "public") + manifest_after = _read_manifest(paths.public.parent) + return manifest_before == manifest_after and manifest_before.get("source_signature") == signature + + +def ensure_skill_projections(storage: SkillStorage) -> SkillProjectionPaths: + """Repair stale projection scopes, otherwise leave their inodes untouched.""" + paths = get_skill_projection_paths(storage) + + try: + public_is_fresh = _public_projection_is_fresh(storage, paths) + except Exception: + # Re-check under the mutation lock before failing closed. A concurrent + # writer may have exposed a transient source/manifest state. + public_is_fresh = False + if not public_is_fresh: + with _projection_lock(paths.public.parent): + try: + if not _public_projection_is_fresh(storage, paths): + _rebuild_public_locked(storage, paths) + except Exception: + _clear_projection_scope(paths.public.parent, paths.public) + raise + + if getattr(storage, "user_id", None) is not None: + with _projection_lock(paths.custom.parent): + try: + manifest = _read_manifest(paths.custom.parent) + signature = _source_signature(storage, "user") + if not paths.custom.is_dir() or not paths.legacy.is_dir() or not paths.integrations.is_dir() or manifest is None or manifest.get("version") != _MANIFEST_VERSION or manifest.get("source_signature") != signature: + _rebuild_user_locked(storage, paths) + except Exception: + _clear_projection_scope(paths.custom.parent, paths.custom, paths.legacy, paths.integrations) + raise + return paths + + +@contextmanager +def skill_projection_mutation( + storage: SkillStorage, + scope: str, + *, + remove: tuple[tuple[SkillCategory, Path], ...] = (), + remove_names: tuple[str, ...] = (), +) -> Iterator[None]: + """Hold a projection scope lock across a source/state mutation.""" + if not isinstance(storage.get_skills_root_path(), Path): + # Lightweight unit-test doubles sometimes return MagicMock here. The + # SkillStorage contract requires a Path; real storage implementations + # therefore never take this compatibility branch. + yield + return + paths = get_skill_projection_paths(storage) + if scope == "public": + scope_root = paths.public.parent + category_roots = {SkillCategory.PUBLIC: paths.public} + + def rebuild() -> None: + _rebuild_public_locked(storage, paths) + + elif scope == "user": + scope_root = paths.custom.parent + category_roots = { + SkillCategory.CUSTOM: paths.custom, + SkillCategory.LEGACY: paths.legacy, + SkillCategory.INTEGRATION: paths.integrations, + } + + def rebuild() -> None: + _rebuild_user_locked(storage, paths) + + else: + raise ValueError(f"Unknown skill projection scope: {scope}") + + removals: set[tuple[Path, Path]] = set() + for category, relative_path in remove: + root = category_roots.get(category) + if root is None: + raise ValueError(f"Skill category {category.value!r} does not belong to projection scope {scope!r}") + _validate_projection_relative_path(relative_path) + removals.add((root, relative_path)) + + with _projection_lock(scope_root): + if remove_names: + names = set(remove_names) + skills = _load_public_skills(storage, enabled_only=False) if scope == "public" else storage.load_skills(enabled_only=False) + for skill in skills: + root = category_roots.get(skill.category) + if skill.name not in names or root is None: + continue + _validate_projection_relative_path(skill.relative_path) + removals.add((root, skill.relative_path)) + + try: + _manifest_path(scope_root).unlink(missing_ok=True) + for root, relative_path in removals: + _remove_projection_relative(root, relative_path) + yield + rebuild() + except Exception: + _clear_projection_scope(scope_root, *category_roots.values()) + raise + + +def ensure_public_skill_projection(*, app_config=None) -> bool: + """Ensure the global public view during boot without scanning user data. + + User projections are repaired lazily by sandbox acquire. Eagerly rebuilding + every historical user would make gateway readiness scale with tenant count, + while providing no additional safety before that user's next acquire. + """ + from deerflow.config import get_app_config + from deerflow.config.paths import get_paths + from deerflow.skills.storage import get_or_new_skill_storage + + try: + config = app_config or get_app_config() + public_storage = get_or_new_skill_storage(app_config=config) + ensure_skill_projections(public_storage) + except Exception: + logger.warning("Failed to ensure the public skill projection during boot; clearing it until a sandbox acquire self-heals it", exc_info=True) + try: + paths = get_paths() + with _projection_lock(paths.public_skills_view_dir.parent): + _clear_projection_scope(paths.public_skills_view_dir.parent, paths.public_skills_view_dir) + except Exception: + logger.error("Failed to clear the public skill projection after a boot-time error", exc_info=True) + return False + return True diff --git a/backend/packages/harness/deerflow/skills/storage/local_skill_storage.py b/backend/packages/harness/deerflow/skills/storage/local_skill_storage.py index 529a23500..e0ffc0a81 100644 --- a/backend/packages/harness/deerflow/skills/storage/local_skill_storage.py +++ b/backend/packages/harness/deerflow/skills/storage/local_skill_storage.py @@ -10,6 +10,7 @@ import os import shutil import tempfile from collections.abc import Iterable +from contextlib import nullcontext from datetime import UTC, datetime from pathlib import Path @@ -107,8 +108,18 @@ class LocalSkillStorage(SkillStorage): ) as tmp_file: tmp_file.write(content) tmp_path = Path(tmp_file.name) - tmp_path.replace(target) - make_skill_written_path_sandbox_readable(self.get_custom_skill_dir(name), target) + try: + with self._skill_projection_mutation(): + tmp_path.replace(target) + make_skill_written_path_sandbox_readable(self.get_custom_skill_dir(name), target) + except Exception: + tmp_path.unlink(missing_ok=True) + raise + + def remove_custom_skill_file(self, name: str, relative_path: str) -> str: + removal = ((SkillCategory.CUSTOM, Path(name)),) + with self._skill_projection_mutation(remove=removal): + return super().remove_custom_skill_file(name, relative_path) async def ainstall_skill_from_archive(self, archive_path: str | Path) -> dict: from deerflow.skills.installer import _scan_skill_archive_contents_or_raise @@ -200,11 +211,12 @@ class LocalSkillStorage(SkillStorage): """Stage and move the validated skill into place (blocking; runs off the event loop).""" from deerflow.skills.installer import _move_staged_skill_into_reserved_target - with tempfile.TemporaryDirectory(prefix=f".installing-{skill_name}-", dir=custom_dir) as staging_root: - staging_target = Path(staging_root) / skill_name - shutil.copytree(skill_dir, staging_target) - _move_staged_skill_into_reserved_target(staging_target, target) - make_skill_written_path_sandbox_readable(custom_dir, target) + with self._skill_projection_mutation(): + with tempfile.TemporaryDirectory(prefix=f".installing-{skill_name}-", dir=custom_dir) as staging_root: + staging_target = Path(staging_root) / skill_name + shutil.copytree(skill_dir, staging_target) + _move_staged_skill_into_reserved_target(staging_target, target) + make_skill_written_path_sandbox_readable(custom_dir, target) def delete_custom_skill(self, name: str, *, history_meta: dict | None = None) -> None: self.validate_skill_name(name) @@ -222,8 +234,22 @@ class LocalSkillStorage(SkillStorage): name, e, ) - if target.exists(): - shutil.rmtree(target) + removal = ((SkillCategory.CUSTOM, Path(name)),) + with self._skill_projection_mutation(remove=removal): + if target.exists(): + shutil.rmtree(target) + + def _skill_projection_mutation( + self, + *, + remove: tuple[tuple[SkillCategory, Path], ...] = (), + remove_names: tuple[str, ...] = (), + ): + if getattr(self, "user_id", None) is None: + return nullcontext() + from deerflow.skills.projection import skill_projection_mutation + + return skill_projection_mutation(self, "user", remove=remove, remove_names=remove_names) def append_history(self, name: str, record: dict) -> None: self.validate_skill_name(name) diff --git a/backend/packages/harness/deerflow/skills/storage/skill_storage.py b/backend/packages/harness/deerflow/skills/storage/skill_storage.py index 39e7e5d6e..c31265531 100644 --- a/backend/packages/harness/deerflow/skills/storage/skill_storage.py +++ b/backend/packages/harness/deerflow/skills/storage/skill_storage.py @@ -155,6 +155,15 @@ class SkillStorage(ABC): Origin: ``deerflow.skills.manager.atomic_write``. """ + def remove_custom_skill_file(self, name: str, relative_path: str) -> str: + """Remove a supporting file and return its previous text content.""" + target = self.ensure_safe_support_path(name, relative_path) + if not target.exists(): + raise FileNotFoundError(f"Supporting file '{relative_path}' not found for skill '{name}'.") + previous_content = target.read_text(encoding="utf-8") + target.unlink() + return previous_content + @abstractmethod async def ainstall_skill_from_archive(self, archive_path: str | Path) -> dict: """Async install of a skill from a ``.skill`` ZIP archive. diff --git a/backend/packages/harness/deerflow/skills/storage/user_scoped_skill_storage.py b/backend/packages/harness/deerflow/skills/storage/user_scoped_skill_storage.py index 8cd0b99ef..700edf66a 100644 --- a/backend/packages/harness/deerflow/skills/storage/user_scoped_skill_storage.py +++ b/backend/packages/harness/deerflow/skills/storage/user_scoped_skill_storage.py @@ -82,6 +82,7 @@ class UserScopedSkillStorage(LocalSkillStorage): self._user_id = _validate_user_id(user_id) paths = get_paths() + self._paths = paths self._user_custom_root: Path = paths.user_custom_skills_dir(self._user_id) self._integrations_root: Path = paths.integration_skills_dir() self._user_skills_root: Path = paths.user_skills_dir(self._user_id) @@ -153,9 +154,11 @@ class UserScopedSkillStorage(LocalSkillStorage): def set_skill_enabled_state(self, skill_name: str, enabled: bool) -> None: """Set the enabled state for a custom/legacy skill and persist.""" - states = self._read_skill_states() - states[skill_name] = {"enabled": enabled} - self._write_skill_states(states) + removal_names = (skill_name,) if not enabled else () + with self._skill_projection_mutation(remove_names=removal_names): + states = self._read_skill_states() + states[skill_name] = {"enabled": enabled} + self._write_skill_states(states) # ------------------------------------------------------------------ # Path helpers — redirect custom skill paths to user directory @@ -204,10 +207,12 @@ class UserScopedSkillStorage(LocalSkillStorage): # being silently re-enabled by an absent per-user entry, while still # letting the per-user state override the global default when both # are present. PUBLIC skill state remains governed solely by - # extensions_config (handled by ``super().load_skills`` above). - from deerflow.config.extensions_config import get_extensions_config + # extensions_config (handled by ``super().load_skills`` above). Re-read + # from disk here too so another worker's update cannot be masked by + # this process's singleton cache while rebuilding a user projection. + from deerflow.config.extensions_config import ExtensionsConfig - extensions_config = get_extensions_config() + extensions_config = ExtensionsConfig.from_file() skills = [ dataclasses.replace(s, enabled=self.get_skill_enabled_state(s.name) and extensions_config.is_skill_enabled(s.name, s.category.value if hasattr(s.category, "value") else s.category)) if dataclasses.is_dataclass(s) and not isinstance(s, type) and (s.category.value if hasattr(s.category, "value") else s.category) != SkillCategory.PUBLIC.value @@ -369,8 +374,13 @@ class UserScopedSkillStorage(LocalSkillStorage): ) as tmp_file: tmp_file.write(content) tmp_path = Path(tmp_file.name) - tmp_path.replace(target) - make_skill_written_path_sandbox_readable(self.get_custom_skill_dir(name), target) + try: + with self._skill_projection_mutation(): + tmp_path.replace(target) + make_skill_written_path_sandbox_readable(self.get_custom_skill_dir(name), target) + except Exception: + tmp_path.unlink(missing_ok=True) + raise # ------------------------------------------------------------------ # Public helpers diff --git a/backend/packages/harness/deerflow/tools/skill_manage_tool.py b/backend/packages/harness/deerflow/tools/skill_manage_tool.py index a62286add..779de1bc4 100644 --- a/backend/packages/harness/deerflow/tools/skill_manage_tool.py +++ b/backend/packages/harness/deerflow/tools/skill_manage_tool.py @@ -242,11 +242,7 @@ async def _skill_manage_impl( await _to_thread(skill_storage.ensure_custom_skill_is_editable, name) if path is None: raise ValueError("path is required for remove_file.") - target = await _to_thread(skill_storage.ensure_safe_support_path, name, path) - if not await _to_thread(target.exists): - raise FileNotFoundError(f"Supporting file '{path}' not found for skill '{name}'.") - prev_content = await _to_thread(target.read_text, encoding="utf-8") - await _to_thread(target.unlink) + prev_content = await _to_thread(skill_storage.remove_custom_skill_file, name, path) await _to_thread( skill_storage.append_history, name, diff --git a/backend/tests/blocking_io/test_skills_update_router.py b/backend/tests/blocking_io/test_skills_update_router.py index aa01fe44c..0bbc3695d 100644 --- a/backend/tests/blocking_io/test_skills_update_router.py +++ b/backend/tests/blocking_io/test_skills_update_router.py @@ -119,6 +119,22 @@ async def test_update_skill_writes_from_snapshot_without_mutating_singleton(tmp_ assert "middlewares" in written +async def test_update_skill_persists_state_when_source_omits_skills(tmp_path: Path, monkeypatch) -> None: + config_path = tmp_path / "extensions_config.json" + await asyncio.to_thread( + config_path.write_text, + json.dumps({"mcpServers": {}, "middlewares": ["pkg:Middleware"]}), + encoding="utf-8", + ) + _patch_config_infra(monkeypatch, config_path) + + await update_skill("demo-skill", SkillUpdateRequest(enabled=False), _admin_request(), SimpleNamespace()) + + written = json.loads(await asyncio.to_thread(config_path.read_text, encoding="utf-8")) + assert written["skills"] == {"demo-skill": {"enabled": False}} + assert written["middlewares"] == ["pkg:Middleware"] + + @pytest.mark.allow_blocking_io # gate-exempt: needs real worker-thread overlap to observe serialization async def test_update_skill_serializes_concurrent_writes(tmp_path: Path, monkeypatch) -> None: state_lock = threading.Lock() diff --git a/backend/tests/test_aio_sandbox_provider.py b/backend/tests/test_aio_sandbox_provider.py index b603d776b..3fabd9757 100644 --- a/backend/tests/test_aio_sandbox_provider.py +++ b/backend/tests/test_aio_sandbox_provider.py @@ -219,8 +219,9 @@ def test_get_user_skill_mounts_mounts_only_global_integrations(tmp_path, monkeyp assert set(alice) == {"/mnt/skills/integrations"} assert set(bob) == {"/mnt/skills/integrations"} - assert alice["/mnt/skills/integrations"] == bob["/mnt/skills/integrations"] - assert alice["/mnt/skills/integrations"] == str(tmp_path / "home" / "integrations" / "skills") + assert alice["/mnt/skills/integrations"] != bob["/mnt/skills/integrations"] + assert alice["/mnt/skills/integrations"] == str(tmp_path / "home" / "users" / "alice" / "skills_view" / "integrations") + assert bob["/mnt/skills/integrations"] == str(tmp_path / "home" / "users" / "bob" / "skills_view" / "integrations") def test_get_extra_mounts_provisioner_payload_has_unique_container_paths(tmp_path, monkeypatch, provisioner_module): @@ -243,7 +244,7 @@ def test_get_extra_mounts_provisioner_payload_has_unique_container_paths(tmp_pat monkeypatch.setattr(aio_mod, "get_app_config", lambda: config) monkeypatch.setattr(aio_mod, "get_paths", lambda: Paths(base_dir=home)) monkeypatch.setattr(aio_mod, "get_effective_user_id", lambda: "default") - monkeypatch.setattr(aio_mod, "user_should_see_legacy_skills", lambda *_args, **_kwargs: False) + monkeypatch.setattr(remote_backend, "user_should_see_legacy_skills", lambda *_args, **_kwargs: False) provider = _make_provider(tmp_path) mounts = provider._get_extra_mounts("thread-1", user_id="alice") @@ -526,7 +527,10 @@ async def test_acquire_internal_async_offloads_cached_reuse_health_check(tmp_pat sandbox_id = await provider._acquire_internal_async("thread-cached-async", user_id="default") assert sandbox_id == "sandbox-cached-async" - assert to_thread_calls == [(provider._reuse_in_process_sandbox, ("thread-cached-async",))] + assert to_thread_calls == [ + (provider._ensure_skills_projection, ("default",)), + (provider._reuse_in_process_sandbox, ("thread-cached-async",)), + ] def test_remote_backend_create_forwards_effective_user_id(monkeypatch): @@ -548,7 +552,7 @@ def test_remote_backend_create_forwards_effective_user_id(monkeypatch): return _Response() monkeypatch.setattr(remote_mod.requests, "post", _post) - monkeypatch.setattr(remote_mod, "user_should_see_legacy_skills", lambda user_id: True) + monkeypatch.setattr(remote_mod, "user_should_see_legacy_skills", lambda _user_id: True) try: backend.create("thread-42", "sandbox-42") @@ -585,7 +589,7 @@ def test_remote_backend_create_prefers_explicit_user_id(monkeypatch): monkeypatch.setattr(remote_mod.requests, "post", _post) monkeypatch.setattr(remote_mod, "get_effective_user_id", lambda: "default") - monkeypatch.setattr(remote_mod, "user_should_see_legacy_skills", lambda user_id: False) + monkeypatch.setattr(remote_mod, "user_should_see_legacy_skills", lambda _user_id: False) backend.create("thread-42", "sandbox-42", user_id="ou-user") diff --git a/backend/tests/test_client.py b/backend/tests/test_client.py index 5516632ad..0a41ddcdd 100644 --- a/backend/tests/test_client.py +++ b/backend/tests/test_client.py @@ -1861,10 +1861,8 @@ class TestSkillsManagement: skill = self._make_skill(enabled=True) updated_skill = self._make_skill(enabled=False) - ext_config = ExtensionsConfig() - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - json.dump({}, f) + json.dump({"mcpServers": {}, "skills": {"untouched-skill": {"enabled": False}}}, f) tmp_path = Path(f.name) try: @@ -1881,12 +1879,37 @@ class TestSkillsManagement: side_effect=[[skill], [skill], [updated_skill], [updated_skill]], ), patch("deerflow.client.ExtensionsConfig.resolve_config_path", return_value=tmp_path), - patch("deerflow.client.get_extensions_config", return_value=ext_config), patch("deerflow.client.reload_extensions_config"), ): result = client.update_skill("test-skill", enabled=False) assert result["enabled"] is False assert client._agent is None # M2: agent invalidated + persisted = json.loads(tmp_path.read_text(encoding="utf-8")) + assert persisted["skills"]["untouched-skill"] == {"enabled": False} + finally: + tmp_path.unlink() + + def test_update_skill_persists_state_when_source_omits_skills(self, client): + skill = self._make_skill(enabled=True) + updated_skill = self._make_skill(enabled=False) + + with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: + json.dump({"mcpServers": {}}, f) + tmp_path = Path(f.name) + + try: + with ( + patch( + "deerflow.skills.storage.local_skill_storage.LocalSkillStorage.load_skills", + side_effect=[[skill], [skill], [updated_skill], [updated_skill]], + ), + patch("deerflow.client.ExtensionsConfig.resolve_config_path", return_value=tmp_path), + patch("deerflow.client.reload_extensions_config"), + ): + client.update_skill("test-skill", enabled=False) + + persisted = json.loads(tmp_path.read_text(encoding="utf-8")) + assert persisted["skills"] == {"test-skill": {"enabled": False}} finally: tmp_path.unlink() diff --git a/backend/tests/test_e2b_sandbox_provider.py b/backend/tests/test_e2b_sandbox_provider.py index e49a5f435..51d90172c 100644 --- a/backend/tests/test_e2b_sandbox_provider.py +++ b/backend/tests/test_e2b_sandbox_provider.py @@ -10,6 +10,7 @@ import threading import time from collections import OrderedDict from concurrent.futures import ThreadPoolExecutor +from pathlib import Path from types import SimpleNamespace from typing import Any from unittest.mock import MagicMock @@ -313,6 +314,100 @@ def _install_fake_sdk(monkeypatch, provider) -> FakeSandboxClass: return fake_cls +def _write_skill(root: Path, name: str) -> None: + target = root / name / "SKILL.md" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(f"---\nname: {name}\ndescription: test\n---\n", encoding="utf-8") + + +def test_apply_mounts_uploads_only_enabled_skill_projection(monkeypatch, tmp_path): + from deerflow.config.extensions_config import ExtensionsConfig, SkillStateConfig + + mod = importlib.import_module("deerflow.community.e2b_sandbox.e2b_sandbox_provider") + paths = Paths(base_dir=tmp_path) + skills_root = tmp_path / "skills" + _write_skill(skills_root / "public", "enabled-skill") + _write_skill(skills_root / "public", "disabled-skill") + (skills_root / "custom").mkdir() + _write_skill(paths.integration_skills_dir() / "lark-cli", "enabled-integration") + _write_skill(paths.integration_skills_dir() / "lark-cli", "disabled-integration") + user_skills_root = paths.user_skills_dir("user-1") + user_skills_root.mkdir(parents=True, exist_ok=True) + (user_skills_root / "_skill_states.json").write_text( + json.dumps({"disabled-integration": {"enabled": False}}), + encoding="utf-8", + ) + extensions = ExtensionsConfig(skills={"disabled-skill": SkillStateConfig(enabled=False)}) + config = SimpleNamespace( + skills=SimpleNamespace( + get_skills_path=lambda: skills_root, + container_path="/mnt/skills", + use="deerflow.skills.storage.local_skill_storage:LocalSkillStorage", + ) + ) + monkeypatch.setattr(mod, "get_app_config", lambda: config) + monkeypatch.setattr("deerflow.config.paths.get_paths", lambda: paths) + monkeypatch.setattr("deerflow.config.extensions_config.ExtensionsConfig.from_file", lambda *_args, **_kwargs: extensions) + monkeypatch.setattr("deerflow.config.extensions_config.get_extensions_config", lambda: extensions) + + provider = _make_provider() + client = FakeClient() + provider._apply_mounts(client, user_id="user-1") + + uploaded_paths = {path for path, _content in client.files.write_calls} + assert "/mnt/skills/public/enabled-skill/SKILL.md" in uploaded_paths + assert "/mnt/skills/public/disabled-skill/SKILL.md" not in uploaded_paths + assert "/mnt/skills/integrations/lark-cli/enabled-integration/SKILL.md" in uploaded_paths + assert "/mnt/skills/integrations/lark-cli/disabled-integration/SKILL.md" not in uploaded_paths + + +def test_skill_projection_mounts_swallows_projection_failure(monkeypatch): + """``_skill_projection_mounts`` must not raise — a projection failure used + to propagate out of ``_apply_mounts`` before the configured-mounts loop + ran, dropping the operator's own configured mounts as collateral (#4107 + review).""" + mod = importlib.import_module("deerflow.community.e2b_sandbox.e2b_sandbox_provider") + + config = SimpleNamespace(skills=SimpleNamespace(container_path="/mnt/skills")) + monkeypatch.setattr(mod, "get_app_config", lambda: config) + monkeypatch.setattr( + "deerflow.skills.projection.ensure_skill_projections", + lambda storage: (_ for _ in ()).throw(RuntimeError("simulated projection failure")), + ) + + provider = _make_provider() + + assert provider._skill_projection_mounts("user-1") == [] + + +def test_apply_mounts_keeps_configured_mounts_when_projection_fails(monkeypatch, tmp_path): + """End-to-end: a skills-projection failure must not drop the operator's + own configured mounts too — the two mount sources are independent.""" + mod = importlib.import_module("deerflow.community.e2b_sandbox.e2b_sandbox_provider") + + host_dir = tmp_path / "operator-mount" + host_dir.mkdir() + (host_dir / "notes.txt").write_text("hello", encoding="utf-8") + + config = SimpleNamespace(skills=SimpleNamespace(container_path="/mnt/skills")) + monkeypatch.setattr(mod, "get_app_config", lambda: config) + monkeypatch.setattr( + "deerflow.skills.projection.ensure_skill_projections", + lambda storage: (_ for _ in ()).throw(RuntimeError("simulated projection failure")), + ) + + provider = _make_provider() + provider._config["mounts"] = [ + SimpleNamespace(host_path=str(host_dir), container_path="/mnt/operator", read_only=True), + ] + + client = FakeClient() + provider._apply_mounts(client, user_id="user-1") + + uploaded_paths = {path for path, _content in client.files.write_calls} + assert "/mnt/operator/notes.txt" in uploaded_paths + + def _make_sandbox(client: FakeClient, *, sandbox_id: str | None = None) -> Any: mod = importlib.import_module("deerflow.community.e2b_sandbox.e2b_sandbox") return mod.E2BSandbox( diff --git a/backend/tests/test_gateway_lifespan_shutdown.py b/backend/tests/test_gateway_lifespan_shutdown.py index 7ee799051..049d84d14 100644 --- a/backend/tests/test_gateway_lifespan_shutdown.py +++ b/backend/tests/test_gateway_lifespan_shutdown.py @@ -52,6 +52,7 @@ async def _run_lifespan_with_hanging_stop() -> float: patch("app.gateway.app.get_app_config", return_value=startup_config), patch("app.gateway.app.get_gateway_config", return_value=MagicMock(host="x", port=0)), patch("app.gateway.app.langgraph_runtime", _noop_langgraph_runtime), + patch("deerflow.skills.projection.ensure_public_skill_projection"), patch("app.gateway.app.auth.close_oidc_service", close_oidc_service), patch("app.channels.service.start_channel_service", side_effect=fake_start), patch("app.channels.service.stop_channel_service", side_effect=hang_forever), @@ -98,6 +99,7 @@ async def _run_lifespan_with_upload_staging_cleanup(): patch("app.gateway.app.get_app_config", return_value=startup_config), patch("app.gateway.app.get_gateway_config", return_value=MagicMock(host="x", port=0)), patch("app.gateway.app.langgraph_runtime", _noop_langgraph_runtime), + patch("deerflow.skills.projection.ensure_public_skill_projection"), patch("app.gateway.app.cleanup_stale_upload_staging_files", cleanup_upload_staging_files), patch("app.gateway.app.auth.close_oidc_service", close_oidc_service), patch("app.channels.service.start_channel_service", side_effect=fake_start), @@ -156,6 +158,7 @@ async def _run_lifespan_with_memory_flush(*, enabled: bool, flush_return: bool | patch("app.gateway.app.get_app_config", return_value=startup_config), patch("app.gateway.app.get_gateway_config", return_value=MagicMock(host="x", port=0)), patch("app.gateway.app.langgraph_runtime", _noop_langgraph_runtime), + patch("deerflow.skills.projection.ensure_public_skill_projection"), patch("app.gateway.app.auth.close_oidc_service", close_oidc_service), patch("app.channels.service.start_channel_service", side_effect=fake_start), patch("app.channels.service.stop_channel_service", stop_channel_service), diff --git a/backend/tests/test_local_sandbox_provider_mounts.py b/backend/tests/test_local_sandbox_provider_mounts.py index a6ae73037..a0093537e 100644 --- a/backend/tests/test_local_sandbox_provider_mounts.py +++ b/backend/tests/test_local_sandbox_provider_mounts.py @@ -544,7 +544,7 @@ class TestMultipleMounts: class TestLocalSandboxProviderMounts: - def test_thread_mappings_mount_global_integrations_for_every_user(self, tmp_path): + def test_thread_mappings_mount_per_user_integration_projections(self, tmp_path): from deerflow.config.paths import Paths paths = Paths(base_dir=tmp_path / "home") @@ -568,10 +568,11 @@ class TestLocalSandboxProviderMounts: alice_integrations = next(mapping for mapping in alice if mapping.container_path == "/mnt/skills/integrations") bob_integrations = next(mapping for mapping in bob if mapping.container_path == "/mnt/skills/integrations") - expected = str(tmp_path / "home" / "integrations" / "skills") - assert alice_integrations.local_path == expected - assert bob_integrations.local_path == expected + assert alice_integrations.local_path == str(paths.user_integration_skills_view_dir("alice")) + assert bob_integrations.local_path == str(paths.user_integration_skills_view_dir("bob")) + assert alice_integrations.local_path != bob_integrations.local_path assert alice_integrations.read_only is True + assert bob_integrations.read_only is True def test_setup_path_mappings_uses_configured_skills_container_path_as_reserved_prefix(self, tmp_path): skills_dir = tmp_path / "skills" diff --git a/backend/tests/test_monocle_tracing.py b/backend/tests/test_monocle_tracing.py index b6a10a312..5491988d9 100644 --- a/backend/tests/test_monocle_tracing.py +++ b/backend/tests/test_monocle_tracing.py @@ -339,6 +339,7 @@ def test_gateway_lifespan_initializes_monocle(): patch("app.gateway.app.get_app_config", return_value=startup_config), patch("app.gateway.app.get_gateway_config", return_value=MagicMock(host="x", port=0)), patch("app.gateway.app.langgraph_runtime", _noop_langgraph_runtime), + patch("deerflow.skills.projection.ensure_public_skill_projection"), patch("app.gateway.app.setup_monocle_tracing_if_enabled", setup_spy), patch("app.gateway.app.auth.close_oidc_service", AsyncMock()), patch("app.channels.service.start_channel_service", side_effect=fake_start), @@ -381,6 +382,7 @@ def test_gateway_lifespan_survives_monocle_setup_failure(caplog): patch("app.gateway.app.get_app_config", return_value=startup_config), patch("app.gateway.app.get_gateway_config", return_value=MagicMock(host="x", port=0)), patch("app.gateway.app.langgraph_runtime", _noop_langgraph_runtime), + patch("deerflow.skills.projection.ensure_public_skill_projection"), patch("app.gateway.app.setup_monocle_tracing_if_enabled", setup_spy), patch("app.gateway.app.auth.close_oidc_service", AsyncMock()), patch("app.channels.service.start_channel_service", side_effect=fake_start), diff --git a/backend/tests/test_provisioner_pvc_volumes.py b/backend/tests/test_provisioner_pvc_volumes.py index 9acb96252..656654d12 100644 --- a/backend/tests/test_provisioner_pvc_volumes.py +++ b/backend/tests/test_provisioner_pvc_volumes.py @@ -10,12 +10,12 @@ class TestBuildVolumes: # ── hostPath mode (default) ──────────────────────────────────────── - def test_hostpath_without_legacy_returns_three_volumes(self, provisioner_module): - """hostPath mode omits legacy volume unless the backend requests it.""" + def test_hostpath_uses_three_projection_volumes(self, provisioner_module): + """hostPath mode always mounts stable public/custom/legacy views.""" provisioner_module.SKILLS_PVC_NAME = "" provisioner_module.USERDATA_PVC_NAME = "" volumes = provisioner_module._build_volumes("thread-1") - assert len(volumes) == 3 + assert len(volumes) == 4 def test_hostpath_skills_public_volume(self, provisioner_module): """First skills volume mounts public/ subdirectory.""" @@ -24,7 +24,7 @@ class TestBuildVolumes: pub = volumes[0] assert pub.name == "skills-public" assert pub.host_path is not None - assert pub.host_path.path.endswith("/public") + assert pub.host_path.path.endswith("/skills_view/public") assert pub.host_path.type == "Directory" assert pub.persistent_volume_claim is None @@ -35,11 +35,11 @@ class TestBuildVolumes: custom = volumes[1] assert custom.name == "skills-custom" assert custom.host_path is not None - assert "users/user-7/skills/custom" in custom.host_path.path - assert custom.host_path.type == "DirectoryOrCreate" + assert "users/user-7/skills_view/custom" in custom.host_path.path + assert custom.host_path.type == "Directory" def test_hostpath_skills_legacy_volume(self, provisioner_module): - """Legacy global-custom directory is mounted only when requested.""" + """Legacy projection is per-user and stable regardless of visibility.""" provisioner_module.SKILLS_PVC_NAME = "" volumes = provisioner_module._build_volumes( "thread-1", @@ -48,16 +48,17 @@ class TestBuildVolumes: legacy = volumes[2] assert legacy.name == "skills-legacy" assert legacy.host_path is not None - assert legacy.host_path.path.endswith("/custom") + assert "users/default/skills_view/legacy" in legacy.host_path.path assert legacy.host_path.type == "Directory" - def test_hostpath_without_legacy_has_no_legacy_volume(self, provisioner_module): - """Fresh installs should not require a missing global legacy directory.""" + def test_hostpath_without_legacy_flag_still_has_empty_capable_mount(self, provisioner_module): + """Visibility changes update contents without recreating the Pod.""" provisioner_module.SKILLS_PVC_NAME = "" volumes = provisioner_module._build_volumes("thread-1") assert [volume.name for volume in volumes] == [ "skills-public", "skills-custom", + "skills-legacy", "user-data", ] @@ -129,8 +130,8 @@ class TestBuildVolumes: volumes = provisioner_module._build_volumes("thread-1", extra_mounts=extra_mounts) - # skills-public + skills-custom + user-data (3 base) + 1 extra volume. - assert len(volumes) == 4 + # Three skill projections + user-data (4 base) + 1 extra volume. + assert len(volumes) == 5 extra_vol = volumes[-1] assert extra_vol.name == "extra-0" assert extra_vol.host_path.path == "/state/users/alice/integrations/lark-cli/config" @@ -151,8 +152,8 @@ class TestBuildVolumes: volumes = provisioner_module._build_volumes("thread-1", extra_mounts=extra_mounts) - # skills-public + skills-custom + user-data (3 base) + 1 extra volume. - assert len(volumes) == 4 + # Three skill projections + user-data (4 base) + 1 extra volume. + assert len(volumes) == 5 extra_vol = volumes[-1] assert extra_vol.name == "extra-0" assert extra_vol.persistent_volume_claim is not None @@ -184,12 +185,12 @@ class TestBuildVolumeMounts: # ── hostPath mode ────────────────────────────────────────────────── - def test_hostpath_without_legacy_returns_three_mounts(self, provisioner_module): - """hostPath mode omits legacy mount unless the backend requests it.""" + def test_hostpath_uses_three_projection_mounts(self, provisioner_module): + """hostPath mode always mounts stable public/custom/legacy views.""" provisioner_module.SKILLS_PVC_NAME = "" provisioner_module.USERDATA_PVC_NAME = "" mounts = provisioner_module._build_volume_mounts("thread-1") - assert len(mounts) == 3 + assert len(mounts) == 4 def test_hostpath_skills_public_mount(self, provisioner_module): """Public skills mount at /mnt/skills/public, read-only.""" @@ -218,13 +219,14 @@ class TestBuildVolumeMounts: assert mounts[2].mount_path == "/mnt/skills/legacy" assert mounts[2].read_only is True - def test_hostpath_without_legacy_has_no_legacy_mount(self, provisioner_module): - """Users with custom skills should not see hidden legacy content in the sandbox.""" + def test_hostpath_without_legacy_flag_still_has_legacy_mount(self, provisioner_module): + """Hidden legacy content is represented by an empty mounted view.""" provisioner_module.SKILLS_PVC_NAME = "" mounts = provisioner_module._build_volume_mounts("thread-1") assert [mount.name for mount in mounts] == [ "skills-public", "skills-custom", + "skills-legacy", "user-data", ] @@ -353,19 +355,19 @@ class TestBuildVolumeMounts: class TestBuildPodVolumes: """Integration: _build_pod should wire volumes and mounts correctly.""" - def test_pod_hostpath_without_legacy_has_three_volumes(self, provisioner_module): - """hostPath Pod spec should omit legacy volume by default.""" + def test_pod_hostpath_has_four_volumes(self, provisioner_module): + """hostPath Pod spec includes all three stable projection categories.""" provisioner_module.SKILLS_PVC_NAME = "" provisioner_module.USERDATA_PVC_NAME = "" pod = provisioner_module._build_pod("sandbox-1", "thread-1") - assert len(pod.spec.volumes) == 3 + assert len(pod.spec.volumes) == 4 - def test_pod_hostpath_without_legacy_has_three_mounts(self, provisioner_module): - """hostPath container should omit legacy mount by default.""" + def test_pod_hostpath_has_four_mounts(self, provisioner_module): + """hostPath container includes all three stable projection categories.""" provisioner_module.SKILLS_PVC_NAME = "" provisioner_module.USERDATA_PVC_NAME = "" pod = provisioner_module._build_pod("sandbox-1", "thread-1") - assert len(pod.spec.containers[0].volume_mounts) == 3 + assert len(pod.spec.containers[0].volume_mounts) == 4 def test_pod_hostpath_with_legacy_has_four_volumes(self, provisioner_module): """Legacy volume should be present when the backend requests it.""" @@ -438,8 +440,8 @@ class TestBuildPodVolumes: extra_mounts=extra_mounts, ) - # skills-public + skills-custom + user-data (3 base) + 2 extra mounts. - assert len(pod.spec.volumes) == 5 + # Three skill projections + user-data (4 base) + 2 extra mounts. + assert len(pod.spec.volumes) == 6 mount_paths = {mount.mount_path for mount in pod.spec.containers[0].volume_mounts} assert "/mnt/integrations/lark-cli/config" in mount_paths assert "/mnt/integrations/lark-cli/data" in mount_paths diff --git a/backend/tests/test_provisioner_request_threading.py b/backend/tests/test_provisioner_request_threading.py index b93ed0467..4c5c2c8be 100644 --- a/backend/tests/test_provisioner_request_threading.py +++ b/backend/tests/test_provisioner_request_threading.py @@ -167,7 +167,7 @@ async def test_sandbox_business_routes_run_k8s_client_off_event_loop_thread( [ ( False, - ["skills-public", "skills-custom", "user-data"], + ["skills-public", "skills-custom", "skills-legacy", "user-data"], ), ( True, diff --git a/backend/tests/test_skill_manage_tool.py b/backend/tests/test_skill_manage_tool.py index f94182d6e..eb5462fd8 100644 --- a/backend/tests/test_skill_manage_tool.py +++ b/backend/tests/test_skill_manage_tool.py @@ -210,6 +210,48 @@ def test_skill_manage_rejects_support_path_traversal(monkeypatch, tmp_path): ) +def test_skill_manage_remove_file_updates_sandbox_projection_before_return(monkeypatch, tmp_path): + skills_root = tmp_path / "skills" + config = _make_config(skills_root) + monkeypatch.setattr("deerflow.config.get_app_config", lambda: config) + monkeypatch.setattr("deerflow.skills.security_scanner.get_app_config", lambda: config) + from deerflow.config.paths import Paths + + monkeypatch.setattr("deerflow.config.paths.get_paths", lambda: Paths(base_dir=tmp_path)) + monkeypatch.setattr("deerflow.config.paths._paths", None) + + async def _refresh(user_id: str): + return None + + monkeypatch.setattr(skill_manage_module, "refresh_user_skills_system_prompt_cache_async", _refresh) + monkeypatch.setattr(skill_manage_module, "scan_skill_content", lambda *args, **kwargs: _async_result("allow", "ok")) + + runtime = _make_runtime(user_id="default") + anyio.run(skill_manage_module.skill_manage_tool.coroutine, runtime, "create", "demo-skill", _skill_content("demo-skill")) + anyio.run( + skill_manage_module.skill_manage_tool.coroutine, + runtime, + "write_file", + "demo-skill", + "supporting content", + "references/guide.md", + ) + projected_file = tmp_path / "users" / "default" / "skills_view" / "custom" / "demo-skill" / "references" / "guide.md" + assert projected_file.read_text(encoding="utf-8") == "supporting content" + + result = anyio.run( + skill_manage_module.skill_manage_tool.coroutine, + runtime, + "remove_file", + "demo-skill", + None, + "references/guide.md", + ) + + assert result == "Removed 'references/guide.md' from custom skill 'demo-skill'." + assert not projected_file.exists() + + def test_skill_manage_static_critical_blocks_create_before_llm(monkeypatch, tmp_path): skills_root = tmp_path / "skills" config = _make_config(skills_root) diff --git a/backend/tests/test_skill_projection.py b/backend/tests/test_skill_projection.py new file mode 100644 index 000000000..c7922f6e3 --- /dev/null +++ b/backend/tests/test_skill_projection.py @@ -0,0 +1,612 @@ +"""Enabled-only filesystem projections exposed to sandbox providers.""" + +from __future__ import annotations + +import errno +import shutil +import zipfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from threading import Barrier, Event +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from deerflow.config.extensions_config import ExtensionsConfig, SkillStateConfig +from deerflow.config.paths import Paths +from deerflow.skills.projection import ensure_public_skill_projection, ensure_skill_projections, rebuild_skill_projections, skill_projection_mutation +from deerflow.skills.storage.user_scoped_skill_storage import UserScopedSkillStorage + + +def _skill_content(name: str, marker: str = "v1") -> str: + return f"---\nname: {name}\ndescription: {marker}\n---\n\n# {name}\n\n{marker}\n" + + +def _write_skill(root: Path, name: str, marker: str = "v1") -> Path: + target = root / name / "SKILL.md" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(_skill_content(name, marker), encoding="utf-8") + return target + + +@pytest.fixture +def projection_env(tmp_path: Path): + skills_root = tmp_path / "skills" + (skills_root / "public").mkdir(parents=True) + (skills_root / "custom").mkdir() + paths = Paths(base_dir=tmp_path) + config = SimpleNamespace( + skills=SimpleNamespace( + get_skills_path=lambda: skills_root, + container_path="/mnt/skills", + use="deerflow.skills.storage.local_skill_storage:LocalSkillStorage", + ) + ) + extensions = ExtensionsConfig() + + with ( + patch("deerflow.config.paths.get_paths", return_value=paths), + patch("deerflow.config.extensions_config.ExtensionsConfig.from_file", return_value=extensions), + patch("deerflow.config.extensions_config.get_extensions_config", return_value=extensions), + ): + storage = UserScopedSkillStorage("alice", host_path=str(skills_root), app_config=config) + yield SimpleNamespace( + root=tmp_path, + skills_root=skills_root, + paths=paths, + config=config, + extensions=extensions, + storage=storage, + ) + + +def test_projection_contains_only_enabled_skills(projection_env) -> None: + env = projection_env + enabled = _write_skill(env.skills_root / "public", "enabled-skill") + _write_skill(env.skills_root / "public", "disabled-skill") + env.extensions.skills["disabled-skill"] = SkillStateConfig(enabled=False) + + projected = rebuild_skill_projections(env.storage) + + enabled_view = projected.public / "enabled-skill" / "SKILL.md" + assert enabled_view.read_text(encoding="utf-8") == enabled.read_text(encoding="utf-8") + assert not (projected.public / "disabled-skill").exists() + + +def test_projection_rebuild_removes_newly_disabled_skill(projection_env) -> None: + env = projection_env + _write_skill(env.skills_root / "public", "demo-skill") + projected = rebuild_skill_projections(env.storage) + assert (projected.public / "demo-skill" / "SKILL.md").is_file() + + env.extensions.skills["demo-skill"] = SkillStateConfig(enabled=False) + rebuild_skill_projections(env.storage) + + assert not (projected.public / "demo-skill").exists() + + +def test_nested_skill_frontmatter_is_supporting_data_inside_parent_package(projection_env) -> None: + env = projection_env + parent_root = env.skills_root / "public" / "parent-skill" + _write_skill(env.skills_root / "public", "parent-skill") + nested = _write_skill(parent_root / "fixtures", "nested-skill") + env.extensions.skills["nested-skill"] = SkillStateConfig(enabled=False) + + projected = rebuild_skill_projections(env.storage) + nested_view = projected.public / nested.parent.relative_to(env.skills_root / "public") + + assert (projected.public / "parent-skill" / "SKILL.md").is_file() + assert (nested_view / "SKILL.md").is_file() + + +def test_projection_falls_back_to_copy_when_hardlink_is_unavailable(projection_env, monkeypatch) -> None: + env = projection_env + source = _write_skill(env.skills_root / "public", "demo-skill") + + def _cross_device_link(*_args, **_kwargs): + raise OSError(errno.EXDEV, "cross-device link") + + monkeypatch.setattr("deerflow.skills.projection.os.link", _cross_device_link) + projected = rebuild_skill_projections(env.storage) + target = projected.public / "demo-skill" / "SKILL.md" + + assert target.read_text(encoding="utf-8") == source.read_text(encoding="utf-8") + assert target.stat().st_ino != source.stat().st_ino + + +def test_atomic_custom_skill_rewrite_refreshes_projection(projection_env) -> None: + env = projection_env + env.storage.write_custom_skill("demo-skill", "SKILL.md", _skill_content("demo-skill", "before")) + projected = rebuild_skill_projections(env.storage) + target = projected.custom / "demo-skill" / "SKILL.md" + old_inode = target.stat().st_ino + + env.storage.write_custom_skill("demo-skill", "SKILL.md", _skill_content("demo-skill", "after")) + + assert "after" in target.read_text(encoding="utf-8") + assert target.stat().st_ino != old_inode + + +def test_custom_content_write_keeps_unrelated_skill_visible_during_rebuild(projection_env, monkeypatch) -> None: + env = projection_env + env.storage.write_custom_skill("alpha", "SKILL.md", _skill_content("alpha")) + env.storage.write_custom_skill("beta", "SKILL.md", _skill_content("beta", "before")) + projected = rebuild_skill_projections(env.storage) + alpha_view = projected.custom / "alpha" / "SKILL.md" + + from deerflow.skills import projection as projection_module + + real_stage_skill = projection_module._stage_skill + staging_started = Event() + release_staging = Event() + + def _delayed_stage_skill(*args, **kwargs): + if not staging_started.is_set(): + staging_started.set() + assert release_staging.wait(timeout=5) + return real_stage_skill(*args, **kwargs) + + monkeypatch.setattr(projection_module, "_stage_skill", _delayed_stage_skill) + + with ThreadPoolExecutor(max_workers=1) as executor: + write_future = executor.submit(env.storage.write_custom_skill, "beta", "SKILL.md", _skill_content("beta", "after")) + assert staging_started.wait(timeout=5) + unrelated_skill_remained_visible = alpha_view.is_file() + release_staging.set() + write_future.result(timeout=5) + + assert unrelated_skill_remained_visible + assert "after" in (projected.custom / "beta" / "SKILL.md").read_text(encoding="utf-8") + + +def test_per_user_toggle_removes_custom_skill_before_returning(projection_env) -> None: + env = projection_env + env.storage.write_custom_skill("demo-skill", "SKILL.md", _skill_content("demo-skill")) + projected = rebuild_skill_projections(env.storage) + assert (projected.custom / "demo-skill" / "SKILL.md").is_file() + + env.storage.set_skill_enabled_state("demo-skill", False) + + assert not (projected.custom / "demo-skill").exists() + + +def test_managed_integration_projection_is_filtered_per_user(projection_env) -> None: + env = projection_env + integration_root = env.paths.integration_skills_dir() + _write_skill(integration_root / "lark-cli", "lark-doc") + + alice_projection = rebuild_skill_projections(env.storage) + alice_skill = alice_projection.integrations / "lark-cli" / "lark-doc" / "SKILL.md" + assert alice_skill.is_file() + + env.storage.set_skill_enabled_state("lark-doc", False) + assert not alice_skill.exists() + + bob_storage = UserScopedSkillStorage("bob", host_path=str(env.skills_root), app_config=env.config) + bob_projection = rebuild_skill_projections(bob_storage) + assert (bob_projection.integrations / "lark-cli" / "lark-doc" / "SKILL.md").is_file() + + +def test_disabling_custom_skill_hides_only_target_while_rebuilding(projection_env, monkeypatch) -> None: + env = projection_env + env.storage.write_custom_skill("alpha", "SKILL.md", _skill_content("alpha")) + env.storage.write_custom_skill("beta", "SKILL.md", _skill_content("beta")) + projected = rebuild_skill_projections(env.storage) + + from deerflow.skills import projection as projection_module + + real_stage_skill = projection_module._stage_skill + staging_started = Event() + release_staging = Event() + + def _delayed_stage_skill(*args, **kwargs): + if not staging_started.is_set(): + staging_started.set() + assert release_staging.wait(timeout=5) + return real_stage_skill(*args, **kwargs) + + monkeypatch.setattr(projection_module, "_stage_skill", _delayed_stage_skill) + + with ThreadPoolExecutor(max_workers=1) as executor: + disable_future = executor.submit(env.storage.set_skill_enabled_state, "beta", False) + assert staging_started.wait(timeout=5) + alpha_remained_visible = (projected.custom / "alpha" / "SKILL.md").is_file() + beta_was_hidden = not (projected.custom / "beta").exists() + release_staging.set() + disable_future.result(timeout=5) + + assert alpha_remained_visible + assert beta_was_hidden + + +def test_disabling_namespaced_skill_hides_its_real_projection_path(projection_env) -> None: + env = projection_env + _write_skill(env.skills_root / "custom" / "team", "helper") + projected = rebuild_skill_projections(env.storage) + target = projected.legacy / "team" / "helper" + assert (target / "SKILL.md").is_file() + + with skill_projection_mutation(env.storage, "user", remove_names=("helper",)): + env.storage._write_skill_states({"helper": {"enabled": False}}) + assert not target.exists() + + assert not target.exists() + + +def test_disabling_duplicate_namespaced_public_skills_hides_every_path(projection_env) -> None: + env = projection_env + _write_skill(env.skills_root / "public" / "team-a", "helper") + _write_skill(env.skills_root / "public" / "team-b", "helper") + projected = rebuild_skill_projections(env.storage) + targets = [projected.public / team / "helper" for team in ("team-a", "team-b")] + assert all((target / "SKILL.md").is_file() for target in targets) + + with skill_projection_mutation(env.storage, "public", remove_names=("helper",)): + env.extensions.skills["helper"] = SkillStateConfig(enabled=False) + assert all(not target.exists() for target in targets) + + assert all(not target.exists() for target in targets) + + +def test_mutation_failure_clears_projection_scope(projection_env) -> None: + env = projection_env + env.storage.write_custom_skill("alpha", "SKILL.md", _skill_content("alpha")) + projected = rebuild_skill_projections(env.storage) + manifest = projected.custom.parent / ".projection-manifest.json" + assert (projected.custom / "alpha" / "SKILL.md").is_file() + + with pytest.raises(OSError, match="mutation failed"): + with skill_projection_mutation(env.storage, "user"): + raise OSError("mutation failed") + + assert list(projected.custom.iterdir()) == [] + assert list(projected.legacy.iterdir()) == [] + assert list(projected.integrations.iterdir()) == [] + assert not manifest.exists() + + +def test_targeted_removal_failure_clears_drifted_projection_scope(projection_env) -> None: + env = projection_env + _write_skill(env.skills_root / "public" / "team", "helper") + projected = rebuild_skill_projections(env.storage) + manifest = projected.public.parent / ".projection-manifest.json" + namespace = projected.public / "team" + shutil.rmtree(namespace) + namespace.write_text("drifted file", encoding="utf-8") + + with pytest.raises(NotADirectoryError): + with skill_projection_mutation(env.storage, "public", remove_names=("helper",)): + env.extensions.skills["helper"] = SkillStateConfig(enabled=False) + + assert list(projected.public.iterdir()) == [] + assert not manifest.exists() + + +def test_user_custom_skill_replaces_legacy_projection(projection_env) -> None: + env = projection_env + _write_skill(env.skills_root / "custom", "legacy-skill") + projected = rebuild_skill_projections(env.storage) + assert (projected.legacy / "legacy-skill" / "SKILL.md").is_file() + + env.storage.write_custom_skill("custom-skill", "SKILL.md", _skill_content("custom-skill")) + + assert (projected.custom / "custom-skill" / "SKILL.md").is_file() + assert list(projected.legacy.iterdir()) == [] + + +@pytest.mark.parametrize("category", ["custom", "legacy"]) +def test_ensure_user_projection_uses_fresh_global_state_across_workers(projection_env, category: str) -> None: + env = projection_env + skill_name = f"{category}-skill" + source_root = env.storage.get_user_custom_root() if category == "custom" else env.skills_root / "custom" + _write_skill(source_root, skill_name) + projected = rebuild_skill_projections(env.storage) + target = getattr(projected, category) / skill_name / "SKILL.md" + manifest = projected.custom.parent / ".projection-manifest.json" + assert target.is_file() + assert manifest.is_file() + + fresh_extensions = ExtensionsConfig() + fresh_extensions.skills[skill_name] = SkillStateConfig(enabled=False) + with patch("deerflow.config.extensions_config.ExtensionsConfig.from_file", return_value=fresh_extensions): + ensure_skill_projections(env.storage) + assert not target.exists() + assert manifest.is_file() + rebuilt_manifest = manifest.read_text(encoding="utf-8") + + # The rebuilt manifest must not mark the stale visible view as fresh. + ensure_skill_projections(env.storage) + assert not target.exists() + assert manifest.read_text(encoding="utf-8") == rebuilt_manifest + + +def test_ensure_repairs_direct_atomic_source_replacement(projection_env) -> None: + env = projection_env + source = _write_skill(env.skills_root / "public", "demo-skill", "before") + projected = rebuild_skill_projections(env.storage) + target = projected.public / "demo-skill" / "SKILL.md" + old_projected_inode = target.stat().st_ino + + replacement = source.with_suffix(".replacement") + replacement.write_text(_skill_content("demo-skill", "after"), encoding="utf-8") + replacement.replace(source) + ensure_skill_projections(env.storage) + + assert "after" in target.read_text(encoding="utf-8") + assert target.stat().st_ino != old_projected_inode + + +def test_ensure_without_source_changes_keeps_projected_inode(projection_env) -> None: + env = projection_env + _write_skill(env.skills_root / "public", "demo-skill") + projected = rebuild_skill_projections(env.storage) + target = projected.public / "demo-skill" / "SKILL.md" + projected_inode = target.stat().st_ino + + ensure_skill_projections(env.storage) + + assert target.stat().st_ino == projected_inode + + +def test_ensure_steady_state_public_signature_checks_do_not_serialize(projection_env, monkeypatch) -> None: + env = projection_env + _write_skill(env.skills_root / "public", "demo-skill") + rebuild_skill_projections(env.storage) + + from deerflow.skills import projection as projection_module + + real_source_signature = projection_module._source_signature + public_signatures = Barrier(2, timeout=2) + + def _synchronized_source_signature(storage, scope): + if scope == "public": + public_signatures.wait() + return real_source_signature(storage, scope) + + monkeypatch.setattr(projection_module, "_source_signature", _synchronized_source_signature) + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(ensure_skill_projections, env.storage) for _ in range(2)] + for future in futures: + future.result(timeout=5) + + +def test_unlocked_public_snapshot_detects_manifest_change_during_signature_scan(projection_env, monkeypatch) -> None: + env = projection_env + _write_skill(env.skills_root / "public", "demo-skill") + projected = rebuild_skill_projections(env.storage) + manifest = projected.public.parent / ".projection-manifest.json" + + from deerflow.skills import projection as projection_module + + real_source_signature = projection_module._source_signature + signature_read = Event() + release_signature = Event() + + def _delayed_source_signature(storage, scope): + signature = real_source_signature(storage, scope) + if scope == "public": + signature_read.set() + assert release_signature.wait(timeout=5) + return signature + + monkeypatch.setattr(projection_module, "_source_signature", _delayed_source_signature) + + with ThreadPoolExecutor(max_workers=1) as executor: + ensure_future = executor.submit(ensure_skill_projections, env.storage) + assert signature_read.wait(timeout=5) + manifest.unlink() + release_signature.set() + ensure_future.result(timeout=5) + + assert manifest.is_file() + + +def test_concurrent_stale_public_ensure_rebuilds_once_after_lock_recheck(projection_env) -> None: + env = projection_env + source = _write_skill(env.skills_root / "public", "demo-skill", "before") + rebuild_skill_projections(env.storage) + + replacement = source.with_suffix(".replacement") + replacement.write_text(_skill_content("demo-skill", "after"), encoding="utf-8") + replacement.replace(source) + + from deerflow.skills import projection as projection_module + + with patch.object(projection_module, "_rebuild_public_locked", wraps=projection_module._rebuild_public_locked) as rebuild: + with ThreadPoolExecutor(max_workers=8) as executor: + list(executor.map(ensure_skill_projections, [env.storage] * 8)) + + assert rebuild.call_count == 1 + + +def test_rebuild_keeps_category_root_inode_stable(projection_env) -> None: + env = projection_env + _write_skill(env.skills_root / "public", "demo-skill") + projected = rebuild_skill_projections(env.storage) + root_inode = projected.public.stat().st_ino + + env.extensions.skills["demo-skill"] = SkillStateConfig(enabled=False) + rebuild_skill_projections(env.storage) + + assert projected.public.stat().st_ino == root_inode + + +def test_rebuild_failure_clears_old_projection(projection_env, monkeypatch) -> None: + env = projection_env + source = _write_skill(env.skills_root / "public", "demo-skill", "before") + projected = rebuild_skill_projections(env.storage) + assert (projected.public / "demo-skill" / "SKILL.md").is_file() + + replacement = source.with_suffix(".replacement") + replacement.write_text(_skill_content("demo-skill", "after"), encoding="utf-8") + replacement.replace(source) + monkeypatch.setattr("deerflow.skills.projection._stage_skill", lambda *_args: (_ for _ in ()).throw(OSError("disk full"))) + + with pytest.raises(OSError, match="disk full"): + ensure_skill_projections(env.storage) + + assert list(projected.public.iterdir()) == [] + + +def test_signature_failure_clears_old_projection_and_manifest(projection_env, monkeypatch) -> None: + env = projection_env + _write_skill(env.skills_root / "public", "demo-skill") + projected = rebuild_skill_projections(env.storage) + manifest = projected.public.parent / ".projection-manifest.json" + assert (projected.public / "demo-skill" / "SKILL.md").is_file() + assert manifest.is_file() + + monkeypatch.setattr( + "deerflow.skills.projection._source_signature", + lambda *_args, **_kwargs: (_ for _ in ()).throw(PermissionError("source metadata unavailable")), + ) + + with pytest.raises(PermissionError, match="source metadata unavailable"): + ensure_skill_projections(env.storage) + + assert list(projected.public.iterdir()) == [] + assert not manifest.exists() + + +def test_boot_ensures_public_projection_without_scanning_known_users(projection_env, monkeypatch) -> None: + env = projection_env + _write_skill(env.skills_root / "public", "public-skill") + _write_skill(env.paths.user_custom_skills_dir("bob"), "custom-skill") + + def _unexpected_user_storage(*_args, **_kwargs): + raise AssertionError("gateway boot must not enumerate user skill storage") + + monkeypatch.setattr("deerflow.skills.storage.get_or_new_user_skill_storage", _unexpected_user_storage) + + assert ensure_public_skill_projection(app_config=env.config) is True + + assert (env.paths.public_skills_view_dir / "public-skill" / "SKILL.md").is_file() + assert not env.paths.user_custom_skills_view_dir("bob").exists() + + bob_storage = UserScopedSkillStorage("bob", host_path=str(env.skills_root), app_config=env.config) + ensure_skill_projections(bob_storage) + assert (env.paths.user_custom_skills_view_dir("bob") / "custom-skill" / "SKILL.md").is_file() + + +def test_boot_public_projection_failure_is_fail_closed_without_aborting(projection_env, monkeypatch) -> None: + env = projection_env + _write_skill(env.skills_root / "public", "public-skill", "before") + rebuild_skill_projections(env.storage, include_user=False) + monkeypatch.setattr( + "deerflow.skills.projection._source_signature", + lambda *_args, **_kwargs: (_ for _ in ()).throw(PermissionError("source unavailable")), + ) + + assert ensure_public_skill_projection(app_config=env.config) is False + + assert list(env.paths.public_skills_view_dir.iterdir()) == [] + + +def test_boot_public_storage_factory_failure_is_fail_closed_without_aborting(projection_env, monkeypatch) -> None: + env = projection_env + _write_skill(env.skills_root / "public", "public-skill") + rebuild_skill_projections(env.storage, include_user=False) + monkeypatch.setattr( + "deerflow.skills.storage.get_or_new_skill_storage", + lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("storage init failed")), + ) + + assert ensure_public_skill_projection(app_config=env.config) is False + assert list(env.paths.public_skills_view_dir.iterdir()) == [] + + +def test_boot_factory_failure_cleanup_waits_for_concurrent_public_rebuild(projection_env, monkeypatch) -> None: + env = projection_env + _write_skill(env.skills_root / "public", "public-skill") + from deerflow.skills import projection as projection_module + + before_manifest = Event() + release_rebuild = Event() + cleanup_finished = Event() + real_write_manifest = projection_module._write_manifest + + def _delayed_write_manifest(scope_root, signature): + before_manifest.set() + assert release_rebuild.wait(timeout=5) + real_write_manifest(scope_root, signature) + + monkeypatch.setattr(projection_module, "_write_manifest", _delayed_write_manifest) + monkeypatch.setattr( + "deerflow.skills.storage.get_or_new_skill_storage", + lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("storage init failed")), + ) + + def _rebuild() -> None: + rebuild_skill_projections(env.storage, include_user=False) + + def _fail_boot_ensure() -> None: + assert ensure_public_skill_projection(app_config=env.config) is False + cleanup_finished.set() + + with ThreadPoolExecutor(max_workers=2) as executor: + rebuild_future = executor.submit(_rebuild) + assert before_manifest.wait(timeout=5) + cleanup_future = executor.submit(_fail_boot_ensure) + cleanup_waited_for_rebuild = not cleanup_finished.wait(timeout=0.2) + release_rebuild.set() + rebuild_future.result(timeout=5) + cleanup_future.result(timeout=5) + + assert cleanup_waited_for_rebuild + assert list(env.paths.public_skills_view_dir.iterdir()) == [] + assert not (env.paths.public_skills_view_dir.parent / ".projection-manifest.json").exists() + + +@pytest.mark.anyio +async def test_archive_install_is_projected_before_return(projection_env, monkeypatch, tmp_path) -> None: + from deerflow.skills.security_scanner import ScanResult + + env = projection_env + archive = tmp_path / "archive-skill.skill" + with zipfile.ZipFile(archive, "w") as bundle: + bundle.writestr("archive-skill/SKILL.md", _skill_content("archive-skill")) + bundle.writestr("archive-skill/references/guide.md", "# Guide\n") + + async def _allow_scan(*_args, **_kwargs): + return ScanResult(decision="allow", reason="test") + + monkeypatch.setattr("deerflow.skills.installer.scan_skill_content", _allow_scan) + + result = await env.storage.ainstall_skill_from_archive(archive) + + projected = env.paths.user_custom_skills_view_dir("alice") / "archive-skill" + assert result["success"] is True + assert (projected / "SKILL.md").is_file() + assert (projected / "references" / "guide.md").read_text(encoding="utf-8") == "# Guide\n" + + +def test_concurrent_custom_skill_writes_do_not_lose_projected_entries(projection_env) -> None: + env = projection_env + names = [f"skill-{index}" for index in range(8)] + + def _write(name: str) -> None: + env.storage.write_custom_skill(name, "SKILL.md", _skill_content(name)) + + with ThreadPoolExecutor(max_workers=4) as executor: + list(executor.map(_write, names)) + + projected_names = {path.name for path in env.paths.user_custom_skills_view_dir("alice").iterdir()} + assert projected_names == set(names) + + +def test_concurrent_custom_skill_toggles_do_not_lose_state(projection_env) -> None: + env = projection_env + names = ("skill-a", "skill-b") + for name in names: + env.storage.write_custom_skill(name, "SKILL.md", _skill_content(name)) + + with ThreadPoolExecutor(max_workers=2) as executor: + list(executor.map(lambda name: env.storage.set_skill_enabled_state(name, False), names)) + + assert env.storage._read_skill_states() == { + "skill-a": {"enabled": False}, + "skill-b": {"enabled": False}, + } + assert list(env.paths.user_custom_skills_view_dir("alice").iterdir()) == [] diff --git a/backend/tests/test_skills_custom_router.py b/backend/tests/test_skills_custom_router.py index 27d72c1b3..37bf38886 100644 --- a/backend/tests/test_skills_custom_router.py +++ b/backend/tests/test_skills_custom_router.py @@ -628,7 +628,6 @@ def test_update_skill_refreshes_prompt_cache_before_return(monkeypatch, tmp_path mock_storage = _FakeUserScopedStorage() monkeypatch.setattr(skills_router, "_get_user_skill_storage", lambda cfg: mock_storage) monkeypatch.setattr(skills_router, "get_effective_user_id", lambda: "default") - monkeypatch.setattr("app.gateway.routers.skills.get_extensions_config", lambda: SimpleNamespace(mcp_servers={}, skills={})) monkeypatch.setattr("app.gateway.routers.skills.reload_extensions_config", lambda: None) monkeypatch.setattr(skills_router.ExtensionsConfig, "resolve_config_path", staticmethod(lambda config_path=None: config_path)) monkeypatch.setattr("app.gateway.routers.skills.refresh_user_skills_system_prompt_cache_async", _refresh) @@ -705,7 +704,6 @@ def test_public_skill_toggle_clears_all_users_cache(monkeypatch, tmp_path): return config_path monkeypatch.setattr(skills_router.ExtensionsConfig, "resolve_config_path", staticmethod(_resolve)) - monkeypatch.setattr("app.gateway.routers.skills.get_extensions_config", lambda: __import__("deerflow.config.extensions_config", fromlist=["ExtensionsConfig"]).ExtensionsConfig.from_file(config_path)) monkeypatch.setattr("app.gateway.routers.skills.reload_extensions_config", lambda: None) monkeypatch.setattr("app.gateway.routers.skills.clear_skills_system_prompt_cache", _clear) monkeypatch.setattr("app.gateway.routers.skills.refresh_user_skills_system_prompt_cache_async", _refresh) @@ -725,6 +723,107 @@ def test_public_skill_toggle_clears_all_users_cache(monkeypatch, tmp_path): assert persisted["skills"]["public-skill"]["enabled"] is False +def test_public_skill_toggle_creates_missing_extensions_config(monkeypatch, tmp_path): + backend_dir = tmp_path / "backend" + backend_dir.mkdir() + monkeypatch.chdir(backend_dir) + config_path = tmp_path / "extensions_config.json" + + def _load_skills(*, enabled_only: bool): + enabled = True + if config_path.exists(): + enabled = json.loads(config_path.read_text(encoding="utf-8"))["skills"]["public-skill"]["enabled"] + skill = _make_skill("public-skill", enabled=enabled) + return [] if enabled_only and not enabled else [skill] + + storage = SimpleNamespace(load_skills=_load_skills) + monkeypatch.setattr(skills_router, "_get_user_skill_storage", lambda _config: storage) + + def _resolve_config_path(explicit_path=None): + if explicit_path is None: + return None + path = Path(explicit_path) + if not path.exists(): + raise FileNotFoundError(path) + return path + + monkeypatch.setattr(skills_router.ExtensionsConfig, "resolve_config_path", staticmethod(_resolve_config_path)) + monkeypatch.setattr(skills_router, "reload_extensions_config", lambda: None) + monkeypatch.setattr(skills_router, "clear_skills_system_prompt_cache", lambda: None) + + with TestClient(_make_test_app(SimpleNamespace())) as client: + response = client.put("/api/skills/public-skill", json={"enabled": False}) + + assert response.status_code == 200, response.text + assert response.json()["enabled"] is False + assert json.loads(config_path.read_text(encoding="utf-8")) == { + "mcpServers": {}, + "skills": {"public-skill": {"enabled": False}}, + "middlewares": [], + } + + +def test_public_skill_toggle_rebuilds_projection_before_response(monkeypatch, tmp_path): + from deerflow.config.extensions_config import ExtensionsConfig, SkillStateConfig, reset_extensions_config, set_extensions_config + from deerflow.config.paths import Paths + from deerflow.skills.projection import rebuild_skill_projections + + skills_root = tmp_path / "skills" + skill_file = skills_root / "public" / "public-skill" / "SKILL.md" + skill_file.parent.mkdir(parents=True) + skill_file.write_text(_skill_content("public-skill"), encoding="utf-8") + (skills_root / "custom").mkdir() + config_path = tmp_path / "extensions_config.json" + config_path.write_text( + json.dumps( + { + "mcpServers": {}, + "skills": { + "public-skill": {"enabled": True}, + "untouched-skill": {"enabled": False}, + }, + } + ), + encoding="utf-8", + ) + monkeypatch.setenv("DEER_FLOW_EXTENSIONS_CONFIG_PATH", str(config_path)) + + paths = Paths(base_dir=tmp_path) + config = SimpleNamespace( + skills=SimpleNamespace( + get_skills_path=lambda: skills_root, + container_path="/mnt/skills", + use="deerflow.skills.storage.local_skill_storage:LocalSkillStorage", + ), + skill_evolution=SimpleNamespace(enabled=True, moderation_model_name=None), + ) + monkeypatch.setattr("deerflow.config.paths.get_paths", lambda: paths) + monkeypatch.setattr("deerflow.config.paths._paths", None) + monkeypatch.setattr(skills_router, "get_effective_user_id", lambda: "default") + monkeypatch.setattr(skills_router, "clear_skills_system_prompt_cache", lambda: None) + # Simulate another worker having updated the file after this worker cached + # an older snapshot. The public toggle must reload from disk under the + # cross-process projection lock before its read-modify-write. + set_extensions_config(ExtensionsConfig(skills={"public-skill": SkillStateConfig(enabled=True)})) + + storage = UserScopedSkillStorage("default", host_path=str(skills_root), app_config=config) + monkeypatch.setattr(skills_router, "_get_user_skill_storage", lambda _config: storage) + projected = rebuild_skill_projections(storage) + assert (projected.public / "public-skill" / "SKILL.md").is_file() + + try: + with TestClient(_make_test_app(config)) as client: + response = client.put("/api/skills/public-skill", json={"enabled": False}) + + assert response.status_code == 200, response.text + assert response.json()["enabled"] is False + assert not (projected.public / "public-skill").exists() + persisted = json.loads(config_path.read_text(encoding="utf-8")) + assert persisted["skills"]["untouched-skill"] == {"enabled": False} + finally: + reset_extensions_config() + + class TestMultiUserSkillIsolation: """End-to-end integration tests verifying per-user skill isolation through the HTTP router → _get_user_skill_storage → filesystem chain. @@ -955,7 +1054,6 @@ class TestMultiUserSkillIsolation: ) monkeypatch.setattr(skills_router, "_get_user_skill_storage", lambda cfg: alice_storage) monkeypatch.setattr(skills_router, "get_effective_user_id", lambda: "alice") - monkeypatch.setattr("app.gateway.routers.skills.get_extensions_config", lambda: SimpleNamespace(mcp_servers={}, skills={})) monkeypatch.setattr("app.gateway.routers.skills.reload_extensions_config", lambda: None) app = _make_test_app(config) @@ -1009,7 +1107,6 @@ class TestMultiUserSkillIsolation: monkeypatch.setattr(skills_router, "_get_user_skill_storage", lambda cfg: alice_storage) monkeypatch.setattr(skills_router, "get_effective_user_id", lambda: "alice") - monkeypatch.setattr("app.gateway.routers.skills.get_extensions_config", lambda: SimpleNamespace(mcp_servers={}, skills={})) monkeypatch.setattr("app.gateway.routers.skills.reload_extensions_config", lambda: None) monkeypatch.setattr("app.gateway.routers.skills.refresh_user_skills_system_prompt_cache_async", _noop_async) diff --git a/backend/tests/test_skills_router_authz.py b/backend/tests/test_skills_router_authz.py index 955705e81..b46102b82 100644 --- a/backend/tests/test_skills_router_authz.py +++ b/backend/tests/test_skills_router_authz.py @@ -125,6 +125,17 @@ def test_enable_toggle_allowed_for_admin(monkeypatch, tmp_path): from deerflow.skills.types import Skill config_path = tmp_path / "extensions_config.json" + config_path.write_text( + json.dumps( + { + "mcpServers": {}, + "skills": {}, + "middlewares": ["pkg:Middleware"], + "mcpInterceptors": ["pkg.interceptor:build"], + } + ), + encoding="utf-8", + ) def _load_skills(*, enabled_only: bool): return [ @@ -141,21 +152,12 @@ def test_enable_toggle_allowed_for_admin(monkeypatch, tmp_path): ] app = _make_app(system_role="admin") + # Not a real LocalSkillStorage instance, so _write_extensions_skill_state's + # projection-mutation branch is skipped (nullcontext) and it reads the + # config_path fresh via ExtensionsConfig.from_file. monkeypatch.setattr(skills_router, "_get_user_skill_storage", lambda cfg: SimpleNamespace(load_skills=_load_skills)) - from deerflow.config.extensions_config import ExtensionsConfig - - monkeypatch.setattr( - skills_router, - "get_extensions_config", - lambda: ExtensionsConfig( - mcp_servers={}, - skills={}, - middlewares=["pkg:Middleware"], - mcpInterceptors=["pkg.interceptor:build"], - ), - ) monkeypatch.setattr(skills_router, "reload_extensions_config", lambda: None) - monkeypatch.setattr(skills_router.ExtensionsConfig, "resolve_config_path", staticmethod(lambda: config_path)) + monkeypatch.setattr(skills_router.ExtensionsConfig, "resolve_config_path", staticmethod(lambda _config_path=None: config_path)) async def _refresh(_user_id: str): return None diff --git a/backend/tests/test_three_way_skills_mount_e2e.py b/backend/tests/test_three_way_skills_mount_e2e.py index bb166740a..c6ef25d66 100644 --- a/backend/tests/test_three_way_skills_mount_e2e.py +++ b/backend/tests/test_three_way_skills_mount_e2e.py @@ -1,8 +1,8 @@ -"""End-to-end tests for three-way skills mount across sandbox providers. +"""End-to-end tests for enabled-only skill mounts across sandbox providers. -Verifies that (a) public, (b) per-user custom, and (c) legacy global-custom -skills all resolve to correct container paths that the sandbox providers -actually mount — covering ``LocalSandboxProvider`` and +Verifies that public, per-user custom, legacy global-custom, and managed +integration skills all resolve to correct container paths that the sandbox +providers actually mount — covering ``LocalSandboxProvider`` and ``AioSandboxProvider`` (DooD / local-backend path). Includes a full-pipeline test that exercises the actual path the model @@ -16,9 +16,12 @@ from unittest.mock import patch import pytest +from deerflow.config.extensions_config import ExtensionsConfig, SkillStateConfig from deerflow.config.paths import Paths from deerflow.sandbox.local.local_sandbox import PathMapping from deerflow.sandbox.local.local_sandbox_provider import LocalSandboxProvider +from deerflow.skills.projection import rebuild_skill_projections +from deerflow.skills.storage.user_scoped_skill_storage import UserScopedSkillStorage from deerflow.skills.types import SKILL_MD_FILE, Skill, SkillCategory _AIO_MODULE = "deerflow.community.aio_sandbox.aio_sandbox_provider" @@ -93,6 +96,32 @@ class TestThreeWayMountEndToEnd: idx = _local_mounts(provider, "thread-1", user_id="user-1") assert "/mnt/skills/public" in idx assert idx["/mnt/skills/public"].read_only is True + assert Path(idx["/mnt/skills/public"].local_path) == paths.public_skills_view_dir + + def test_local_acquire_recovers_public_mount_after_initial_projection_failure(self, skills_fs): + cfg = _build_config(skills_fs["root"]) + paths = Paths(base_dir=skills_fs["users_dir"].parent) + projection = SimpleNamespace( + public=paths.public_skills_view_dir, + custom=paths.user_custom_skills_view_dir("user-1"), + legacy=paths.user_legacy_skills_view_dir("user-1"), + integrations=paths.user_integration_skills_view_dir("user-1"), + ) + for root in (projection.public, projection.custom, projection.legacy, projection.integrations): + root.mkdir(parents=True, exist_ok=True) + + with ( + patch("deerflow.config.get_app_config", return_value=cfg), + patch("deerflow.config.paths.get_paths", return_value=paths), + patch.object(LocalSandboxProvider, "_ensure_skills_projection", side_effect=[OSError("transient"), projection]), + ): + provider = LocalSandboxProvider() + sandbox_id = provider.acquire("thread-1", user_id="user-1") + sandbox = provider.get(sandbox_id) + + mappings = {mapping.container_path: mapping for mapping in sandbox.path_mappings} + assert Path(mappings["/mnt/skills/public"].local_path) == projection.public + assert mappings["/mnt/skills/public"].read_only is True def test_local_per_user_custom_skill_mounted(self, skills_fs): cfg = _build_config(skills_fs["root"]) @@ -101,7 +130,17 @@ class TestThreeWayMountEndToEnd: provider = LocalSandboxProvider() idx = _local_mounts(provider, "thread-1", user_id="user-1") assert "/mnt/skills/custom" in idx - assert str(skills_fs["user_custom"]) in idx["/mnt/skills/custom"].local_path + assert Path(idx["/mnt/skills/custom"].local_path) == paths.user_custom_skills_view_dir("user-1") + + def test_local_managed_integrations_use_per_user_projection(self, skills_fs): + cfg = _build_config(skills_fs["root"]) + paths = Paths(base_dir=skills_fs["users_dir"].parent) + with patch("deerflow.config.get_app_config", return_value=cfg), patch("deerflow.config.paths.get_paths", return_value=paths): + provider = LocalSandboxProvider() + idx = _local_mounts(provider, "thread-1", user_id="user-1") + assert "/mnt/skills/integrations" in idx + assert Path(idx["/mnt/skills/integrations"].local_path) == paths.user_integration_skills_view_dir("user-1") + assert idx["/mnt/skills/integrations"].read_only is True def test_local_legacy_mounted_for_user_without_custom(self, skills_fs): cfg = _build_config(skills_fs["root"]) @@ -110,7 +149,7 @@ class TestThreeWayMountEndToEnd: provider = LocalSandboxProvider() idx = _local_mounts(provider, "thread-1", user_id="noob") assert "/mnt/skills/legacy" in idx - assert str(skills_fs["legacy_global"]) in idx["/mnt/skills/legacy"].local_path + assert Path(idx["/mnt/skills/legacy"].local_path) == paths.user_legacy_skills_view_dir("noob") def test_local_legacy_not_mounted_when_user_has_custom(self, skills_fs): cfg = _build_config(skills_fs["root"]) @@ -118,7 +157,8 @@ class TestThreeWayMountEndToEnd: with patch("deerflow.config.get_app_config", return_value=cfg), patch("deerflow.config.paths.get_paths", return_value=paths): provider = LocalSandboxProvider() idx = _local_mounts(provider, "thread-1", user_id="user-1") - assert "/mnt/skills/legacy" not in idx + assert "/mnt/skills/legacy" in idx + assert list(Path(idx["/mnt/skills/legacy"].local_path).iterdir()) == [] def test_local_legacy_still_mounted_when_user_has_only_non_skill_subdir(self, skills_fs): (skills_fs["users_dir"] / "ghost" / "skills" / "custom" / "dangling-dir").mkdir(parents=True, exist_ok=True) @@ -204,6 +244,40 @@ class TestThreeWayMountEndToEnd: assert cp == "/mnt/skills/legacy/leg-skill/SKILL.md" assert "leg-skill" in sandbox_noob.read_file(cp) + def test_local_bash_observes_toggle_without_sandbox_recreation(self, tmp_path): + skills_root = tmp_path / "skills" + _write_skill(skills_root / "public", "secret-skill", "SECRET_PROCEDURE") + paths = Paths(base_dir=tmp_path) + cfg = _build_config(skills_root) + extensions = ExtensionsConfig(skills={"secret-skill": SkillStateConfig(enabled=False)}) + + with ( + patch("deerflow.config.get_app_config", return_value=cfg), + patch("deerflow.config.paths.get_paths", return_value=paths), + patch("deerflow.config.extensions_config.ExtensionsConfig.from_file", return_value=extensions), + patch("deerflow.config.extensions_config.get_extensions_config", return_value=extensions), + ): + storage = UserScopedSkillStorage("user-1", host_path=str(skills_root), app_config=cfg) + rebuild_skill_projections(storage) + provider = LocalSandboxProvider() + sandbox_id = provider.acquire("thread-1", user_id="user-1") + sandbox = provider.get(sandbox_id) + assert sandbox is not None + + disabled = sandbox.execute_command("cat /mnt/skills/public/secret-skill/SKILL.md") + assert "SECRET_PROCEDURE" not in disabled + + extensions.skills["secret-skill"] = SkillStateConfig(enabled=True) + rebuild_skill_projections(storage) + enabled = sandbox.execute_command("cat /mnt/skills/public/secret-skill/SKILL.md") + assert "SECRET_PROCEDURE" in enabled + assert provider.acquire("thread-1", user_id="user-1") == sandbox_id + + extensions.skills["secret-skill"] = SkillStateConfig(enabled=False) + rebuild_skill_projections(storage) + disabled_again = sandbox.execute_command("cat /mnt/skills/public/secret-skill/SKILL.md") + assert "SECRET_PROCEDURE" not in disabled_again + # ── AioSandboxProvider ────────────────────────────────────────────── def test_aio_public_skill_mount(self, skills_fs, aio_mod): @@ -212,6 +286,8 @@ class TestThreeWayMountEndToEnd: mounts = aio_mod.AioSandboxProvider._get_skills_mounts(user_id="user-1") idx = {m[1]: m for m in mounts} assert "/mnt/skills/public" in idx + host, _, _ = idx["/mnt/skills/public"] + assert "skills_view/public" in host.replace("\\", "/") def test_aio_per_user_custom_skill_mount(self, skills_fs, aio_mod, monkeypatch): cfg = _build_config(skills_fs["root"]) @@ -222,7 +298,7 @@ class TestThreeWayMountEndToEnd: idx = {m[1]: m for m in mounts} assert "/mnt/skills/custom" in idx host, _, _ = idx["/mnt/skills/custom"] - assert "users/user-1/skills/custom" in host.replace("\\", "/") + assert "users/user-1/skills_view/custom" in host.replace("\\", "/") def test_aio_legacy_mounted_for_user_without_custom(self, skills_fs, aio_mod, monkeypatch): cfg = _build_config(skills_fs["root"]) @@ -240,7 +316,7 @@ class TestThreeWayMountEndToEnd: with patch(_AIO_GET_CONFIG, return_value=cfg), patch("deerflow.config.paths.get_paths", return_value=paths): mounts = aio_mod.AioSandboxProvider._get_skills_mounts(user_id="user-1") idx = {m[1]: m for m in mounts} - assert "/mnt/skills/legacy" not in idx + assert "/mnt/skills/legacy" in idx def test_aio_legacy_still_mounted_when_user_has_only_non_skill_subdir(self, skills_fs, aio_mod, monkeypatch): (skills_fs["users_dir"] / "ghost" / "skills" / "custom" / "dangling-dir").mkdir(parents=True, exist_ok=True) @@ -286,7 +362,11 @@ class TestThreeWayMountEndToEnd: assert "/mnt/skills/custom" in mount_entries assert "dst=/mnt/skills/custom" in mount_entries["/mnt/skills/custom"] - assert "users/noob/skills/custom" in mount_entries["/mnt/skills/custom"] + assert "users/noob/skills_view/custom" in mount_entries["/mnt/skills/custom"] + + assert "/mnt/skills/integrations" in mount_entries + assert "dst=/mnt/skills/integrations" in mount_entries["/mnt/skills/integrations"] + assert "users/noob/skills_view/integrations" in mount_entries["/mnt/skills/integrations"] # noob has no per-user custom → legacy is mounted assert "/mnt/skills/legacy" in mount_entries diff --git a/backend/tests/test_user_scoped_skill_storage.py b/backend/tests/test_user_scoped_skill_storage.py index d205a7d06..77a6bfaf6 100644 --- a/backend/tests/test_user_scoped_skill_storage.py +++ b/backend/tests/test_user_scoped_skill_storage.py @@ -539,11 +539,9 @@ class TestSkillLoadingRespectsGlobalDisable: skills={"shared-skill": SimpleNamespace(enabled=False)}, is_skill_enabled=lambda name, _cat: not (name == "shared-skill"), ) - # The function inside ``load_skills`` does a - # function-local ``from deerflow.config.extensions_config - # import get_extensions_config``, so patch the - # extension_config module symbol. - with patch("deerflow.config.extensions_config.get_extensions_config", return_value=ext_cfg): + # User-scoped loading re-reads disk state so another + # worker's global disable is not hidden by a stale cache. + with patch("deerflow.config.extensions_config.ExtensionsConfig.from_file", return_value=ext_cfg): storage = get_or_new_user_skill_storage("alice", app_config=cfg) loaded = storage.load_skills(enabled_only=False) shared = [s for s in loaded if s.name == "shared-skill" and s.category == SkillCategory.LEGACY] diff --git a/deploy/helm/deer-flow/templates/gateway-deployment.yaml b/deploy/helm/deer-flow/templates/gateway-deployment.yaml index 4c907a7d2..8b893a8fe 100644 --- a/deploy/helm/deer-flow/templates/gateway-deployment.yaml +++ b/deploy/helm/deer-flow/templates/gateway-deployment.yaml @@ -91,8 +91,6 @@ spec: value: http://gateway:8001 - name: DEER_FLOW_HOST_BASE_DIR value: /app/backend/.deer-flow - - name: DEER_FLOW_HOST_SKILLS_PATH - value: /app/skills - name: GATEWAY_HOST value: 0.0.0.0 - name: GATEWAY_PORT diff --git a/docker/docker-compose-dev.yaml b/docker/docker-compose-dev.yaml index a51f432c9..6ff1d9cb3 100644 --- a/docker/docker-compose-dev.yaml +++ b/docker/docker-compose-dev.yaml @@ -57,7 +57,6 @@ services: # On Docker Desktop/OrbStack, use your actual host paths like /Users/username/... # Set these in your shell before running docker-compose: # export DEER_FLOW_ROOT=/absolute/path/to/deer-flow - - SKILLS_HOST_PATH=${DEER_FLOW_ROOT}/skills - THREADS_HOST_PATH=${DEER_FLOW_ROOT}/backend/.deer-flow/threads # Per-user data base directory for user-scoped skill mounts - DEER_FLOW_HOST_BASE_DIR=${DEER_FLOW_ROOT}/backend/.deer-flow @@ -201,7 +200,6 @@ services: - DEER_FLOW_CHANNELS_GATEWAY_URL=${DEER_FLOW_CHANNELS_GATEWAY_URL:-http://gateway:8001} - DEER_FLOW_INTERNAL_AUTH_TOKEN=${DEER_FLOW_INTERNAL_AUTH_TOKEN:-} - DEER_FLOW_HOST_BASE_DIR=${DEER_FLOW_ROOT}/backend/.deer-flow - - DEER_FLOW_HOST_SKILLS_PATH=${DEER_FLOW_ROOT}/skills - DEER_FLOW_SANDBOX_HOST=host.docker.internal # Pass PROVISIONER_API_KEY into the gateway container so config.yaml can reference it # as sandbox.provisioner_api_key: $PROVISIONER_API_KEY diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index edc7e353c..461d8ec3a 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -127,7 +127,6 @@ services: - DEER_FLOW_INTERNAL_AUTH_TOKEN=${DEER_FLOW_INTERNAL_AUTH_TOKEN} # DooD path/network translation - DEER_FLOW_HOST_BASE_DIR=${DEER_FLOW_HOME} - - DEER_FLOW_HOST_SKILLS_PATH=${DEER_FLOW_REPO_ROOT}/skills - DEER_FLOW_SANDBOX_HOST=host.docker.internal # Proxy values (HTTP_PROXY/HTTPS_PROXY/ALL_PROXY) are inherited from ../.env via env_file. # Only NO_PROXY is declared here so internal service hostnames are always exempt from the proxy. @@ -167,7 +166,6 @@ services: # plaintext config/data are never mounted into the sandbox. Supersedes # LARK_CLI_INIT_IMAGE when both are set. Empty ⇒ broker off. - LARK_CLI_BROKER_IMAGE=${LARK_CLI_BROKER_IMAGE:-} - - SKILLS_HOST_PATH=${DEER_FLOW_REPO_ROOT}/skills - THREADS_HOST_PATH=${DEER_FLOW_HOME}/threads - DEER_FLOW_HOST_BASE_DIR=${DEER_FLOW_HOME} - KUBECONFIG_PATH=/root/.kube/config diff --git a/docker/provisioner/README.md b/docker/provisioner/README.md index aa8213d6d..fd156f808 100644 --- a/docker/provisioner/README.md +++ b/docker/provisioner/README.md @@ -25,7 +25,7 @@ The **Sandbox Provisioner** is a FastAPI service that dynamically manages sandbo 2. **Pod Creation**: The provisioner creates a dedicated Pod in the `deer-flow` namespace with: - The sandbox container image (all-in-one-sandbox) - HostPath volumes mounted for: - - `/mnt/skills` → Read-only access to public skills + - `/mnt/skills/{public,custom,legacy}` → Read-only enabled-only skill projections - `/mnt/user-data` → Read-write access to thread-specific data - Resource limits (CPU, memory, ephemeral storage) - Readiness/liveness probes @@ -153,8 +153,8 @@ The provisioner is configured via environment variables (set in [docker-compose- | `SANDBOX_IMAGE` | `enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest` | AIO-compatible container image for sandbox Pods | | `LARK_CLI_INIT_IMAGE` | empty (feature off) | Optional lark-cli init image (Pattern A). When set, sandbox Pods requesting the lark-cli runtime get an init container + shared `emptyDir` that provisions `lark-cli`, instead of a hostPath/PVC runtime mount. See [`docker/lark-cli-init`](../lark-cli-init/README.md) | | `LARK_CLI_BROKER_IMAGE` | empty (feature off) | Optional lark-cli broker image (Pattern B, issue #4338). When set, sandbox Pods requesting the broker get a shim init container + a `lark-cli-broker` sidecar that holds the credentials; the plaintext `config`/`data` are mounted into the **sidecar only**, never the sandbox. Supersedes `LARK_CLI_INIT_IMAGE` when both are set. See [`docker/lark-cli-broker`](../lark-cli-broker/README.md) | -| `SKILLS_HOST_PATH` | - | **Host machine** path to skills directory (must be absolute) | | `THREADS_HOST_PATH` | - | **Host machine** path to threads data directory (must be absolute) | +| `DEER_FLOW_HOST_BASE_DIR` | `/.deer-flow` | **Host machine** DeerFlow data root containing global and per-user `skills_view` projections | | `SKILLS_PVC_NAME` | empty (use hostPath) | PVC name for skills volume; when set, sandbox Pods use PVC instead of hostPath | | `SKILLS_PVC_SUBPATH_TEMPLATE` | empty | Optional `subPath` template for `SKILLS_PVC_NAME`. Supports `{user_id}` and `{thread_id}`. When empty, the skills PVC root is mounted unchanged | | `USERDATA_PVC_NAME` | empty (use hostPath) | PVC name for user-data volume; when set, uses PVC with `subPath: deer-flow/users/{user_id}/threads/{thread_id}/user-data` | @@ -208,7 +208,9 @@ PYTHONPATH=. python scripts/migrate_user_isolation.py --user-id This moves legacy `threads/{thread_id}/user-data` data under `users//threads/{thread_id}/user-data`, which matches the new provisioner PVC subPath when the gateway base directory is mounted at `deer-flow/` on the PVC. Use `default` as the target user only when the legacy data should remain in the default no-auth user namespace. Run the migration while no gateway or sandbox Pods are writing to those paths. -When skills are materialized per thread on the same PVC, set `SKILLS_PVC_NAME` to that PVC and configure `SKILLS_PVC_SUBPATH_TEMPLATE=deer-flow/users/{user_id}/threads/{thread_id}/skills`. Leaving the template empty preserves the legacy behavior of mounting the skills PVC root at `/mnt/skills`. +In hostPath mode, the gateway materializes enabled-only views under `skills_view/public` and `users/{user_id}/skills_view/{custom,legacy}` beneath `DEER_FLOW_HOST_BASE_DIR`; the provisioner mounts those stable directories. When skills are materialized per thread on the same PVC, set `SKILLS_PVC_NAME` to that PVC and configure `SKILLS_PVC_SUBPATH_TEMPLATE=deer-flow/users/{user_id}/threads/{thread_id}/skills`. Leaving the template empty preserves the legacy behavior of mounting the skills PVC root at `/mnt/skills`. The gateway does not yet populate that PVC layout dynamically, so PVC-backed skills do not receive hostPath projection updates. + +**hostPath skills volumes require the gateway and the K8s node to see the same `DEER_FLOW_HOST_BASE_DIR`** (single-node deployment, or NFS/shared storage mounted at that path on every node). The gateway writes the projection there before every sandbox acquire, so as long as that path is shared, the directory the provisioner mounts always exists by the time the Pod is scheduled — even a boot-time rebuild failure for one user self-heals on their next acquire, before the provisioner is called. `skills-custom` and `skills-legacy` use hostPath type `Directory` (not `DirectoryOrCreate`): if the shared-storage assumption is violated — the gateway wrote to a different node than the one the Pod lands on — Pod creation now fails visibly instead of silently mounting an empty directory. Use `SKILLS_PVC_NAME` instead of hostPath for genuinely multi-node clusters without shared storage. ### Important: K8S_API_SERVER Override @@ -248,7 +250,7 @@ kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}' - Read Namespaces (to create `deer-flow` if missing) 4. **Host Paths**: - - The `SKILLS_HOST_PATH` and `THREADS_HOST_PATH` must be **absolute paths on the host machine** + - `DEER_FLOW_HOST_BASE_DIR` and `THREADS_HOST_PATH` must be **absolute paths on the host machine** - These paths are mounted into sandbox Pods via K8s HostPath volumes - The paths must exist and be readable by the K8s node @@ -350,7 +352,7 @@ docker exec deer-flow-gateway curl -s $SANDBOX_URL/v1/sandbox **Cause**: HostPath volumes contain invalid paths (e.g., relative paths with `..`). **Solution**: -- Use absolute paths for `SKILLS_HOST_PATH` and `THREADS_HOST_PATH` +- Use absolute paths for `DEER_FLOW_HOST_BASE_DIR` and `THREADS_HOST_PATH` - Verify the paths exist on your host machine: ```bash ls -la /path/to/skills diff --git a/docker/provisioner/app.py b/docker/provisioner/app.py index 8aa3c35f8..0f81aa32e 100644 --- a/docker/provisioner/app.py +++ b/docker/provisioner/app.py @@ -85,7 +85,6 @@ LARK_BROKER_SIDECAR_DATA_PATH = "/var/lark/data" LARK_BROKER_CONFIG_VOLUME_NAME = "lark-cli-config" LARK_BROKER_DATA_VOLUME_NAME = "lark-cli-data" LARK_BROKER_URL = "http://127.0.0.1:8788" -SKILLS_HOST_PATH = os.environ.get("SKILLS_HOST_PATH", "/skills") THREADS_HOST_PATH = os.environ.get("THREADS_HOST_PATH", "/.deer-flow/threads") DEER_FLOW_HOST_BASE_DIR = os.environ.get("DEER_FLOW_HOST_BASE_DIR", "/.deer-flow") SKILLS_PVC_NAME = os.environ.get("SKILLS_PVC_NAME", "") @@ -494,6 +493,7 @@ def _build_volumes( ``LocalSandboxProvider`` and ``AioSandboxProvider``. """ volumes: list[k8s_client.V1Volume] = [] + del include_legacy_skills # retained for request compatibility # ── Skills volumes ──────────────────────────────────────────────── @@ -512,7 +512,7 @@ def _build_volumes( ) else: # hostPath mode: three-way layout - public_path = join_host_path(SKILLS_HOST_PATH, "public") + public_path = join_host_path(DEER_FLOW_HOST_BASE_DIR, "skills_view", "public") volumes.append( k8s_client.V1Volume( name="skills-public", @@ -527,7 +527,7 @@ def _build_volumes( DEER_FLOW_HOST_BASE_DIR, "users", user_id, - "skills", + "skills_view", "custom", ) volumes.append( @@ -535,22 +535,23 @@ def _build_volumes( name="skills-custom", host_path=k8s_client.V1HostPathVolumeSource( path=user_custom_path, - type="DirectoryOrCreate", + type="Directory", ), ) ) - if include_legacy_skills: - legacy_path = join_host_path(SKILLS_HOST_PATH, "custom") - volumes.append( - k8s_client.V1Volume( - name="skills-legacy", - host_path=k8s_client.V1HostPathVolumeSource( - path=legacy_path, - type="Directory", - ), - ) + legacy_path = join_host_path( + DEER_FLOW_HOST_BASE_DIR, "users", user_id, "skills_view", "legacy" + ) + volumes.append( + k8s_client.V1Volume( + name="skills-legacy", + host_path=k8s_client.V1HostPathVolumeSource( + path=legacy_path, + type="Directory", + ), ) + ) # ── User-data volume ────────────────────────────────────────────── @@ -638,6 +639,7 @@ def _build_volume_mounts( scope that mount with ``SKILLS_PVC_SUBPATH_TEMPLATE``. """ mounts: list[k8s_client.V1VolumeMount] = [] + del include_legacy_skills # retained for request compatibility if SKILLS_PVC_NAME: skills_mount = k8s_client.V1VolumeMount( @@ -664,16 +666,13 @@ def _build_volume_mounts( mount_path="/mnt/skills/custom", read_only=True, ), - ] - ) - if include_legacy_skills: - mounts.append( k8s_client.V1VolumeMount( name="skills-legacy", mount_path="/mnt/skills/legacy", read_only=True, - ) - ) + ), + ] + ) userdata_mount = k8s_client.V1VolumeMount( name="user-data", From 72c97014104873be568c4a59dfd2edb31ae20e98 Mon Sep 17 00:00:00 2001 From: Tu Naichao <102199610+TuNaiChao@users.noreply.github.com> Date: Fri, 31 Jul 2026 18:06:45 +0800 Subject: [PATCH 06/31] fix(memory): reject duplicate facts inside the create critical section (#4599) * fix(memory): reject duplicate facts inside the create critical section memory_add's duplicate check ran outside the storage critical section (the tools.py comment called this out): two concurrent tool calls for the same user could both pass the check and both store the same fact. Move authoritative duplicate rejection into MemoryUpdater.create_memory_fact: the candidate's normalized content key is checked against the fresh memory snapshot on every revision-conflict retry, so the loser of a concurrent create reloads, sees the winner's fact, and is rejected with ValueError("Duplicate fact"). The tool-layer pre-check stays as a fast path. The REST router now maps the duplicate ValueError to 409 with a clear detail instead of the misleading "content cannot be empty" 400. Tests: backend-level duplicate rejection, a simulated concurrent-commit-during-conflict-retry race, and the router 409 mapping. * fix(memory): retry legacy-path fact creation on save conflict Wrap the legacy single-file save in the same bounded conflict-retry loop as the apply_changes path: reload the fresh snapshot and re-run _raise_if_duplicate_fact_content on every retry, so a concurrent duplicate commit is rejected with ValueError("Duplicate fact") (tool error / REST 409) instead of the generic OSError save failure (REST 500). No-duplicate conflicts now retry and store instead of failing the create. Addresses review feedback on bytedance/deer-flow#4599. --- backend/app/gateway/routers/memory.py | 2 + .../backends/deermem/deermem/core/updater.py | 55 +++++-- .../harness/deerflow/agents/memory/tools.py | 8 +- backend/tests/test_memory_router.py | 14 ++ backend/tests/test_memory_updater.py | 153 +++++++++++++++++- 5 files changed, 219 insertions(+), 13 deletions(-) diff --git a/backend/app/gateway/routers/memory.py b/backend/app/gateway/routers/memory.py index 291983df2..2046ee546 100644 --- a/backend/app/gateway/routers/memory.py +++ b/backend/app/gateway/routers/memory.py @@ -116,6 +116,8 @@ def _map_memory_fact_value_error(exc: ValueError) -> HTTPException: detail = "Invalid confidence value; must be between 0 and 1." elif exc.args and exc.args[0] == "agent_name": detail = "An agent name is required for fact operations; user-global memory stores summaries only." + elif exc.args and exc.args[0] == "Duplicate fact": + return HTTPException(status_code=409, detail="A fact with the same content already exists.") else: detail = "Memory fact content cannot be empty." return HTTPException(status_code=400, detail=detail) diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py index 7f81c0e3e..11f5f5e45 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py @@ -372,6 +372,20 @@ def _fact_content_key(content: Any) -> str | None: return stripped.casefold() +def _raise_if_duplicate_fact_content(memory_data: dict[str, Any], content_key: str | None) -> None: + """Reject a candidate fact whose normalized content already exists. + + Callers must invoke this against the freshest snapshot available inside + their read-check-write critical section (i.e. on every revision-conflict + retry), so two concurrent creators of the same content cannot both pass + the check and store duplicate facts.""" + if content_key is None: + return + for fact in memory_data.get("facts", []): + if isinstance(fact, dict) and _fact_content_key(fact.get("content")) == content_key: + raise ValueError("Duplicate fact") + + # ── Staleness review helpers ────────────────────────────────────────────── @@ -815,6 +829,13 @@ class MemoryUpdater: "added" status. This restores both the max_facts cap and the post-trim existence check (upstream's ``create_memory_fact_with_created_fact``), which the vendored copy had dropped together to avoid the dangling id. + + Duplicate rejection is enforced here (not only by callers): the + candidate's normalized content key is checked against the fresh + memory snapshot inside the revision-conflict retry loop of both + storage paths (apply_changes and legacy single-file save), so + concurrent creators cannot both store the same content. Raises + ``ValueError("Duplicate fact")`` on a normalized-content match. """ if agent_name is None: raise ValueError("agent_name") @@ -823,6 +844,7 @@ class MemoryUpdater: raise ValueError("content") normalized_category = category.strip() or "context" validated_confidence = _validate_confidence(confidence) + candidate_key = _fact_content_key(normalized_content) now = utc_now_iso_z() fact_id = f"fact_{uuid.uuid4().hex[:8]}" candidate = { @@ -836,6 +858,12 @@ class MemoryUpdater: if getattr(type(self._storage), "apply_changes", None) is not MemoryStorage.apply_changes: for attempt in range(3): memory_data = self.get_memory_data(agent_name, user_id=user_id) if attempt == 0 else self.reload_memory_data(agent_name, user_id=user_id) + # Duplicate rejection lives inside the conflict-retry loop so + # it is re-evaluated against the fresh snapshot after every + # revision conflict: two concurrent creators of the same + # content cannot both store it (the loser reloads, sees the + # winner's fact, and is rejected here). + _raise_if_duplicate_fact_content(memory_data, candidate_key) updated_memory = dict(memory_data) updated_memory["facts"] = _trim_facts_to_max([*memory_data.get("facts", []), copy.deepcopy(candidate)], self._config.max_facts) kept_ids = {str(fact.get("id")) for fact in updated_memory["facts"]} @@ -860,15 +888,24 @@ class MemoryUpdater: raise logger.info("Retrying capped fact creation from a fresh snapshot after a revision conflict") raise AssertionError("bounded create retry did not return or raise") - memory_data = self.get_memory_data(agent_name, user_id=user_id) - updated_memory = dict(memory_data) - updated_memory["facts"] = _trim_facts_to_max([*memory_data.get("facts", []), candidate], self._config.max_facts) - if not self._save_memory_to_file(updated_memory, agent_name, user_id=user_id, expected_revision=int(memory_data.get("revision") or 0)): - raise OSError("Failed to save memory data after creating fact") - # If the cap evicted the just-added (lower-confidence) fact, signal via - # None so callers don't report a dangling id as "added". - stored = any(f.get("id") == fact_id for f in updated_memory["facts"]) - return updated_memory, (fact_id if stored else None) + # Legacy single-file path: same duplicate-rejection contract as the + # apply_changes path above. A revision-conflicted save (False) reloads + # the fresh snapshot and re-runs the duplicate check, so a concurrent + # creator's commit is rejected with ValueError("Duplicate fact") + # instead of surfacing as a generic save failure. + for attempt in range(3): + memory_data = self.get_memory_data(agent_name, user_id=user_id) if attempt == 0 else self.reload_memory_data(agent_name, user_id=user_id) + _raise_if_duplicate_fact_content(memory_data, candidate_key) + updated_memory = dict(memory_data) + updated_memory["facts"] = _trim_facts_to_max([*memory_data.get("facts", []), copy.deepcopy(candidate)], self._config.max_facts) + if self._save_memory_to_file(updated_memory, agent_name, user_id=user_id, expected_revision=int(memory_data.get("revision") or 0)): + # If the cap evicted the just-added (lower-confidence) fact, + # signal via None so callers don't report a dangling id as + # "added". + stored = any(f.get("id") == fact_id for f in updated_memory["facts"]) + return updated_memory, (fact_id if stored else None) + logger.info("Retrying capped fact creation from a fresh snapshot after a revision conflict") + raise OSError("Failed to save memory data after creating fact") def delete_memory_fact(self, fact_id: str, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, Any]: """Delete a fact by its id and persist the updated memory data.""" diff --git a/backend/packages/harness/deerflow/agents/memory/tools.py b/backend/packages/harness/deerflow/agents/memory/tools.py index 5be20b046..c32e2ff12 100644 --- a/backend/packages/harness/deerflow/agents/memory/tools.py +++ b/backend/packages/harness/deerflow/agents/memory/tools.py @@ -118,9 +118,11 @@ def memory_add_tool( content_key = _memory_content_key(normalized_content) manager = get_memory_manager() existing_facts = manager.get_memory(agent_name=agent_name, user_id=user_id).get("facts", []) - # Tool calls normally run one-at-a-time per user turn. If tool-mode - # writing broadens to multiple concurrent calls for the same user, - # move duplicate rejection into the storage/update critical section. + # Fast-path duplicate rejection to spare a write attempt in the common + # case. The authoritative check lives in the backend's create critical + # section (DeerMem re-checks against a fresh snapshot on every + # revision-conflict retry in create_memory_fact), so concurrent tool + # calls for the same user cannot both store the same content. if any(_memory_content_key(str(fact.get("content", ""))) == content_key for fact in existing_facts): return json.dumps({"error": "Duplicate fact"}) diff --git a/backend/tests/test_memory_router.py b/backend/tests/test_memory_router.py index 7d2e3bd25..9d7177283 100644 --- a/backend/tests/test_memory_router.py +++ b/backend/tests/test_memory_router.py @@ -213,6 +213,20 @@ def test_create_memory_fact_route_maps_conflict_to_409() -> None: assert response.json()["detail"] == "Memory changed concurrently; reload and retry." +def test_create_memory_fact_route_maps_duplicate_to_409() -> None: + app = FastAPI() + app.include_router(memory.router) + mock_mgr = MagicMock() + mock_mgr.create_fact.side_effect = ValueError("Duplicate fact") + + with patch("app.gateway.routers.memory.get_memory_manager", return_value=mock_mgr): + with TestClient(app) as client: + response = client.post("/api/memory/facts", json={"content": "fact"}) + + assert response.status_code == 409 + assert response.json()["detail"] == "A fact with the same content already exists." + + def test_get_memory_route_maps_corruption_to_stable_500() -> None: app = FastAPI() app.include_router(memory.router) diff --git a/backend/tests/test_memory_updater.py b/backend/tests/test_memory_updater.py index c17e30f83..b6a41fb51 100644 --- a/backend/tests/test_memory_updater.py +++ b/backend/tests/test_memory_updater.py @@ -5,7 +5,10 @@ from unittest.mock import AsyncMock, MagicMock, patch from deerflow.agents.memory.backends.deermem.deermem.config import DeerMemConfig from deerflow.agents.memory.backends.deermem.deermem.core.prompt import format_conversation_for_update -from deerflow.agents.memory.backends.deermem.deermem.core.storage import MemoryStorage +from deerflow.agents.memory.backends.deermem.deermem.core.storage import ( + MemoryManifestRevisionConflict, + MemoryStorage, +) from deerflow.agents.memory.backends.deermem.deermem.core.updater import ( MemoryUpdater, _build_staleness_section, @@ -444,6 +447,154 @@ def test_create_memory_fact_rejects_invalid_confidence() -> None: raise AssertionError("Expected ValueError for invalid fact confidence") +class _ConcurrentCommitStorage(_MemoryStorage): + """apply_changes stand-in that simulates a concurrent writer committing a + duplicate fact between the caller's snapshot read and its first apply. + + The first apply commits ``concurrent_fact`` (as a winning concurrent + writer would) and then raises a manifest revision conflict, forcing the + caller into its conflict-retry path with a fresh snapshot. Later applies + succeed and persist the upserts so test assertions can observe what the + caller actually stored. + """ + + def __init__(self, concurrent_fact: dict[str, object], memory: dict[str, object] | None = None): + super().__init__(memory) + self._concurrent_fact = concurrent_fact + self._conflict_injected = False + + def apply_changes(self, change_set, **scope): # noqa: ANN001, ANN201, ANN202 - test fake + if not self._conflict_injected: + self._conflict_injected = True + self.memory["facts"] = [*self.memory.get("facts", []), copy.deepcopy(self._concurrent_fact)] + self.memory["revision"] = int(self.memory.get("revision") or 0) + 1 + raise MemoryManifestRevisionConflict("simulated concurrent commit") + self.memory["facts"] = [*self.memory.get("facts", []), *change_set.get("upserts", [])] + self.memory["revision"] = int(self.memory.get("revision") or 0) + 1 + return {"complete": False} + + +def test_create_memory_fact_rejects_duplicate_content() -> None: + """Backend-level dedup: create_memory_fact itself must reject content that + already exists (normalized), not just the tool layer's pre-check.""" + existing = _make_memory( + facts=[ + { + "id": "fact_existing", + "content": "User prefers dark mode", + "category": "preference", + "confidence": 0.9, + "createdAt": "2026-03-18T00:00:00Z", + "source": "manual", + } + ] + ) + + updater = _make_updater(memory=existing) + try: + updater.create_memory_fact(content=" user prefers DARK mode ", agent_name="researcher") + except ValueError as exc: + assert exc.args == ("Duplicate fact",) + else: + raise AssertionError("Expected ValueError for duplicate fact content") + + +def test_create_memory_fact_rejects_duplicate_committed_during_conflict_retry() -> None: + """Race regression: a concurrent writer commits the same content between + this caller's snapshot read and its first apply. After the revision + conflict, the retry must detect the duplicate from the fresh snapshot + instead of storing a second copy.""" + concurrent_fact = { + "id": "fact_concurrent", + "content": "User prefers dark mode", + "category": "preference", + "confidence": 0.9, + "createdAt": "2026-03-18T00:00:00Z", + "source": "manual", + } + storage = _ConcurrentCommitStorage(concurrent_fact=concurrent_fact) + updater = _make_updater(storage=storage) + + try: + updater.create_memory_fact(content="user prefers DARK mode", agent_name="researcher") + except ValueError as exc: + assert exc.args == ("Duplicate fact",) + else: + raise AssertionError("Expected ValueError for duplicate fact content") + + assert [fact["id"] for fact in storage.memory["facts"]] == ["fact_concurrent"] + + +class _LegacyConcurrentCommitStorage(_MemoryStorage): + """Legacy-path stand-in (no apply_changes override) that simulates a + concurrent writer committing a fact between the caller's snapshot read + and its first save: the first save commits ``concurrent_fact`` and + returns False (revision conflict), later saves persist normally.""" + + def __init__(self, concurrent_fact: dict[str, object], memory: dict[str, object] | None = None): + super().__init__(memory) + self._concurrent_fact = concurrent_fact + self._conflict_injected = False + + def save(self, memory_data, agent_name=None, *, user_id=None, expected_revision=None): # noqa: ANN001, ANN201, ANN202 - test fake + self.save_calls.append((agent_name, user_id, expected_revision)) + if not self._conflict_injected: + self._conflict_injected = True + self.memory["facts"] = [*self.memory.get("facts", []), copy.deepcopy(self._concurrent_fact)] + self.memory["revision"] = int(self.memory.get("revision") or 0) + 1 + return False + self.memory = memory_data + return True + + +def test_create_memory_fact_legacy_path_rejects_duplicate_committed_during_save_conflict() -> None: + """Legacy single-file path race regression: a concurrent writer commits + the same content between this caller's snapshot read and its first save. + After the revision conflict, the retry must detect the duplicate from the + fresh snapshot and raise ValueError("Duplicate fact") instead of the + generic OSError save failure.""" + concurrent_fact = { + "id": "fact_concurrent", + "content": "User prefers dark mode", + "category": "preference", + "confidence": 0.9, + "createdAt": "2026-03-18T00:00:00Z", + "source": "manual", + } + storage = _LegacyConcurrentCommitStorage(concurrent_fact=concurrent_fact) + updater = _make_updater(storage=storage) + + try: + updater.create_memory_fact(content="user prefers DARK mode", agent_name="researcher") + except ValueError as exc: + assert exc.args == ("Duplicate fact",) + else: + raise AssertionError("Expected ValueError for duplicate fact content") + + assert [fact["id"] for fact in storage.memory["facts"]] == ["fact_concurrent"] + + +def test_create_memory_fact_legacy_path_retries_save_conflict_and_stores() -> None: + """Legacy single-file path: a save conflict without a duplicate reloads + the fresh snapshot and retries instead of failing the create.""" + concurrent_fact = { + "id": "fact_concurrent", + "content": "An unrelated concurrent fact", + "category": "context", + "confidence": 0.9, + "createdAt": "2026-03-18T00:00:00Z", + "source": "manual", + } + storage = _LegacyConcurrentCommitStorage(concurrent_fact=concurrent_fact) + updater = _make_updater(storage=storage) + + _, fact_id = updater.create_memory_fact(content="Brand new fact", agent_name="researcher") + + assert fact_id is not None + assert len(storage.save_calls) == 2 + assert [fact["id"] for fact in storage.memory["facts"]] == ["fact_concurrent", fact_id] + + def test_delete_memory_fact_raises_for_unknown_id() -> None: updater = _make_updater() try: From 17461ee52e86071b2ccbcfebc2848737390e594a Mon Sep 17 00:00:00 2001 From: Baldwinzc <56501736+Baldwinzc@users.noreply.github.com> Date: Fri, 31 Jul 2026 18:08:34 +0800 Subject: [PATCH 07/31] fix(frontend): parse uploaded filenames containing parentheses (#4608) parseUploadedFiles stopped the filename capture at the first "(" ([^\n(]+), so an entry like "- photo (1).png (12.3 KB)" failed to match and the file silently disappeared from the message's file chips. Browsers produce such names for duplicate downloads, making this a common real-world shape. Anchor the size group on the "( )" pair the backend emits (uploads_middleware formats sizes as "%.1f KB"/"%.1f MB") and let the filename match greedily up to it, so parenthesized filenames parse correctly. --- frontend/src/core/messages/utils.ts | 6 ++++- .../tests/unit/core/messages/utils.test.ts | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/frontend/src/core/messages/utils.ts b/frontend/src/core/messages/utils.ts index 0b0a09517..85ebed41c 100644 --- a/frontend/src/core/messages/utils.ts +++ b/frontend/src/core/messages/utils.ts @@ -845,7 +845,11 @@ export function parseUploadedFiles(content: string): FileInMessage[] { // Parse file list // Format: - filename (size)\n Path: /path/to/file - const fileRegex = /- ([^\n(]+)\s*\(([^)]+)\)\s*\n\s*Path:\s*([^\n]+)/g; + // The filename itself may contain parentheses (e.g. "photo (1).png"), so + // the size group is anchored on the trailing "( )" pair the + // backend emits instead of stopping the filename at the first "(". + const fileRegex = + /- (.+)\s*\(([\d.]+\s*(?:B|KB|MB|GB|TB))\)\s*\n\s*Path:\s*([^\n]+)/gi; const files: FileInMessage[] = []; let fileMatch; diff --git a/frontend/tests/unit/core/messages/utils.test.ts b/frontend/tests/unit/core/messages/utils.test.ts index 5b201794a..027352b42 100644 --- a/frontend/tests/unit/core/messages/utils.test.ts +++ b/frontend/tests/unit/core/messages/utils.test.ts @@ -603,6 +603,31 @@ describe("human message internal context stripping", () => { ]); }); + test("parses uploaded filenames that contain parentheses", () => { + // Browsers name duplicate downloads "photo (1).png"; the backend emits the + // filename verbatim, so the parser must not stop the name at the first "(". + const content = + "\nThe following files were uploaded in this message:\n\n- photo (1).png (12.3 KB)\n Path: /mnt/user-data/uploads/photo (1).png\n- report (final) (2).docx (1.5 MB)\n Path: /mnt/user-data/uploads/report (final) (2).docx\n- normal.pdf (3.0 KB)\n Path: /mnt/user-data/uploads/normal.pdf\n\n\nSummarize"; + + expect(parseUploadedFiles(content)).toEqual([ + { + filename: "photo (1).png", + size: Math.round(12.3 * 1024), + path: "/mnt/user-data/uploads/photo (1).png", + }, + { + filename: "report (final) (2).docx", + size: Math.round(1.5 * 1024 * 1024), + path: "/mnt/user-data/uploads/report (final) (2).docx", + }, + { + filename: "normal.pdf", + size: 3 * 1024, + path: "/mnt/user-data/uploads/normal.pdf", + }, + ]); + }); + test("stripInternalMarkers removes current_uploads blocks on export", () => { const content = "\n- paper.docx (177.6 KB)\n Path: /mnt/user-data/uploads/paper.docx\n\n\nExport me"; From 85c3909c2e7ebfad3b8dfad4ae050db7bbe52b8a Mon Sep 17 00:00:00 2001 From: Amorend Date: Fri, 31 Jul 2026 21:57:22 +0800 Subject: [PATCH 08/31] feat: show real-time context window usage (#3125) (#3183) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: show real-time context window usage in chat UI (#3125) Adds a `context_usage` block to `GET /api/threads/{id}/token-usage` (token count from the live checkpoint, the thread model's `context_window`, and a percentage), introduces a new `ModelConfig.context_window` distinct from the per-call `max_tokens` output cap, and surfaces the percentage in the chat header — inside `TokenUsageIndicator` when token-usage tracking is on, or as a standalone badge when it's off so context capacity stays visible independent of cost tracking. Co-Authored-By: Claude Opus 4.7 * feat: per-category breakdown for context window usage Replace the single-number context_usage payload with a Claude-Code-style breakdown — messages, system prompt, skills, system/MCP tools (active + deferred), custom agents, memory injection, autocompact buffer, and free space — and surface it in the chat UI with a segmented progress bar and per-row table. Co-Authored-By: Claude Opus 4.7 * docs(config): document context_window across model examples Add `context_window` to every example model in config.example.yaml so the new chat-UI "% context used" indicator works out of the box for whichever example a user adopts. Each value is the published default at the time of writing; users are pointed at the official model spec to verify. Bumps config_version to 11 so `make config-upgrade` flags outdated user configs. Co-Authored-By: Claude Opus 4.7 * style: ruff format (line-length 240) No behavior change — collapses two multi-line expressions that fit on one line under the project's 240-char limit. Picked up by `make format`. Co-Authored-By: Claude Opus 4.7 * review: address Copilot bot comments on #3183 - token-usage-indicator: switch `{contextPercentage && (...)}` to an explicit `!= null` check. (The string `"0"` is actually truthy in JS so the original code wasn't buggy, but the explicit check is clearer.) - context-usage-breakdown: drop the `useMemo` around segments/totals — the computation is O(n) over a handful of rows and the previous memo deps omitted `t.contextUsage.categories`, so the bar's tooltips/aria-labels could stay in the old language after a locale switch. - context_usage._split_tools: snapshot MCP names from `get_cached_mcp_tools()` directly instead of re-reading `extensions_config.json` after `get_available_tools()` already loaded it. Removes redundant file I/O on every `/token-usage` poll. (`get_available_tools()` still emits its own INFO logs — silencing those is out of scope here.) Co-Authored-By: Claude Opus 4.7 * style(frontend): prettier --write context-usage-breakdown CI's `pnpm format` (prettier --check) caught two lines previously formatted by hand. Collapses one comma to fit on one line; no behavior change. Co-Authored-By: Claude Opus 4.7 * fix(gateway): correct context-usage breakdown + add exact token counting The context-usage indicator shipped two bugs that silently zeroed whole breakdown rows (both caught by try/except, so the feature looked alive but produced wrong numbers): 1. _count_system_prompt passed app_config= to get_deferred_tools_prompt_section, which only accepts deferred_names -> TypeError swallowed -> system_prompt row always 0, and used_tokens/percentage undercounted by the full prompt. Also subtracted the deferred section twice (the rendered prompt already excluded it). Fix: derive deferred names deterministically and pass them to apply_prompt_template; drop the redundant subtraction. 2. _split_tools imported a non-existent get_deferred_registry -> ImportError swallowed -> all four tool-category rows always 0. Fix: classify via the public is_mcp_tool predicate + tool_search.enabled (mirrors build_deferred_tool_setup); the MCP tag is set by get_available_tools. Added token_usage.counting (approximate|exact). 'exact' routes text/schema/ message counting through the model tokenizer (tiktoken cl100k_base) via the existing memory-module machinery (lazy load + cache + cooldown + CJK-aware fallback), so CJK-heavy threads stop being undercounted by chars//4. Regression + e2e tests added; 6621 backend tests pass. * fix(gateway): harden context usage accounting * fix(gateway): count promoted MCP tools as active in context usage Promoted tools (deferred MCP tools the thread has fetched via tool_search) have their full schema bound on every subsequent turn by DeferredToolFilterMiddleware, so they consume context like any active tool. The breakdown previously left them in the reserved *_deferred rows, under- counting the thread's used_tokens. Classification now treats a tool as deferred only when tool_search is enabled, it is MCP-sourced, AND it has not been promoted. The promoted set is read from the checkpoint's channel_values and scoped by catalog hash — matching the runtime middleware, so a stale promotion from MCP-config drift cannot inflate the active count. The static system prompt still lists all deferred tool names (promotions only affect schema binding, not the prompt), so _count_system_prompt's deferred rendering is intentionally left unchanged. 8 new tests cover classification, catalog-hash scoping (match / drift / compute-failure / malformed), and checkpoint extraction. * fix(context): address review feedback * fix(context): count structured message payloads * fix(context): harden usage accounting * fix(config): bump schema for context usage fields * refactor: narrow context usage to core indicator --------- Co-authored-by: Claude Opus 4.7 Co-authored-by: Willem Jiang --- README.md | 2 + backend/AGENTS.md | 2 +- backend/app/gateway/context_usage.py | 86 ++++++++ backend/app/gateway/routers/thread_runs.py | 11 +- .../harness/deerflow/config/model_config.py | 10 + .../harness/deerflow/models/factory.py | 3 + backend/tests/test_gateway_checkpoint_mode.py | 46 +++- backend/tests/test_model_config.py | 13 ++ backend/tests/test_model_factory.py | 13 ++ backend/tests/test_thread_token_usage.py | 200 +++++++++++++----- config.example.yaml | 26 ++- frontend/AGENTS.md | 2 + .../[agent_name]/chats/[thread_id]/page.tsx | 36 ++-- .../app/workspace/chats/[thread_id]/page.tsx | 36 ++-- .../workspace/context-usage-badge.tsx | 54 +++++ .../workspace/context-usage-format.ts | 19 ++ .../workspace/token-usage-indicator.tsx | 16 ++ frontend/src/core/i18n/locales/en-US.ts | 7 + frontend/src/core/i18n/locales/types.ts | 6 + frontend/src/core/i18n/locales/zh-CN.ts | 6 + frontend/src/core/threads/hooks.ts | 9 +- frontend/src/core/threads/token-usage.ts | 27 +++ frontend/src/core/threads/types.ts | 7 + .../workspace/context-usage-badge.test.ts | 42 ++++ .../workspace/context-usage-format.test.ts | 24 +++ .../unit/core/threads/token-usage.test.ts | 155 +++++++++++++- 26 files changed, 771 insertions(+), 87 deletions(-) create mode 100644 backend/app/gateway/context_usage.py create mode 100644 frontend/src/components/workspace/context-usage-badge.tsx create mode 100644 frontend/src/components/workspace/context-usage-format.ts create mode 100644 frontend/tests/unit/components/workspace/context-usage-badge.test.ts create mode 100644 frontend/tests/unit/components/workspace/context-usage-format.test.ts diff --git a/README.md b/README.md index 7064e36e3..e8891daef 100644 --- a/README.md +++ b/README.md @@ -891,6 +891,8 @@ The Web UI shows the active goal above the composer. The same command is availab Use `/compact` in the Web UI composer to summarize older context for the current thread. DeerFlow keeps the full chat visible, but future model calls use the compacted summary plus recent messages. The command is ignored when there is not enough history to compact, and it is blocked while the thread has a run in flight, including when that run is owned by another Gateway worker. If a multi-worker reservation loses its lease, DeerFlow cancels the checkpoint writer before the replacing run proceeds and returns a retryable conflict after cleanup. Thread-title edits are serialized through the same state-write boundary and show a conflict without closing the rename dialog when a run is active. +The chat header also shows a context-window gauge when the selected model has a positive `context_window` configured. It estimates the latest materialized checkpoint's message tokens and keeps the previous same-thread percentage visible while data refetches, independently of the cumulative token-usage setting. + ### Sub-Agents Sub-agents are an optimization, not the default response to a complex request. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 9c3f185e0..016a04f20 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -495,7 +495,7 @@ Localhost persistence deliberately reads the direct request `Host` and ignores ` | **Artifacts** (`/api/threads/{id}/artifacts`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types | | **Suggestions** (`/api/suggestions`) | `GET /config` - returns global suggestions config boolean; `POST /threads/{id}/suggestions` - generate follow-up questions; rich list/block model content is normalized and inline reasoning (`...`, including unclosed/truncated blocks from reasoning models like MiniMax-M3) is stripped before JSON parsing | | **Input Polish** (`/api/input-polish`) | `POST /` - rewrite a composer draft before it is sent. This is a short authenticated `runs:create` LLM request using `input_polish` config; it does not create a LangGraph run, persist a message, or modify thread state. Shares the non-graph one-shot LLM path (`deerflow.utils.oneshot_llm.run_oneshot_llm`) with the suggestions route so model build + Langfuse metadata + invoke stay in one place; validates the same stripped view of the draft it sends to the model, and preserves literal `` substrings in the rewrite (`strip_think_blocks(truncate_unclosed=False)`) | -| **Thread Runs** (`/api/threads/{id}/runs`) | `POST /` - create background run; `POST /stream` - create + SSE stream; `POST /wait` - create + block. Before the first journaled run, an empty run-event message feed is seeded from an existing checkpoint head so legacy checkpoint-only history receives earlier thread-global sequence numbers and remains visible after the new run; a thread with no checkpoint or an already-populated feed skips this compatibility path. `POST /regenerate/prepare` - prepare clean input + checkpoint metadata for regenerating the latest completed or interrupted assistant answer, carrying the latest non-empty thread title in graph input so resuming an older checkpoint cannot roll back a later manual rename (#4457); `POST /edit-regenerate/prepare` - prepare a checkpoint replay from the latest editable human turn with a replacement user message and edit replay metadata; it carries the current thread title the same way, but only when the replay base already has one — an untitled base belongs to a thread the title middleware has not named yet, so pinning the current title there would keep a name generated from the prompt the edit just replaced; `GET /` - list runs; `GET /{rid}` - run details; `POST /{rid}/cancel` - cancel; `GET /{rid}/join` - join SSE; `GET /{rid}/messages` - paginated per-run messages `{data, has_more}`; `GET /{rid}/events` - full event stream; `GET /{rid}/workspace-changes` - workspace/output file change summary and optional diffs; `GET /../messages` - legacy thread message array; `GET /../messages/page` - backward thread-global `seq` history page with middleware/subagent-AI/successful-regenerate/edit-replay filtering and page-run-scoped feedback enrichment; subagent AI callbacks remain available through run events while parent `task` ToolMessages stay visible for card restoration; `GET /../token-usage` - aggregate tokens | +| **Thread Runs** (`/api/threads/{id}/runs`) | `POST /` - create background run; `POST /stream` - create + SSE stream; `POST /wait` - create + block. Before the first journaled run, an empty run-event message feed is seeded from an existing checkpoint head so legacy checkpoint-only history receives earlier thread-global sequence numbers and remains visible after the new run; a thread with no checkpoint or an already-populated feed skips this compatibility path. `POST /regenerate/prepare` - prepare clean input + checkpoint metadata for regenerating the latest completed or interrupted assistant answer, carrying the latest non-empty thread title in graph input so resuming an older checkpoint cannot roll back a later manual rename (#4457); `POST /edit-regenerate/prepare` - prepare a checkpoint replay from the latest editable human turn with a replacement user message and edit replay metadata; it carries the current thread title the same way, but only when the replay base already has one — an untitled base belongs to a thread the title middleware has not named yet, so pinning the current title there would keep a name generated from the prompt the edit just replaced; `GET /` - list runs; `GET /{rid}` - run details; `POST /{rid}/cancel` - cancel; `GET /{rid}/join` - join SSE; `GET /{rid}/messages` - paginated per-run messages `{data, has_more}`; `GET /{rid}/events` - full event stream; `GET /{rid}/workspace-changes` - workspace/output file change summary and optional diffs; `GET /../messages` - legacy thread message array; `GET /../messages/page` - backward thread-global `seq` history page with middleware/subagent-AI/successful-regenerate/edit-replay filtering and page-run-scoped feedback enrichment; subagent AI callbacks remain available through run events while parent `task` ToolMessages stay visible for card restoration; `GET /../token-usage` - aggregate tokens plus an optional `context_usage` percentage. Context usage approximately counts messages from the latest materialized thread state through `build_thread_checkpoint_state_accessor`, so full and delta checkpoint modes expose the same input. The percentage uses the latest run's model and its configured `context_window`. | | **Feedback** (`/api/threads/{id}/runs/{rid}/feedback`) | `PUT /` - upsert feedback; `DELETE /` - delete user feedback; `POST /` - create feedback; `GET /` - list feedback; `GET /stats` - aggregate stats; `DELETE /{fid}` - delete specific | | **Runs** (`/api/runs`) | `POST /stream` - stateless run + SSE; `POST /wait` - stateless run + block; `GET /{rid}/messages` - paginated messages by run_id `{data, has_more}` (cursor: `after_seq`/`before_seq`); `GET /{rid}/feedback` - list feedback by run_id | | **GitHub Webhooks** (`/api/webhooks/github`) | `POST /` - receive GitHub App / repo webhook deliveries. Verifies `X-Hub-Signature-256` against `GITHUB_WEBHOOK_SECRET`; exempt from auth + CSRF because authenticity is enforced by HMAC. The route is fail-closed: mounted only when `GITHUB_WEBHOOK_SECRET` is set, or when explicit dev opt-in `DEER_FLOW_ALLOW_UNVERIFIED_GITHUB_WEBHOOKS=1` is set. Recognized events include `ping`, `issues`, `issue_comment`, `pull_request`, `pull_request_review`, and `pull_request_review_comment`; unknown events return 200 with `handled=false`. Fan-out runtime failures return 503, keeping the delivery recorded as failed for manual/API/scripted redelivery (GitHub does not automatically retry any failed delivery, 5xx included); permanent/non-retryable conditions such as `channels.github.enabled: false`, unknown events, malformed payloads, or unavailable channel service return 200 with a skipped/handled response. | diff --git a/backend/app/gateway/context_usage.py b/backend/app/gateway/context_usage.py new file mode 100644 index 000000000..04c4634ab --- /dev/null +++ b/backend/app/gateway/context_usage.py @@ -0,0 +1,86 @@ +"""Compute the current message-context usage for a thread.""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +from fastapi import HTTPException, Request + +from app.gateway.deps import get_config +from app.gateway.services import build_thread_checkpoint_state_accessor + +logger = logging.getLogger(__name__) + + +def _count_messages_approximately(messages: list[Any]) -> int: + """Count checkpoint messages with LangChain's network-free heuristic.""" + if not messages: + return 0 + from langchain_core.messages.utils import count_tokens_approximately + + return int(count_tokens_approximately(messages)) + + +async def _load_checkpoint_messages(accessor: Any, config: dict[str, Any]) -> list[Any]: + """Read materialized messages so full and delta checkpoints behave alike.""" + snapshot = await accessor.aget(config) + values = getattr(snapshot, "values", None) or {} + if not isinstance(values, dict): + return [] + return list(values.get("messages") or []) + + +async def _resolve_thread_model_name(run_store: Any, thread_id: str, app_config: Any) -> str | None: + """Prefer the latest run's model, then fall back to the first configured model.""" + try: + runs = await run_store.list_by_thread(thread_id, limit=1) + except Exception: + runs = [] + if runs: + latest = runs[0] + name = latest.get("model_name") if isinstance(latest, dict) else getattr(latest, "model_name", None) + if isinstance(name, str) and name: + return name + models = getattr(app_config, "models", None) or [] + return models[0].name if models else None + + +def build_context_usage_payload(*, token_count: int, max_context_tokens: int | None) -> dict[str, Any]: + """Build the stable API payload for a message count and model capacity.""" + percentage: float | None = None + if max_context_tokens and max_context_tokens > 0: + percentage = round(token_count / max_context_tokens * 100, 1) + return { + "token_count": token_count, + "max_context_tokens": max_context_tokens, + "percentage": percentage, + } + + +async def build_context_usage(request: Request, thread_id: str, run_store: Any) -> dict[str, Any] | None: + """Return approximate usage for the latest materialized thread checkpoint.""" + try: + app_config = get_config() + except HTTPException: + return None + + try: + accessor, checkpoint_config = await build_thread_checkpoint_state_accessor(request, thread_id=thread_id) + messages = await _load_checkpoint_messages(accessor, checkpoint_config) + except Exception: + logger.warning("Failed to load checkpoint for context usage on thread %s", thread_id, exc_info=True) + return None + + try: + token_count = await asyncio.to_thread(_count_messages_approximately, messages) + except Exception: + logger.warning("Failed to count context messages for thread %s", thread_id, exc_info=True) + return None + + model_name = await _resolve_thread_model_name(run_store, thread_id, app_config) + model_config = app_config.get_model_config(model_name) if model_name else None + configured_window = getattr(model_config, "context_window", None) if model_config is not None else None + max_context_tokens = int(configured_window) if configured_window else None + return build_context_usage_payload(token_count=token_count, max_context_tokens=max_context_tokens) diff --git a/backend/app/gateway/routers/thread_runs.py b/backend/app/gateway/routers/thread_runs.py index 475491042..de2dbb0b7 100644 --- a/backend/app/gateway/routers/thread_runs.py +++ b/backend/app/gateway/routers/thread_runs.py @@ -33,6 +33,7 @@ from app.gateway.checkpoint_lineage import ( find_checkpoint_before_message_chronologically, is_duration_only_checkpoint, ) +from app.gateway.context_usage import build_context_usage from app.gateway.deps import get_current_user, get_feedback_repo, get_run_event_store, get_run_manager, get_run_store, get_stream_bridge from app.gateway.pagination import trim_run_message_page from app.gateway.run_models import RunCreateRequest @@ -180,6 +181,12 @@ class ThreadTokenUsageCallerBreakdown(BaseModel): middleware: int = 0 +class ThreadContextUsage(BaseModel): + token_count: int = 0 + max_context_tokens: int | None = None + percentage: float | None = None + + class ThreadTokenUsageResponse(BaseModel): thread_id: str total_tokens: int = 0 @@ -188,6 +195,7 @@ class ThreadTokenUsageResponse(BaseModel): total_runs: int = 0 by_model: dict[str, ThreadTokenUsageModelBreakdown] = Field(default_factory=dict) by_caller: ThreadTokenUsageCallerBreakdown = Field(default_factory=ThreadTokenUsageCallerBreakdown) + context_usage: ThreadContextUsage | None = None # --------------------------------------------------------------------------- @@ -1459,4 +1467,5 @@ async def thread_token_usage( agg = await run_store.aggregate_tokens_by_thread(thread_id, include_active=True) else: agg = await run_store.aggregate_tokens_by_thread(thread_id) - return ThreadTokenUsageResponse(thread_id=thread_id, **agg) + context_usage = await build_context_usage(request, thread_id, run_store) + return ThreadTokenUsageResponse(thread_id=thread_id, context_usage=context_usage, **agg) diff --git a/backend/packages/harness/deerflow/config/model_config.py b/backend/packages/harness/deerflow/config/model_config.py index b747eec99..0d212bdd1 100644 --- a/backend/packages/harness/deerflow/config/model_config.py +++ b/backend/packages/harness/deerflow/config/model_config.py @@ -32,6 +32,16 @@ class ModelConfig(BaseModel): description="Extra settings to be passed to the model when thinking is disabled", ) supports_vision: bool = Field(default_factory=lambda: False, description="Whether the model supports vision/image inputs") + context_window: int | None = Field( + default=None, + gt=0, + description=( + "Positive total context window size in tokens (prompt + completion). Used to compute the real-time " + "context usage percentage displayed in the chat UI. Distinct from `max_tokens`, which is the " + "per-call output cap passed to the provider. Leave unset if unknown; the UI will hide the " + "percentage." + ), + ) stream_chunk_timeout: float | None = Field( default=None, description=( diff --git a/backend/packages/harness/deerflow/models/factory.py b/backend/packages/harness/deerflow/models/factory.py index c6cd3bc3f..21f541a3d 100644 --- a/backend/packages/harness/deerflow/models/factory.py +++ b/backend/packages/harness/deerflow/models/factory.py @@ -219,6 +219,9 @@ def create_chat_model(name: str | None = None, thinking_enabled: bool = False, * "when_thinking_disabled", "thinking", "supports_vision", + # Runtime/UI metadata used to size the context indicator. Provider + # clients do not accept this as a model-constructor argument. + "context_window", # Presentation-only metadata (consumed by the console's cost # display) — must never reach the provider client, which would # forward unknown kwargs into the completion request payload. diff --git a/backend/tests/test_gateway_checkpoint_mode.py b/backend/tests/test_gateway_checkpoint_mode.py index 2d78e862f..8f6ff165b 100644 --- a/backend/tests/test_gateway_checkpoint_mode.py +++ b/backend/tests/test_gateway_checkpoint_mode.py @@ -1,7 +1,8 @@ """Dual-mode (full/delta) parity for the gateway thread-state endpoints. -Drives ``GET /api/threads/{id}``, ``GET /api/threads/{id}/state`` and -``POST /api/threads/{id}/history`` through the real route stack +Drives ``GET /api/threads/{id}``, ``GET /api/threads/{id}/state``, +``POST /api/threads/{id}/history``, and the context-usage checkpoint reader +through the real materialization stack (``build_thread_checkpoint_state_accessor`` -> factory-built graph -> ``CheckpointStateAccessor``) against a real ``InMemorySaver``, once per checkpoint channel mode, and asserts the wire responses are identical apart @@ -24,6 +25,7 @@ from langgraph.checkpoint.memory import InMemorySaver from langgraph.graph import StateGraph from langgraph.store.memory import InMemoryStore +from app.gateway import context_usage from app.gateway import services as gateway_services from app.gateway.routers import threads from deerflow.agents.thread_state import get_thread_state_schema @@ -113,6 +115,46 @@ def test_thread_state_endpoints_are_mode_invariant(_stub_app_config, monkeypatch assert any(full["history_messages"]), "expected history snapshots with messages" +@pytest.mark.parametrize("mode", ["full", "delta"]) +def test_context_usage_reads_materialized_messages_in_both_modes( + mode: str, + _stub_app_config, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Context usage must not read raw delta ``channel_values``.""" + app = make_authed_test_app() + store = InMemoryStore() + checkpointer = InMemorySaver() + app.state.store = store + app.state.checkpointer = checkpointer + app.state.thread_store = MemoryThreadMetaStore(store) + app.state.checkpoint_channel_mode = mode + app.state.run_event_store = SimpleNamespace() + + graph = _build_reply_graph(mode, checkpointer) + monkeypatch.setattr( + gateway_services, + "resolve_agent_factory", + lambda assistant_id=None: lambda config: graph, + ) + + config: dict[str, Any] = {"configurable": {"thread_id": _THREAD_ID}} + inject_checkpoint_mode(config, mode) + for i in range(2): + asyncio.run(graph.ainvoke({"messages": [HumanMessage(content=f"question-{i}", id=f"h{i}")]}, config)) + + request = SimpleNamespace(app=app) + accessor, read_config = asyncio.run(gateway_services.build_thread_checkpoint_state_accessor(request, thread_id=_THREAD_ID)) + messages = asyncio.run(context_usage._load_checkpoint_messages(accessor, read_config)) + + assert [(message.type, message.content, message.id) for message in messages] == [ + ("human", "question-0", "h0"), + ("ai", "answer-1", "a1"), + ("human", "question-1", "h1"), + ("ai", "answer-3", "a3"), + ] + + def test_full_mode_gateway_rejects_delta_thread_with_409(_stub_app_config, monkeypatch: pytest.MonkeyPatch) -> None: """Fail-closed gate at the HTTP boundary, against a real checkpointer. diff --git a/backend/tests/test_model_config.py b/backend/tests/test_model_config.py index 91f8e70aa..2044a66df 100644 --- a/backend/tests/test_model_config.py +++ b/backend/tests/test_model_config.py @@ -1,3 +1,6 @@ +import pytest +from pydantic import ValidationError + from deerflow.config.model_config import ModelConfig @@ -28,3 +31,13 @@ def test_responses_api_fields_round_trip_in_model_dump(): assert dumped["use_responses_api"] is True assert dumped["output_version"] == "responses/v1" + + +def test_context_window_round_trips_when_positive(): + assert _make_model(context_window=128_000).context_window == 128_000 + + +@pytest.mark.parametrize("context_window", [0, -1]) +def test_context_window_rejects_non_positive_capacity(context_window): + with pytest.raises(ValidationError, match="context_window"): + _make_model(context_window=context_window) diff --git a/backend/tests/test_model_factory.py b/backend/tests/test_model_factory.py index fb849bd07..c7daea616 100644 --- a/backend/tests/test_model_factory.py +++ b/backend/tests/test_model_factory.py @@ -141,6 +141,19 @@ def test_pricing_metadata_never_reaches_the_provider_client(monkeypatch): assert "pricing" not in FakeChatModel.captured_kwargs +def test_context_window_never_reaches_the_provider_client(monkeypatch): + """Context sizing metadata belongs to DeerFlow, not the provider SDK.""" + model = _make_model("large-context") + model.context_window = 200_000 + cfg = _make_app_config([model]) + _patch_factory(monkeypatch, cfg) + + FakeChatModel.captured_kwargs = {} + factory_module.create_chat_model(name="large-context") + + assert "context_window" not in FakeChatModel.captured_kwargs + + def test_appends_all_tracing_callbacks(monkeypatch): cfg = _make_app_config([_make_model("alpha")]) _patch_factory(monkeypatch, cfg) diff --git a/backend/tests/test_thread_token_usage.py b/backend/tests/test_thread_token_usage.py index 19f8e0c19..636d453f0 100644 --- a/backend/tests/test_thread_token_usage.py +++ b/backend/tests/test_thread_token_usage.py @@ -1,46 +1,20 @@ -"""Tests for thread-level token usage aggregation API.""" +"""Tests for thread-level token usage and context-window usage.""" from __future__ import annotations +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock +import pytest from _router_auth_helpers import make_authed_test_app from fastapi.testclient import TestClient +from app.gateway import context_usage from app.gateway.routers import thread_runs -def _make_app(run_store: MagicMock): - app = make_authed_test_app() - app.include_router(thread_runs.router) - app.state.run_store = run_store - return app - - -def test_thread_token_usage_returns_stable_shape(): - run_store = MagicMock() - run_store.aggregate_tokens_by_thread = AsyncMock( - return_value={ - "total_tokens": 150, - "total_input_tokens": 90, - "total_output_tokens": 60, - "total_runs": 2, - "by_model": {"unknown": {"tokens": 150, "runs": 2}}, - "by_caller": { - "lead_agent": 120, - "subagent": 25, - "middleware": 5, - }, - }, - ) - app = _make_app(run_store) - - with TestClient(app) as client: - response = client.get("/api/threads/thread-1/token-usage") - - assert response.status_code == 200 - assert response.json() == { - "thread_id": "thread-1", +def _aggregate_result() -> dict: + return { "total_tokens": 150, "total_input_tokens": 90, "total_output_tokens": 60, @@ -52,31 +26,157 @@ def test_thread_token_usage_returns_stable_shape(): "middleware": 5, }, } - run_store.aggregate_tokens_by_thread.assert_awaited_once_with("thread-1") -def test_thread_token_usage_can_include_active_runs(): +def _make_run_store(*, model_name: str | None = None) -> MagicMock: run_store = MagicMock() - run_store.aggregate_tokens_by_thread = AsyncMock( - return_value={ - "total_tokens": 175, - "total_input_tokens": 120, - "total_output_tokens": 55, - "total_runs": 3, - "by_model": {"unknown": {"tokens": 175, "runs": 3}}, - "by_caller": { - "lead_agent": 145, - "subagent": 25, - "middleware": 5, - }, - }, - ) + run_store.aggregate_tokens_by_thread = AsyncMock(return_value=_aggregate_result()) + runs = [{"model_name": model_name}] if model_name else [] + run_store.list_by_thread = AsyncMock(return_value=runs) + return run_store + + +def _make_app(run_store: MagicMock): + app = make_authed_test_app() + app.include_router(thread_runs.router) + app.state.run_store = run_store + return app + + +def test_thread_token_usage_returns_stable_shape(monkeypatch: pytest.MonkeyPatch) -> None: + run_store = _make_run_store() + build_context_usage = AsyncMock(return_value=None) + monkeypatch.setattr(thread_runs, "build_context_usage", build_context_usage) + app = _make_app(run_store) + + with TestClient(app) as client: + response = client.get("/api/threads/thread-1/token-usage") + + assert response.status_code == 200 + assert response.json() == { + "thread_id": "thread-1", + **_aggregate_result(), + "context_usage": None, + } + run_store.aggregate_tokens_by_thread.assert_awaited_once_with("thread-1") + build_context_usage.assert_awaited_once() + + +def test_thread_token_usage_can_include_active_runs(monkeypatch: pytest.MonkeyPatch) -> None: + run_store = _make_run_store() + build_context_usage = AsyncMock(return_value=None) + monkeypatch.setattr(thread_runs, "build_context_usage", build_context_usage) app = _make_app(run_store) with TestClient(app) as client: response = client.get("/api/threads/thread-1/token-usage?include_active=true") assert response.status_code == 200 - assert response.json()["total_tokens"] == 175 - assert response.json()["total_runs"] == 3 run_store.aggregate_tokens_by_thread.assert_awaited_once_with("thread-1", include_active=True) + + +def test_thread_token_usage_serializes_context_percentage(monkeypatch: pytest.MonkeyPatch) -> None: + run_store = _make_run_store() + monkeypatch.setattr( + thread_runs, + "build_context_usage", + AsyncMock( + return_value={ + "token_count": 350, + "max_context_tokens": 1000, + "percentage": 35.0, + } + ), + ) + app = _make_app(run_store) + + with TestClient(app) as client: + response = client.get("/api/threads/thread-1/token-usage") + + assert response.status_code == 200 + assert response.json()["context_usage"] == { + "token_count": 350, + "max_context_tokens": 1000, + "percentage": 35.0, + } + + +def test_build_context_usage_payload_computes_percentage() -> None: + assert context_usage.build_context_usage_payload(token_count=350, max_context_tokens=1000) == { + "token_count": 350, + "max_context_tokens": 1000, + "percentage": 35.0, + } + + +def test_build_context_usage_payload_handles_unknown_capacity() -> None: + assert context_usage.build_context_usage_payload(token_count=350, max_context_tokens=None) == { + "token_count": 350, + "max_context_tokens": None, + "percentage": None, + } + + +@pytest.mark.asyncio +async def test_resolve_thread_model_prefers_latest_run() -> None: + run_store = _make_run_store(model_name="thread-model") + app_config = SimpleNamespace(models=[SimpleNamespace(name="fallback-model")]) + + assert await context_usage._resolve_thread_model_name(run_store, "thread-1", app_config) == "thread-model" + + +@pytest.mark.asyncio +async def test_resolve_thread_model_falls_back_to_first_configured_model() -> None: + run_store = _make_run_store() + app_config = SimpleNamespace(models=[SimpleNamespace(name="fallback-model")]) + + assert await context_usage._resolve_thread_model_name(run_store, "thread-1", app_config) == "fallback-model" + + +@pytest.mark.asyncio +async def test_build_context_usage_counts_materialized_messages(monkeypatch: pytest.MonkeyPatch) -> None: + messages = [SimpleNamespace(content="hello")] + snapshot = SimpleNamespace(values={"messages": messages}) + accessor = SimpleNamespace(aget=AsyncMock(return_value=snapshot)) + monkeypatch.setattr( + context_usage, + "build_thread_checkpoint_state_accessor", + AsyncMock(return_value=(accessor, {"configurable": {"thread_id": "thread-1"}})), + ) + model_config = SimpleNamespace(context_window=1000) + app_config = SimpleNamespace( + models=[SimpleNamespace(name="fallback-model")], + get_model_config=lambda name: model_config if name == "thread-model" else None, + ) + monkeypatch.setattr(context_usage, "get_config", lambda: app_config) + monkeypatch.setattr(context_usage, "_count_messages_approximately", lambda value: 250 if value == messages else 0) + + result = await context_usage.build_context_usage( + request=SimpleNamespace(app=SimpleNamespace()), + thread_id="thread-1", + run_store=_make_run_store(model_name="thread-model"), + ) + + assert result == { + "token_count": 250, + "max_context_tokens": 1000, + "percentage": 25.0, + } + + +@pytest.mark.asyncio +async def test_build_context_usage_returns_none_when_checkpoint_read_fails(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + context_usage, + "build_thread_checkpoint_state_accessor", + AsyncMock(side_effect=RuntimeError("checkpoint unavailable")), + ) + monkeypatch.setattr(context_usage, "get_config", lambda: SimpleNamespace()) + + result = await context_usage.build_context_usage( + request=SimpleNamespace(app=SimpleNamespace()), + thread_id="thread-1", + run_store=_make_run_store(), + ) + + assert result is None diff --git a/config.example.yaml b/config.example.yaml index 2578beba1..d7b7b6ea2 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -94,7 +94,15 @@ max_recursion_limit: 1000 # ============================================================================ # Models Configuration # ============================================================================ -# Configure available LLM models for the agent to use +# Configure available LLM models for the agent to use. +# +# Two token fields look similar but mean different things: +# - `max_tokens` is the per-call OUTPUT cap passed to the provider. +# - `context_window` is a positive integer for the total context capacity +# (prompt + completion) and drives the real-time "% context used" indicator +# in the chat UI. +# Leave `context_window` unset if the provider limit is unknown; the percentage +# will not render. Verify configured values against the provider's model docs. # # Optional per-model pricing (powers the real-cost display on the workspace # console). Add a `pricing` block to any model entry; use ONE currency across @@ -118,6 +126,7 @@ models: # api_key: $VOLCENGINE_API_KEY # timeout: 600.0 # max_retries: 2 + # context_window: 262144 # Total prompt + completion capacity # supports_thinking: true # supports_vision: true # supports_reasoning_effort: true @@ -182,7 +191,8 @@ models: # api_key: $OPENAI_API_KEY # Use environment variable # request_timeout: 600.0 # max_retries: 2 - # max_tokens: 4096 + # max_tokens: 4096 # Per-call output cap + # context_window: 128000 # Total prompt + completion capacity # temperature: 0.7 # supports_vision: true # Enable vision support for view_image tool @@ -196,6 +206,7 @@ models: # max_retries: 2 # use_responses_api: true # output_version: responses/v1 + # context_window: 400000 # supports_vision: true # Example: Ollama (native provider — preserves thinking/reasoning content) @@ -216,6 +227,7 @@ models: # num_predict: 8192 # temperature: 0.7 # reasoning: true # Passes think:true to Ollama native API + # context_window: 32768 # Match the context length served by Ollama # supports_thinking: true # supports_vision: false # @@ -227,6 +239,7 @@ models: # num_predict: 8192 # temperature: 0.7 # reasoning: true + # context_window: 32768 # supports_thinking: true # supports_vision: true # @@ -245,7 +258,8 @@ models: # api_key: $ANTHROPIC_API_KEY # default_request_timeout: 600.0 # max_retries: 2 - # max_tokens: 16000 + # max_tokens: 16000 # Per-call output cap + # context_window: 200000 # Total prompt + completion capacity # supports_vision: true # supports_thinking: true # when_thinking_enabled: @@ -265,6 +279,7 @@ models: # timeout: 600.0 # max_retries: 2 # max_tokens: 8192 + # context_window: 1048576 # supports_vision: true # Example: Gemini model via OpenAI-compatible gateway (with thinking support) @@ -281,6 +296,7 @@ models: # request_timeout: 600.0 # max_retries: 2 # max_tokens: 16384 + # context_window: 1048576 # supports_thinking: true # supports_vision: true # when_thinking_enabled: @@ -739,7 +755,7 @@ tools: # use: deerflow.community.groundroute.tools:web_search_tool # max_results: 5 # Clamped to 1-50 by GroundRoute # # api_key: $GROUNDROUTE_API_KEY # Optional if the env var is set - + # Web search tool (uses fastCRW - Firecrawl-compatible web scraper, single binary, # self-host or cloud. Cloud requires CRW_API_KEY; self-host may need no key.) # - name: web_search @@ -913,7 +929,7 @@ tools: # group: web # use: deerflow.community.groundroute.tools:web_fetch_tool # # api_key: $GROUNDROUTE_API_KEY - + # Web fetch tool (uses fastCRW - Firecrawl-compatible web scraper, single binary, # self-host or cloud. Cloud requires CRW_API_KEY; self-host may need no key.) # NOTE: Only one web_fetch provider can be active at a time. diff --git a/frontend/AGENTS.md b/frontend/AGENTS.md index fb605c627..c9eb6f3e1 100644 --- a/frontend/AGENTS.md +++ b/frontend/AGENTS.md @@ -79,6 +79,8 @@ The frontend is a stateful chat application. Users create **threads** (conversat `["mcpConfig"]` only after success. 6. Components subscribe to thread state and render updates +The chat header's context-window control is intentionally persistent: while `context_usage` is unavailable, `ContextUsageBadge` renders a gauge placeholder rather than unmounting; once data arrives, the same position shows the percentage. `useThreadTokenUsage` retains placeholder data only when the response `thread_id` still matches the active route, so same-thread refetches do not flicker and cross-thread navigation never displays the previous chat's usage. + Run duration is run-scoped UI metadata even though the compatibility field `additional_kwargs.turn_duration` is repeated on historical AI messages. `core/messages/run-duration.ts` folds those copies into one display anchored after the run's last visible message group. `MessageList` owns the temporary client-side duration for a just-completed live turn until authoritative history arrives. The duration is total run wall-clock time, not per-message reasoning time; reasoning disclosure and run activity/duration are rendered separately. The workspace-change card follows the same rule: it is resolved from `(threadId, runId)` alone, so every AI message of a run would render an identical copy. A run ends in more than one terminal assistant bubble whenever the model emits answer text that never gains a tool call, so `core/messages/workspace-change-anchor.ts` picks the run's last assistant bubble and `MessageListItem` renders the badge only for that anchor (#4555). Any future run-scoped display belongs in the same place — do not hang one off every message. The two anchor helpers deliberately differ in which group types they accept as a run's last position, because an anchor is only useful where the display is actually rendered: run duration is emitted by `MessageList` around every group, so it accepts any type, while the workspace-change card comes from `MessageListItem` and so restricts to `assistant`. Keep a new helper's candidate set matched to its own render site rather than unifying them. diff --git a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx index b405fdaab..e3f6c80ba 100644 --- a/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx +++ b/frontend/src/app/workspace/agents/[agent_name]/chats/[thread_id]/page.tsx @@ -10,6 +10,7 @@ import { SidebarTrigger } from "@/components/ui/sidebar"; import { AgentWelcome } from "@/components/workspace/agent-welcome"; import { ArtifactTrigger } from "@/components/workspace/artifacts"; import { ChatBox, useThreadChat } from "@/components/workspace/chats"; +import { ContextUsageBadge } from "@/components/workspace/context-usage-badge"; import { ExportTrigger } from "@/components/workspace/export-trigger"; import { GoalStatus } from "@/components/workspace/goal-status"; import { @@ -47,7 +48,10 @@ import { useThreadStream, useThreadTokenUsage, } from "@/core/threads/hooks"; -import { threadTokenUsageToTokenUsage } from "@/core/threads/token-usage"; +import { + selectContextUsage, + threadTokenUsageToTokenUsage, +} from "@/core/threads/token-usage"; import { textOfMessage } from "@/core/threads/utils"; import { env } from "@/env"; import { cn } from "@/lib/utils"; @@ -73,13 +77,14 @@ export default function AgentChatPage() { const { tokenUsageEnabled } = useModels(); const threadTokenUsage = useThreadTokenUsage( isNewThread || isMock ? undefined : threadId, - { enabled: tokenUsageEnabled && !isMock }, + { enabled: !isMock }, ); const threadMetadata = useThreadMetadata(threadId, { enabled: !isNewThread && !isMock, isMock, }); const backendTokenUsage = threadTokenUsageToTokenUsage(threadTokenUsage.data); + const contextUsage = selectContextUsage(threadTokenUsage.data); const { showNotification } = useNotification(); @@ -276,17 +281,22 @@ export default function AgentChatPage() { {t.agents.newChat} - - setLocalSettings("tokenUsage", preferences) - } - /> + {tokenUsageEnabled ? ( + + setLocalSettings("tokenUsage", preferences) + } + /> + ) : ( + + )} diff --git a/frontend/src/app/workspace/chats/[thread_id]/page.tsx b/frontend/src/app/workspace/chats/[thread_id]/page.tsx index 96b3d9b37..7175db866 100644 --- a/frontend/src/app/workspace/chats/[thread_id]/page.tsx +++ b/frontend/src/app/workspace/chats/[thread_id]/page.tsx @@ -13,6 +13,7 @@ import { useSpecificChatMode, useThreadChat, } from "@/components/workspace/chats"; +import { ContextUsageBadge } from "@/components/workspace/context-usage-badge"; import { ExportTrigger } from "@/components/workspace/export-trigger"; import { GoalStatus } from "@/components/workspace/goal-status"; import { @@ -52,7 +53,10 @@ import { useThreadStream, useThreadTokenUsage, } from "@/core/threads/hooks"; -import { threadTokenUsageToTokenUsage } from "@/core/threads/token-usage"; +import { + selectContextUsage, + threadTokenUsageToTokenUsage, +} from "@/core/threads/token-usage"; import { textOfMessage } from "@/core/threads/utils"; import { env } from "@/env"; import { cn } from "@/lib/utils"; @@ -74,7 +78,7 @@ export default function ChatPage() { const { tokenUsageEnabled } = useModels(); const threadTokenUsage = useThreadTokenUsage( isNewThread || isMock ? undefined : threadId, - { enabled: tokenUsageEnabled && !isMock }, + { enabled: !isMock }, ); const threadMetadata = useThreadMetadata(threadId, { enabled: !isNewThread && !isMock, @@ -82,6 +86,7 @@ export default function ChatPage() { }); const branchThread = useBranchThread(); const backendTokenUsage = threadTokenUsageToTokenUsage(threadTokenUsage.data); + const contextUsage = selectContextUsage(threadTokenUsage.data); const mountedRef = useRef(false); useSpecificChatMode(); @@ -288,17 +293,22 @@ export default function ChatPage() { {!isNewThread && ( )} - - setLocalSettings("tokenUsage", preferences) - } - /> + {tokenUsageEnabled ? ( + + setLocalSettings("tokenUsage", preferences) + } + /> + ) : ( + + )} {browserEnabled && } diff --git a/frontend/src/components/workspace/context-usage-badge.tsx b/frontend/src/components/workspace/context-usage-badge.tsx new file mode 100644 index 000000000..99082f784 --- /dev/null +++ b/frontend/src/components/workspace/context-usage-badge.tsx @@ -0,0 +1,54 @@ +"use client"; + +import { GaugeIcon } from "lucide-react"; + +import { useI18n } from "@/core/i18n/hooks"; +import type { ContextUsage } from "@/core/threads/token-usage"; +import { cn } from "@/lib/utils"; + +import { formatContextUsagePercentage } from "./context-usage-format"; + +interface ContextUsageBadgeProps { + contextUsage: ContextUsage | null; + className?: string; +} + +export function ContextUsageBadge({ + contextUsage, + className, +}: ContextUsageBadgeProps) { + const { t } = useI18n(); + + const formatted = formatContextUsagePercentage(contextUsage?.percentage); + if (formatted == null) { + return ( +
+ +
+ ); + } + + return ( +
+ + {t.contextUsage.label} + {formatted}% +
+ ); +} diff --git a/frontend/src/components/workspace/context-usage-format.ts b/frontend/src/components/workspace/context-usage-format.ts new file mode 100644 index 000000000..6af5753b1 --- /dev/null +++ b/frontend/src/components/workspace/context-usage-format.ts @@ -0,0 +1,19 @@ +/** + * Format a context-usage percentage for display. + * + * Returns `null` when the percentage is unknown so callers can choose to + * hide the indicator entirely rather than render a placeholder. + * + * Whole-number percentages are rendered without a decimal point; otherwise + * a single decimal place is shown to stay readable at high resolution + * (e.g. ``35.4%``). + */ +export function formatContextUsagePercentage( + percentage: number | null | undefined, +): string | null { + if (typeof percentage !== "number" || !Number.isFinite(percentage)) { + return null; + } + const clamped = Math.max(0, percentage); + return Number.isInteger(clamped) ? `${clamped}` : clamped.toFixed(1); +} diff --git a/frontend/src/components/workspace/token-usage-indicator.tsx b/frontend/src/components/workspace/token-usage-indicator.tsx index 0fef98ab0..61b75cdb4 100644 --- a/frontend/src/components/workspace/token-usage-indicator.tsx +++ b/frontend/src/components/workspace/token-usage-indicator.tsx @@ -26,13 +26,17 @@ import { type TokenUsagePreferences, type TokenUsageViewPreset, } from "@/core/messages/usage-model"; +import type { ContextUsage } from "@/core/threads/token-usage"; import { cn } from "@/lib/utils"; +import { formatContextUsagePercentage } from "./context-usage-format"; + interface TokenUsageIndicatorProps { threadId?: string; messages: Message[]; pendingMessages?: Message[]; backendUsage?: TokenUsage | null; + contextUsage?: ContextUsage | null; enabled?: boolean; preferences: TokenUsagePreferences; onPreferencesChange: (preferences: TokenUsagePreferences) => void; @@ -44,6 +48,7 @@ export function TokenUsageIndicator({ messages, pendingMessages, backendUsage, + contextUsage, enabled = false, preferences, onPreferencesChange, @@ -61,6 +66,9 @@ export function TokenUsageIndicator({ [backendUsage, messages, pendingMessages, threadId], ); const preset = getTokenUsageViewPreset(preferences); + const contextPercentage = formatContextUsagePercentage( + contextUsage?.percentage, + ); if (!enabled) { return null; @@ -86,6 +94,14 @@ export function TokenUsageIndicator({ : "-" : t.tokenUsage.presets[presetKeyToTranslationKey(preset)]} + {contextPercentage != null && ( + + {contextPercentage}% + + )} diff --git a/frontend/src/core/i18n/locales/en-US.ts b/frontend/src/core/i18n/locales/en-US.ts index ed1572af1..499e1e12b 100644 --- a/frontend/src/core/i18n/locales/en-US.ts +++ b/frontend/src/core/i18n/locales/en-US.ts @@ -699,6 +699,13 @@ export const enUS: Translations = { removeTodo: (content: string) => `Remove To-do: ${content}`, }, + contextUsage: { + label: "Context", + title: "Context window", + badgeAriaLabel: (percentage: string) => + `Context window ${percentage}% full`, + }, + // Shortcuts shortcuts: { searchActions: "Search actions...", diff --git a/frontend/src/core/i18n/locales/types.ts b/frontend/src/core/i18n/locales/types.ts index e2f035b23..5a7db16d8 100644 --- a/frontend/src/core/i18n/locales/types.ts +++ b/frontend/src/core/i18n/locales/types.ts @@ -582,6 +582,12 @@ export interface Translations { removeTodo: (content: string) => string; }; + contextUsage: { + label: string; + title: string; + badgeAriaLabel: (percentage: string) => string; + }; + // Shortcuts shortcuts: { searchActions: string; diff --git a/frontend/src/core/i18n/locales/zh-CN.ts b/frontend/src/core/i18n/locales/zh-CN.ts index 47fb91b1a..780665817 100644 --- a/frontend/src/core/i18n/locales/zh-CN.ts +++ b/frontend/src/core/i18n/locales/zh-CN.ts @@ -672,6 +672,12 @@ export const zhCN: Translations = { removeTodo: (content: string) => `移除 To-do：${content}`, }, + contextUsage: { + label: "上下文", + title: "上下文窗口", + badgeAriaLabel: (percentage: string) => `上下文窗口已使用 ${percentage}%`, + }, + // Shortcuts shortcuts: { searchActions: "搜索操作...", diff --git a/frontend/src/core/threads/hooks.ts b/frontend/src/core/threads/hooks.ts index 888a4a6c1..fdfbb2539 100644 --- a/frontend/src/core/threads/hooks.ts +++ b/frontend/src/core/threads/hooks.ts @@ -41,7 +41,10 @@ import { filterThreadSearchResults, type ThreadSearchParams, } from "./thread-search-query"; -import { threadTokenUsageQueryKey } from "./token-usage"; +import { + retainThreadTokenUsagePlaceholder, + threadTokenUsageQueryKey, +} from "./token-usage"; import type { AgentThread, AgentThreadState, @@ -2771,6 +2774,10 @@ export function useThreadTokenUsage( enabled: enabled && Boolean(threadId), retry: false, refetchOnWindowFocus: false, + // Keep same-thread data visible during refetches without carrying usage + // from the previous route into a newly selected thread. + placeholderData: (previous) => + retainThreadTokenUsagePlaceholder(previous, threadId), }); } diff --git a/frontend/src/core/threads/token-usage.ts b/frontend/src/core/threads/token-usage.ts index 89455eef9..e6354531d 100644 --- a/frontend/src/core/threads/token-usage.ts +++ b/frontend/src/core/threads/token-usage.ts @@ -6,6 +6,13 @@ export function threadTokenUsageQueryKey(threadId?: string | null) { return ["thread-token-usage", threadId] as const; } +export function retainThreadTokenUsagePlaceholder( + previous: ThreadTokenUsageResponse | null | undefined, + threadId?: string | null, +): ThreadTokenUsageResponse | undefined { + return previous && previous.thread_id === threadId ? previous : undefined; +} + export function threadTokenUsageToTokenUsage( usage: ThreadTokenUsageResponse | null | undefined, ): TokenUsage | null { @@ -18,3 +25,23 @@ export function threadTokenUsageToTokenUsage( totalTokens: usage.total_tokens ?? 0, }; } + +export interface ContextUsage { + tokenCount: number; + maxContextTokens: number | null; + percentage: number | null; +} + +export function selectContextUsage( + usage: ThreadTokenUsageResponse | null | undefined, +): ContextUsage | null { + if (!usage?.context_usage) { + return null; + } + const { token_count, max_context_tokens, percentage } = usage.context_usage; + return { + tokenCount: token_count ?? 0, + maxContextTokens: max_context_tokens ?? null, + percentage: percentage ?? null, + }; +} diff --git a/frontend/src/core/threads/types.ts b/frontend/src/core/threads/types.ts index e4578e147..d7751955f 100644 --- a/frontend/src/core/threads/types.ts +++ b/frontend/src/core/threads/types.ts @@ -62,6 +62,12 @@ export interface RunMessage { created_at: string; } +export interface ThreadContextUsage { + token_count: number; + max_context_tokens: number | null; + percentage: number | null; +} + export interface ThreadTokenUsageResponse { thread_id: string; total_tokens: number; @@ -74,4 +80,5 @@ export interface ThreadTokenUsageResponse { subagent: number; middleware: number; }; + context_usage?: ThreadContextUsage | null; } diff --git a/frontend/tests/unit/components/workspace/context-usage-badge.test.ts b/frontend/tests/unit/components/workspace/context-usage-badge.test.ts new file mode 100644 index 000000000..4cd5fe5a7 --- /dev/null +++ b/frontend/tests/unit/components/workspace/context-usage-badge.test.ts @@ -0,0 +1,42 @@ +import { expect, rs, test } from "@rstest/core"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; + +import { ContextUsageBadge } from "@/components/workspace/context-usage-badge"; + +rs.mock("@/core/i18n/hooks", () => ({ + useI18n: () => ({ + t: { + contextUsage: { + title: "Context window", + label: "Context", + badgeAriaLabel: (percentage: string) => + `Context window ${percentage}% full`, + }, + }, + }), +})); + +test("keeps a gauge placeholder visible while context usage is unavailable", () => { + const html = renderToStaticMarkup( + createElement(ContextUsageBadge, { contextUsage: null }), + ); + + expect(html).toContain('data-context-usage-placeholder="true"'); + expect(html).toContain('aria-label="Context window"'); +}); + +test("renders the current percentage when context usage is available", () => { + const html = renderToStaticMarkup( + createElement(ContextUsageBadge, { + contextUsage: { + tokenCount: 35_000, + maxContextTokens: 100_000, + percentage: 35, + }, + }), + ); + + expect(html).toContain("35%"); + expect(html).toContain('aria-label="Context window 35% full"'); +}); diff --git a/frontend/tests/unit/components/workspace/context-usage-format.test.ts b/frontend/tests/unit/components/workspace/context-usage-format.test.ts new file mode 100644 index 000000000..34654258a --- /dev/null +++ b/frontend/tests/unit/components/workspace/context-usage-format.test.ts @@ -0,0 +1,24 @@ +import { expect, test } from "@rstest/core"; + +import { formatContextUsagePercentage } from "@/components/workspace/context-usage-format"; + +test("returns null for unknown percentages", () => { + expect(formatContextUsagePercentage(null)).toBeNull(); + expect(formatContextUsagePercentage(undefined)).toBeNull(); + expect(formatContextUsagePercentage(Number.NaN)).toBeNull(); +}); + +test("renders whole numbers without a decimal point", () => { + expect(formatContextUsagePercentage(0)).toBe("0"); + expect(formatContextUsagePercentage(35)).toBe("35"); + expect(formatContextUsagePercentage(100)).toBe("100"); +}); + +test("renders fractional percentages with one decimal place", () => { + expect(formatContextUsagePercentage(35.4)).toBe("35.4"); + expect(formatContextUsagePercentage(35.46)).toBe("35.5"); +}); + +test("clamps negative percentages to zero", () => { + expect(formatContextUsagePercentage(-1)).toBe("0"); +}); diff --git a/frontend/tests/unit/core/threads/token-usage.test.ts b/frontend/tests/unit/core/threads/token-usage.test.ts index 6e73cca1b..9b69621c5 100644 --- a/frontend/tests/unit/core/threads/token-usage.test.ts +++ b/frontend/tests/unit/core/threads/token-usage.test.ts @@ -1,6 +1,12 @@ import { expect, test } from "@rstest/core"; +import { QueryClient, QueryObserver } from "@tanstack/react-query"; -import { threadTokenUsageToTokenUsage } from "@/core/threads/token-usage"; +import { + retainThreadTokenUsagePlaceholder, + selectContextUsage, + threadTokenUsageQueryKey, + threadTokenUsageToTokenUsage, +} from "@/core/threads/token-usage"; import type { ThreadTokenUsageResponse } from "@/core/threads/types"; test("maps backend thread token usage to UI token usage", () => { @@ -29,3 +35,150 @@ test("returns null when backend thread token usage is unavailable", () => { expect(threadTokenUsageToTokenUsage(null)).toBeNull(); expect(threadTokenUsageToTokenUsage(undefined)).toBeNull(); }); + +test("retains placeholder usage only for the current thread", () => { + const response: ThreadTokenUsageResponse = { + thread_id: "thread-1", + total_input_tokens: 90, + total_output_tokens: 60, + total_tokens: 150, + total_runs: 2, + by_model: { unknown: { tokens: 150, runs: 2 } }, + by_caller: { + lead_agent: 120, + subagent: 25, + middleware: 5, + }, + }; + + expect(retainThreadTokenUsagePlaceholder(response, "thread-1")).toBe( + response, + ); + expect( + retainThreadTokenUsagePlaceholder(response, "thread-2"), + ).toBeUndefined(); + expect(retainThreadTokenUsagePlaceholder(null, undefined)).toBeUndefined(); +}); + +test("query observer keeps same-thread data but drops it while a new thread is pending", async () => { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + const threadA: ThreadTokenUsageResponse = { + thread_id: "thread-a", + total_input_tokens: 90, + total_output_tokens: 60, + total_tokens: 150, + total_runs: 2, + by_model: { unknown: { tokens: 150, runs: 2 } }, + by_caller: { lead_agent: 120, subagent: 25, middleware: 5 }, + }; + const threadB: ThreadTokenUsageResponse = { + ...threadA, + thread_id: "thread-b", + total_tokens: 200, + }; + let queryResult = Promise.resolve(threadA); + let resolveRefresh: (value: ThreadTokenUsageResponse) => void = () => + undefined; + let resolveThreadB: (value: ThreadTokenUsageResponse) => void = () => + undefined; + const observer = new QueryObserver( + queryClient, + { + queryKey: threadTokenUsageQueryKey("thread-a"), + queryFn: () => queryResult, + placeholderData: (previous) => + retainThreadTokenUsagePlaceholder(previous, "thread-a"), + }, + ); + const unsubscribe = observer.subscribe(() => undefined); + + try { + await observer.refetch(); + expect(observer.getCurrentResult().data).toBe(threadA); + + queryResult = new Promise((resolve) => { + resolveRefresh = resolve; + }); + const sameThreadRefetch = observer.refetch(); + expect(observer.getCurrentResult().data).toBe(threadA); + resolveRefresh(threadA); + await sameThreadRefetch; + + const pendingThreadB = new Promise((resolve) => { + resolveThreadB = resolve; + }); + observer.setOptions({ + queryKey: threadTokenUsageQueryKey("thread-b"), + queryFn: () => pendingThreadB, + retry: false, + placeholderData: (previous) => + retainThreadTokenUsagePlaceholder(previous, "thread-b"), + }); + expect(observer.getCurrentResult().data).toBeUndefined(); + expect(observer.getCurrentResult().isPlaceholderData).toBe(false); + + resolveThreadB(threadB); + await observer.refetch(); + expect(observer.getCurrentResult().data).toBe(threadB); + } finally { + resolveRefresh(threadA); + resolveThreadB(threadB); + unsubscribe(); + queryClient.clear(); + } +}); + +const _baseResponse = { + thread_id: "thread-1", + total_input_tokens: 0, + total_output_tokens: 0, + total_tokens: 0, + total_runs: 0, + by_model: {}, + by_caller: { lead_agent: 0, subagent: 0, middleware: 0 }, +} satisfies ThreadTokenUsageResponse; + +test("selectContextUsage projects the backend block to UI shape", () => { + const response: ThreadTokenUsageResponse = { + ..._baseResponse, + context_usage: { + token_count: 350, + max_context_tokens: 1000, + percentage: 35, + }, + }; + + expect(selectContextUsage(response)).toEqual({ + tokenCount: 350, + maxContextTokens: 1000, + percentage: 35, + }); +}); + +test("selectContextUsage preserves nullable capacity and percentage", () => { + const response: ThreadTokenUsageResponse = { + ..._baseResponse, + context_usage: { + token_count: 200, + max_context_tokens: null, + percentage: null, + }, + }; + + expect(selectContextUsage(response)).toEqual({ + tokenCount: 200, + maxContextTokens: null, + percentage: null, + }); +}); + +test("selectContextUsage returns null when context_usage is missing", () => { + expect(selectContextUsage(_baseResponse)).toBeNull(); + expect( + selectContextUsage({ ..._baseResponse, context_usage: null }), + ).toBeNull(); + expect(selectContextUsage(null)).toBeNull(); + expect(selectContextUsage(undefined)).toBeNull(); +}); From c86071442cbb403f48b3a35dd1ea1d1d40673b9a Mon Sep 17 00:00:00 2001 From: Tu Naichao <102199610+TuNaiChao@users.noreply.github.com> Date: Fri, 31 Jul 2026 22:16:56 +0800 Subject: [PATCH 09/31] fix(memory): truncate mem0 context injection on entry boundaries (#4600) * fix(memory): truncate mem0 context injection on entry boundaries Mem0Manager.get_context built the injection block and then hard-cut it at max_injection_chars, which could sever the last memory mid-line and leave a dangling partial entry in the agent prompt. Accumulate whole entries against the remaining budget instead: a memory that does not fit is skipped (a shorter later one may still fit). When not even the first memory fits, fall back to the previous hard-truncation behavior for that single entry rather than injecting nothing. Tests: entry-boundary truncation, skip-oversized-keep-later, and the oversized-first-entry fallback. * fix(memory): keep entry-boundary guarantee when no mem0 memory fits Follow-up to PR #4600 review: remove the hard-truncation fallback that could inject a partial memory when max_injection_chars is smaller than every recalled entry. Instead return empty context and log a warning that surfaced the undersized budget. --- .../agents/memory/backends/mem0/config.py | 3 +- .../memory/backends/mem0/mem0_manager.py | 30 +++++++++++++++--- backend/tests/test_mem0_memory_backend.py | 31 +++++++++++++++++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/backend/packages/harness/deerflow/agents/memory/backends/mem0/config.py b/backend/packages/harness/deerflow/agents/memory/backends/mem0/config.py index ce48c8cf5..074113504 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/mem0/config.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/mem0/config.py @@ -38,7 +38,8 @@ class Mem0Config: top_k: int = 8 #: Minimum relevance score for search() results (mem0 `threshold`, 0-1). score_threshold: float = 0.1 - #: Hard cap on the injection text returned by get_context. + #: Hard cap on the injection text returned by get_context; memories that + #: do not fit whole are skipped (truncation happens on entry boundaries). max_injection_chars: int = 12000 #: Per-request HTTP timeout in seconds. timeout_seconds: float = 10.0 diff --git a/backend/packages/harness/deerflow/agents/memory/backends/mem0/mem0_manager.py b/backend/packages/harness/deerflow/agents/memory/backends/mem0/mem0_manager.py index 6207c7f33..54c599b96 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/mem0/mem0_manager.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/mem0/mem0_manager.py @@ -196,19 +196,41 @@ class Mem0Manager(MemoryManager): max_items=top_k, ), ) + budget = self._config.max_injection_chars seen: set[str] = set() lines: list[str] = [] + used = 0 + shortest_line: int | None = None for record in records: rid = record.get("id") if rid in seen: continue seen.add(rid) text = str(record.get("memory") or "").strip() - if text: - lines.append(f"- {text}") + if not text: + continue + line = f"- {text}" + line_len = len(line) + shortest_line = line_len if shortest_line is None else min(shortest_line, line_len) + # Truncate on entry boundaries: keep only memories that fit whole + # within the remaining budget (+1 for the joining newline), so the + # injection never ends mid-entry with a dangling partial line. An + # oversized entry is skipped -- a shorter later one may still fit. + added = line_len if not lines else line_len + 1 + if used + added > budget: + continue + lines.append(line) + used += added context = "\n".join(lines) - if len(context) > self._config.max_injection_chars: - context = context[: self._config.max_injection_chars] + if not context and shortest_line is not None: + # Every recalled memory was longer than the configured budget. + # Keep the entry-boundary guarantee and surface the config problem + # with a warning rather than injecting a partial fact. + logger.warning( + "max_injection_chars=%d is smaller than the shortest recalled memory (%d chars); returning empty context", + budget, + shortest_line, + ) return context async def aget_context( diff --git a/backend/tests/test_mem0_memory_backend.py b/backend/tests/test_mem0_memory_backend.py index 070db8392..eda6b1e4d 100644 --- a/backend/tests/test_mem0_memory_backend.py +++ b/backend/tests/test_mem0_memory_backend.py @@ -476,6 +476,37 @@ class TestMem0ManagerGetContext: ctx = mgr.get_context("u1") assert len(ctx) <= 20 + def test_truncates_on_entry_boundary(self) -> None: + """Budget truncation must keep whole entries, never cut a memory + mid-line and leave a dangling partial entry in the prompt.""" + mgr, fake = _manager({"max_injection_chars": 20}) + fake.list_results = [ + {"id": "m1", "memory": "a" * 10}, # "- aaaaaaaaaa" = 12 chars, fits + {"id": "m2", "memory": "b" * 10}, # + "\n" + 12 = 25 > 20, must be dropped whole + ] + ctx = mgr.get_context("u1") + assert ctx == "- " + "a" * 10 + + def test_skips_oversized_entry_and_keeps_later_fitting_one(self) -> None: + """An entry that does not fit whole is skipped; a shorter later entry + may still fit within the remaining budget.""" + mgr, fake = _manager({"max_injection_chars": 20}) + fake.list_results = [ + {"id": "m1", "memory": "x" * 30}, # 32-char line, does not fit whole + {"id": "m2", "memory": "short"}, # "- short" = 7 chars, fits + ] + ctx = mgr.get_context("u1") + assert ctx == "- short" + + def test_oversized_entries_return_empty_with_warning(self, caplog: pytest.LogCaptureFixture) -> None: + """When no memory fits the budget, keep the entry-boundary guarantee by + returning empty context and logging a diagnosable warning.""" + mgr, fake = _manager({"max_injection_chars": 20}) + fake.list_results = [{"id": "m1", "memory": "x" * 30}] + ctx = mgr.get_context("u1") + assert ctx == "" + assert any("max_injection_chars=20" in r.message and "shortest recalled memory" in r.message for r in caplog.records) + def test_async_get_context_offloads_sync_http_client(self) -> None: mgr, fake = _manager() event_loop_thread = threading.get_ident() From 5b7ada0cac7afdcd44ddf0481bb3f1a681fd9504 Mon Sep 17 00:00:00 2001 From: Huixin615 Date: Fri, 31 Jul 2026 22:50:11 +0800 Subject: [PATCH 10/31] fix(frontend): dedupe injected user messages during long runs (#4620) --- frontend/AGENTS.md | 2 +- frontend/src/core/threads/hooks.ts | 88 ++++++++++++- .../threads/local-turn-order.dom.test.tsx | 108 ++++++++++++++++ .../unit/core/threads/message-merge.test.ts | 121 ++++++++++++++++++ 4 files changed, 316 insertions(+), 3 deletions(-) create mode 100644 frontend/tests/unit/core/threads/local-turn-order.dom.test.tsx diff --git a/frontend/AGENTS.md b/frontend/AGENTS.md index c9eb6f3e1..b38e706af 100644 --- a/frontend/AGENTS.md +++ b/frontend/AGENTS.md @@ -70,7 +70,7 @@ The frontend is a stateful chat application. Users create **threads** (conversat File-tool artifact auto-open work must run in an effect with timer cleanup; never schedule timers while rendering streamed `write_file` or `str_replace` updates. `ThreadState.artifacts` remains the authoritative artifact list. The artifacts provider persists only thread-scoped panel UI state (`open`, selected path, and a refresh bootstrap cache) in session storage; an initial empty stream value must not overwrite that restored state before history finishes loading. Formal artifact content is refreshed once when the run finishes; transient `write-file:` previews remain message-driven. -3. `useThreadHistory` loads persisted conversation pages from `GET /api/threads/{id}/messages/page`, preserving the backend's thread-global event `seq`; rendering overlays checkpoint/live copies at their matching canonical identities (a summarized checkpoint may contain a protected early input plus a recent tail). Context-compaction rescue diffs every retained visible identity rather than slicing at the first anchor, and keeps a run-scoped ledger of committed visible messages so replacement updates and repeated rolling checkpoint windows cannot erase an already displayed step. The resolver suppresses checkpoint/transient prefixes whose canonical position is still behind an unloaded cursor page instead of collapsing that unknown gap before a recent anchor, then adds optimistic messages without timestamp re-sorting. History invalidation preserves already-loaded pages so their established ordering positions are not discarded. +3. `useThreadHistory` loads persisted conversation pages from `GET /api/threads/{id}/messages/page`, preserving the backend's thread-global event `seq`; rendering overlays checkpoint/live copies at their matching canonical identities (a summarized checkpoint may contain a protected early input plus a recent tail). Context-compaction rescue diffs every retained visible identity rather than slicing at the first anchor, and keeps a run-scoped ledger of committed visible messages so replacement updates and repeated rolling checkpoint windows cannot erase an already displayed step. The resolver suppresses checkpoint/transient prefixes whose canonical position is still behind an unloaded cursor page instead of collapsing that unknown gap before a recent anchor, then adds optimistic messages without timestamp re-sorting. History invalidation preserves already-loaded pages so their established ordering positions are not discarded. Dynamic context re-keys the submitted user message from `X` to `X__user`; UI identity matching normalizes that reserved suffix only for human messages so the submitted frame and checkpoint replacement remain one visible turn. A locally submitted turn also records its pre-submit identity baseline: if `messages-tuple` publishes new AI/tool steps before `values` publishes that turn's human message, render ordering moves only those non-baseline visible steps behind the new human while leaving history, hidden controls, and reconnected runs untouched. Keep that local order anchor through finish, stop, and stream error because the SDK's settled frame can retain transient event order; replace it on the next local submit and clear it on thread switch or replay-gap recovery. 4. Stop actions call the LangGraph SDK stream stop path; `core/threads/hooks.ts` invalidates current-thread, thread-history, token-usage, and sidebar/search caches immediately and schedules one follow-up refetch because SDK stop may finish via abort + fire-and-forget cancel before backend title finalization commits 5. TanStack Query manages server state; localStorage stores user settings. The Settings > Tools MCP switch calls the targeted `PATCH /api/mcp/config` diff --git a/frontend/src/core/threads/hooks.ts b/frontend/src/core/threads/hooks.ts index fdfbb2539..83223272e 100644 --- a/frontend/src/core/threads/hooks.ts +++ b/frontend/src/core/threads/hooks.ts @@ -166,6 +166,7 @@ export function buildThreadSubmitMessages({ const EMPTY_MESSAGES: Message[] = []; const EMPTY_RUN_MESSAGES: RunMessage[] = []; const EMPTY_MESSAGE_IDENTITIES: readonly string[] = []; +const INJECTED_USER_MESSAGE_ID_SUFFIX = "__user"; const EMPTY_THREAD_VALUES: AgentThreadState = { title: "", @@ -192,7 +193,17 @@ function messageIdentity(message: Message): string | undefined { return `tool:${message.tool_call_id}`; } if (typeof message.id === "string" && message.id.length > 0) { - return `message:${message.id}`; + // DynamicContextMiddleware replaces the submitted HumanMessage(id=X) with + // a hidden SystemMessage(id=X) and the real HumanMessage(id=X__user). + // Treat those human copies as one UI message so a committed render ledger + // cannot retain X beside the later checkpoint copy X__user. + const messageId = + message.type === "human" && + message.id.endsWith(INJECTED_USER_MESSAGE_ID_SUFFIX) + ? message.id.slice(0, -INJECTED_USER_MESSAGE_ID_SUFFIX.length) || + message.id + : message.id; + return `message:${messageId}`; } return undefined; } @@ -581,6 +592,59 @@ export function mergeMessages( }); } +/** + * Keep messages from a locally submitted turn behind that turn's user input. + * LangGraph `messages-tuple` events can publish the first AI/tool steps before + * the `values` event containing the user message. Those steps are not part of + * the pre-submit baseline, so move only that visible pending segment behind the + * first new human message without disturbing established history or hidden + * checkpoint controls. The caller keeps the baseline after stream completion + * because the SDK may retain its transient event order until the next submit. + */ +export function restoreLocalTurnMessageOrder( + messages: Message[], + baselineMessageIdentities: ReadonlySet, +): Message[] { + const pendingHumanIndex = messages.findIndex((message) => { + const identity = messageIdentity(message); + return ( + message.type === "human" && + !isHiddenFromUIMessage(message) && + identity !== undefined && + !baselineMessageIdentities.has(identity) + ); + }); + if (pendingHumanIndex <= 0) { + return messages; + } + + const stablePrefix: Message[] = []; + const earlyPendingSteps: Message[] = []; + for (const message of messages.slice(0, pendingHumanIndex)) { + const identity = messageIdentity(message); + const isVisiblePendingStep = + (message.type === "ai" || message.type === "tool") && + !isHiddenFromUIMessage(message) && + identity !== undefined && + !baselineMessageIdentities.has(identity); + if (isVisiblePendingStep) { + earlyPendingSteps.push(message); + } else { + stablePrefix.push(message); + } + } + if (earlyPendingSteps.length === 0) { + return messages; + } + + return [ + ...stablePrefix, + messages[pendingHumanIndex]!, + ...earlyPendingSteps, + ...messages.slice(pendingHumanIndex + 1), + ]; +} + /** * Keep a run-scoped ledger of every visible message that reached a committed * UI frame. Live checkpoint windows can roll forward between two @@ -1615,6 +1679,7 @@ export function useThreadStream({ transientHistoryThreadIdRef.current = null; summarizedRef.current = new Set(); pendingUsageBaselineMessageIdsRef.current = new Set(); + localTurnOrderBaselineIdentitiesRef.current = null; tasksRef.current = {}; setTasks({}); invalidateStoppedThreadCaches(queryClient, threadIdRef.current, isMock); @@ -1734,6 +1799,12 @@ export function useThreadStream({ const latestMessageCountsRef = useRef({ humanMessageCount }); const sendInFlightRef = useRef(false); const messagesRef = useRef([]); + // Non-null only after a turn submitted by this mounted client. Keep it after + // finish/stop/error because the SDK can retain its transient event order in + // the settled frame. The next local submit replaces it and a thread switch or + // replay gap clears it. An empty set is meaningful for a new thread and must + // not be confused with a reconnect that has no local turn anchor. + const localTurnOrderBaselineIdentitiesRef = useRef | null>(null); // Current-stream lifecycle bridge for messages removed from the checkpoint // tail before the canonical run-event page refetch observes the journal // flush. It is never appended into useThreadHistory's persisted pages. @@ -1782,6 +1853,7 @@ export function useThreadStream({ }; summarizedRef.current = new Set(); pendingUsageBaselineMessageIdsRef.current = new Set(); + localTurnOrderBaselineIdentitiesRef.current = null; pendingPreparedReplayRef.current = null; setPendingSupersededRunIds(new Set()); setPendingSupersededMessageIds(new Set()); @@ -1883,6 +1955,9 @@ export function useThreadStream({ .map(messageIdentity) .filter((id): id is string => Boolean(id)), ); + localTurnOrderBaselineIdentitiesRef.current = new Set( + pendingUsageBaselineMessageIdsRef.current, + ); // Build optimistic files list with uploading status const optimisticFiles: FileInMessage[] = (message.files ?? []).map( @@ -2049,6 +2124,7 @@ export function useThreadStream({ setOptimisticThreadId(null); setLiveMessagesThreadId(null); setIsUploading(false); + localTurnOrderBaselineIdentitiesRef.current = null; throw error; } finally { sendInFlightRef.current = false; @@ -2086,6 +2162,9 @@ export function useThreadStream({ .map(messageIdentity) .filter((id): id is string => Boolean(id)), ); + localTurnOrderBaselineIdentitiesRef.current = new Set( + pendingUsageBaselineMessageIdsRef.current, + ); setLiveMessagesThreadId(threadId); listeners.current.onSend?.(threadId); let preparedSupersededRunId: string | null = null; @@ -2172,6 +2251,7 @@ export function useThreadStream({ setOptimisticMessages([]); setOptimisticThreadId(null); setLiveMessagesThreadId(null); + localTurnOrderBaselineIdentitiesRef.current = null; if (preparedSupersededRunId) { const supersededRunId = preparedSupersededRunId; pendingPreparedReplayRef.current = null; @@ -2332,11 +2412,15 @@ export function useThreadStream({ transientHistoryOrder, previouslyRenderedOrder, ); - return mergeMessages( + const merged = mergeMessages( effectiveHistory, renderMessages, visibleOptimisticMessages, ); + const localTurnOrderBaseline = localTurnOrderBaselineIdentitiesRef.current; + return localTurnOrderBaseline === null + ? merged + : restoreLocalTurnMessageOrder(merged, localTurnOrderBaseline); }, [ previouslyRenderedOrder, renderMessages, diff --git a/frontend/tests/unit/core/threads/local-turn-order.dom.test.tsx b/frontend/tests/unit/core/threads/local-turn-order.dom.test.tsx new file mode 100644 index 000000000..975fd1e5a --- /dev/null +++ b/frontend/tests/unit/core/threads/local-turn-order.dom.test.tsx @@ -0,0 +1,108 @@ +import type { Message } from "@langchain/langgraph-sdk"; +import { expect, rs, test } from "@rstest/core"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, renderHook } from "@testing-library/react"; +import { createElement, type ReactNode } from "react"; + +import { I18nContext } from "@/core/i18n/context"; +import { DEFAULT_LOCAL_SETTINGS } from "@/core/settings/local"; + +const streamMockState = rs.hoisted(() => ({ + isLoading: false, + messages: [] as Message[], + onFinish: undefined as + | ((state: { values: { messages: Message[] } }) => void) + | undefined, + stop: rs.fn(async () => undefined), + submit: rs.fn(async () => undefined), +})); + +rs.mock("@langchain/langgraph-sdk/react", () => ({ + useStream: (options: { + onFinish?: (state: { values: { messages: Message[] } }) => void; + }) => { + streamMockState.onFinish = options.onFinish; + return { + isLoading: streamMockState.isLoading, + messages: streamMockState.messages, + stop: streamMockState.stop, + submit: streamMockState.submit, + values: { + artifacts: [], + messages: streamMockState.messages, + title: "", + todos: [], + }, + }; + }, +})); + +test("keeps early streamed steps behind a local user message after finish", async () => { + const { useThreadStream } = await import("@/core/threads/hooks"); + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + const wrapper = ({ children }: { children: ReactNode }) => + createElement( + QueryClientProvider, + { client: queryClient }, + createElement( + I18nContext.Provider, + { + value: { + locale: "en-US", + setLocale: () => undefined, + }, + }, + children, + ), + ); + const { rerender, result } = renderHook( + () => + useThreadStream({ + context: DEFAULT_LOCAL_SETTINGS.context, + isMock: true, + threadId: "thread-1", + }), + { wrapper }, + ); + + await act(async () => { + await result.current.sendMessage("thread-1", { + files: [], + text: "Build a presentation", + }); + }); + + const earlyAssistantStep = { + id: "early-assistant-step", + type: "ai", + content: "Reading the presentation skill", + } as Message; + const injectedHuman = { + id: "current-request__user", + type: "human", + content: "Build a presentation", + } as Message; + streamMockState.messages = [earlyAssistantStep, injectedHuman]; + streamMockState.isLoading = true; + rerender(); + + expect(result.current.thread.messages).toEqual([ + injectedHuman, + earlyAssistantStep, + ]); + + act(() => { + streamMockState.onFinish?.({ + values: { messages: streamMockState.messages }, + }); + streamMockState.isLoading = false; + rerender(); + }); + + expect(result.current.thread.messages).toEqual([ + injectedHuman, + earlyAssistantStep, + ]); +}); diff --git a/frontend/tests/unit/core/threads/message-merge.test.ts b/frontend/tests/unit/core/threads/message-merge.test.ts index 444a99874..003ba856c 100644 --- a/frontend/tests/unit/core/threads/message-merge.test.ts +++ b/frontend/tests/unit/core/threads/message-merge.test.ts @@ -22,6 +22,7 @@ import { removeSetItems, resolveThreadTransientHistoryBridge, resolveTransientHistoryBridge, + restoreLocalTurnMessageOrder, type ThreadMessagesPageResponse, } from "@/core/threads/hooks"; import type { RunMessage } from "@/core/threads/types"; @@ -1502,6 +1503,126 @@ test("rendered message ledger survives rolling live windows before repeated comp ); }); +test("rendered message ledger replaces a submitted user message with its injected server copy", () => { + const submittedHuman = { + id: "request-1", + type: "human", + content: "Build a presentation", + } as Message; + const injectedSystemReminder = { + id: "request-1", + type: "system", + content: "today", + additional_kwargs: { hide_from_ui: true }, + } as Message; + const injectedMemory = { + id: "request-1__memory", + type: "human", + content: "context", + additional_kwargs: { hide_from_ui: true }, + } as Message; + const injectedHuman = { + id: "request-1__user", + type: "human", + content: "Build a presentation", + name: "user-input", + } as Message; + const assistantStep = { + id: "assistant-step-1", + type: "ai", + content: "Reading the presentation skill", + } as Message; + + const firstLedger = mergeRenderedMessageLedger([], [submittedHuman]); + const nextFrame = mergeMessages( + [submittedHuman], + [injectedSystemReminder, injectedMemory, injectedHuman, assistantStep], + [], + ).filter((message) => message.additional_kwargs?.hide_from_ui !== true); + const nextLedger = mergeRenderedMessageLedger(firstLedger, nextFrame); + + expect(nextLedger).toEqual([injectedHuman, assistantStep]); + expect(nextLedger.filter((message) => message.type === "human")).toHaveLength( + 1, + ); +}); + +test("local turn order keeps early streamed steps behind the user message", () => { + const previousHuman = { + id: "previous-human", + type: "human", + content: "Previous request", + } as Message; + const previousAssistant = { + id: "previous-assistant", + type: "ai", + content: "Previous answer", + } as Message; + const earlyAssistantStep = { + id: "early-assistant-step", + type: "ai", + content: "Reading the presentation skill", + } as Message; + const optimisticHuman = { + id: "opt-human-current", + type: "human", + content: "Build a presentation", + } as Message; + const injectedHuman = { + id: "current-request__user", + type: "human", + content: "Build a presentation", + } as Message; + const injectedMemory = { + id: "current-request__memory", + type: "human", + content: "context", + additional_kwargs: { hide_from_ui: true }, + } as Message; + const laterAssistantStep = { + id: "later-assistant-step", + type: "ai", + content: "Writing the presentation plan", + } as Message; + const baselineIdentities = new Set([ + "message:previous-human", + "message:previous-assistant", + ]); + + expect( + restoreLocalTurnMessageOrder( + [previousHuman, previousAssistant, earlyAssistantStep, optimisticHuman], + baselineIdentities, + ), + ).toEqual([ + previousHuman, + previousAssistant, + optimisticHuman, + earlyAssistantStep, + ]); + + expect( + restoreLocalTurnMessageOrder( + [ + previousHuman, + previousAssistant, + earlyAssistantStep, + injectedMemory, + injectedHuman, + laterAssistantStep, + ], + baselineIdentities, + ), + ).toEqual([ + previousHuman, + previousAssistant, + injectedMemory, + injectedHuman, + earlyAssistantStep, + laterAssistantStep, + ]); +}); + test("rendered message ledger does not retain explicitly superseded messages", () => { const retained = { id: "retained-answer", From 2a143dced6adb80190b07c7db3ca79bff3a853ad Mon Sep 17 00:00:00 2001 From: Nan Gao Date: Sat, 1 Aug 2026 02:35:02 +0200 Subject: [PATCH 11/31] fix(docker): bind the published entry port to loopback by default (#4618) README documents DeerFlow as deployed by default "in a local trusted environment (accessible only via the 127.0.0.1 loopback interface)", but both compose files published nginx as `"${PORT:-2026}:2026"`, which Docker binds to 0.0.0.0 and [::]. The shipped artifact did not match its own documented default, so running it on a LAN or cloud host produced a wider surface than the docs implied without the operator changing anything -- and the agent can execute commands. Publish as `"${BIND_HOST:-127.0.0.1}:${PORT:-2026}:2026"` in both compose files, so the default matches the documented model while operators who front the stack with their own TLS/auth can still widen it via BIND_HOST. The Gateway keeps binding 0.0.0.0:8001 inside the container (nginx reaches it over the compose network) and its port stays unpublished, so the published nginx port is the entire external surface. BREAKING CHANGE: a deployment that relied on the previous 0.0.0.0 default becomes unreachable from other hosts after this upgrade. Set BIND_HOST=0.0.0.0 in .env to restore it, after putting authentication in front and completing first-run setup. Also: - .env.example documents BIND_HOST and PORT with the reasoning. - deploy.sh reports the address the stack actually bound and, when it is not loopback, tells the operator to complete first-run setup immediately. It reads BIND_HOST/PORT from .env via a new read_dotenv_value helper following compose precedence; the shell does not source .env, so reading the environment alone would have reported "loopback only" for a stack .env had exposed. The pre-existing ${PORT} summary line had the same defect and is fixed with it. - test_compose_default_bind_host.py pins the loopback default, that BIND_HOST stays overridable, and that no service in either compose file publishes a port without an explicit bind address, so a later addition cannot drift back to 0.0.0.0 unnoticed. --- .env.example | 8 ++ AGENTS.md | 10 ++ README.md | 11 ++ .../tests/test_compose_default_bind_host.py | 112 ++++++++++++++++++ docker/docker-compose-dev.yaml | 5 +- docker/docker-compose.yaml | 7 +- scripts/deploy.sh | 50 +++++++- 7 files changed, 199 insertions(+), 4 deletions(-) create mode 100644 backend/tests/test_compose_default_bind_host.py diff --git a/.env.example b/.env.example index 3f66d0ae6..3f27fc7c9 100644 --- a/.env.example +++ b/.env.example @@ -13,6 +13,14 @@ INFOQUEST_API_KEY=your-infoquest-api-key # Leave unset when using the unified nginx endpoint, e.g. http://localhost:2026. # GATEWAY_CORS_ORIGINS=http://localhost:3000,http://127.0.0.1:3000 +# Host interface the Docker stack publishes its entry port on. Defaults to +# 127.0.0.1 (loopback only), matching the local-trusted-environment deployment +# model documented in README.md -- the agent can execute commands. +# Set 0.0.0.0 only when the host is protected by your own TLS/auth front door +# or firewall, and complete first-run setup before it becomes reachable. +# BIND_HOST=0.0.0.0 +# PORT=2026 + # Optional: # FIRECRAWL_API_KEY=your-firecrawl-api-key # VOLCENGINE_API_KEY=your-volcengine-api-key diff --git a/AGENTS.md b/AGENTS.md index c9aab70db..1580d4c70 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,6 +36,16 @@ to the Gateway's LangGraph runtime, rewriting it to Gateway's native `/api/*` ro other `/api/*` go straight to the Gateway REST routers. See [backend/AGENTS.md](backend/AGENTS.md) for the runtime and router detail. +Both compose files publish that entry as `"${BIND_HOST:-127.0.0.1}:${PORT:-2026}:2026"` +— **loopback by default**, matching the README's documented deployment model. A bare +`"${PORT}:2026"` binds `0.0.0.0`, which does not. +Nginx itself listens `default_server` on IPv4+IPv6 and the +Gateway binds `0.0.0.0:8001` inside the container on purpose — both are container- +internal; the published nginx port is the entire external surface, and the Gateway's +`8001` is deliberately not published. Any new published port needs an explicit bind +address; `backend/tests/test_compose_default_bind_host.py` pins this for every service +in both compose files. + ## Repository Map ``` diff --git a/README.md b/README.md index e8891daef..976463959 100644 --- a/README.md +++ b/README.md @@ -1136,6 +1136,17 @@ DeerFlow has key high-privilege capabilities including **system command executio - **Unauthorized illegal invocation**: Agent functionality could be discovered by unauthorized third parties or malicious internet scanners, triggering bulk unauthorized requests that execute high-risk operations such as system commands and file read/write, potentially causing serious security consequences. - **Compliance and legal risks**: If the agent is illegally invoked to conduct cyberattacks, data theft, or other illegal activities, it may result in legal liability and compliance risks. +### Deployment Defaults + +The Docker stack publishes its entry port on `127.0.0.1` only, matching the +local-trusted-environment model described above. To reach it from another +machine, set `BIND_HOST` in `.env` (e.g. `BIND_HOST=0.0.0.0`) — and only after +putting the security measures below in place. + +**Complete first-run setup before the host becomes reachable.** A fresh +instance has no accounts yet, so create the admin account through `/setup` +immediately after starting any deployment that is not loopback-only. + ### Security Recommendations **Note: We strongly recommend deploying DeerFlow in a local trusted network environment.** If you need cross-device or cross-network deployment, you must implement strict security measures, such as: diff --git a/backend/tests/test_compose_default_bind_host.py b/backend/tests/test_compose_default_bind_host.py new file mode 100644 index 000000000..1064bb53a --- /dev/null +++ b/backend/tests/test_compose_default_bind_host.py @@ -0,0 +1,112 @@ +"""Regression test for the Docker Compose default published bind address. + +``README.md`` documents DeerFlow as being deployed by default "in a local +trusted environment (accessible only via the 127.0.0.1 loopback interface)", +but the shipped compose files published the nginx entry as +``"${PORT:-2026}:2026"``, which Docker binds to ``0.0.0.0`` (and ``[::]``). The +shipped artifact therefore did not match its own documented default, and an +operator running it on a LAN or cloud host got a wider surface than the docs +implied without changing anything. + +The Gateway itself binds ``0.0.0.0`` inside the container on purpose (nginx has +to reach it over the compose network) and its port is deliberately not +published, so the published nginx port is the whole external surface. This test +pins the loopback default there while keeping it overridable for operators who +intentionally expose the stack behind their own TLS/auth front door. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +COMPOSE_PATHS = { + "prod": REPO_ROOT / "docker" / "docker-compose.yaml", + "dev": REPO_ROOT / "docker" / "docker-compose-dev.yaml", +} + +EXPECTED_NGINX_PORT_MAPPING = "${BIND_HOST:-127.0.0.1}:${PORT:-2026}:2026" + + +def _published_ports(compose_path: Path) -> dict[str, list[str]]: + """Return {service_name: [port mapping, ...]} for every published port.""" + compose = yaml.safe_load(compose_path.read_text(encoding="utf-8")) + published: dict[str, list[str]] = {} + for service_name, service in (compose.get("services") or {}).items(): + ports = service.get("ports") if isinstance(service, dict) else None + if not ports: + continue + published[service_name] = [str(entry) for entry in ports] + return published + + +@pytest.mark.parametrize("variant", sorted(COMPOSE_PATHS)) +def test_nginx_entry_defaults_to_loopback(variant: str): + """With BIND_HOST unset, the entry port must bind 127.0.0.1, not 0.0.0.0.""" + published = _published_ports(COMPOSE_PATHS[variant]) + + assert published.get("nginx") == [EXPECTED_NGINX_PORT_MAPPING], f"{variant} compose must publish nginx as {EXPECTED_NGINX_PORT_MAPPING!r}; got: {published.get('nginx')!r}" + + +@pytest.mark.parametrize("variant", sorted(COMPOSE_PATHS)) +def test_no_service_publishes_on_all_interfaces(variant: str): + """No compose service may publish a port without an explicit bind address. + + A bare ``"HOST:CONTAINER"`` mapping binds every interface. Any port added + later must either stay internal to the compose network or opt in to the + same ``BIND_HOST`` default. + """ + offenders: list[str] = [] + for service_name, mappings in _published_ports(COMPOSE_PATHS[variant]).items(): + for mapping in mappings: + # A bind address is present only when the mapping has three + # colon-separated parts (``ADDR:HOST:CONTAINER``). Variable + # substitutions such as ``${PORT:-2026}`` also contain colons, so + # count separators outside ``${...}`` instead of splitting naively. + if _bind_address(mapping) is None: + offenders.append(f"{service_name}: {mapping}") + + assert not offenders, f"{variant} compose publishes ports on all interfaces (add a bind address): {offenders}" + + +@pytest.mark.parametrize("variant", sorted(COMPOSE_PATHS)) +def test_bind_address_remains_overridable(variant: str): + """Operators fronting the stack themselves must be able to widen the bind.""" + mapping = _published_ports(COMPOSE_PATHS[variant])["nginx"][0] + + assert _bind_address(mapping) == "${BIND_HOST:-127.0.0.1}", f"{variant} compose must keep the bind address overridable via BIND_HOST; got: {mapping!r}" + + +def _bind_address(mapping: str) -> str | None: + """Return the bind-address segment of a compose port mapping, if any. + + Splits on ``:`` at nesting depth zero so ``${PORT:-2026}`` is treated as a + single segment rather than two. + """ + segments: list[str] = [] + current: list[str] = [] + depth = 0 + index = 0 + while index < len(mapping): + char = mapping[index] + if mapping.startswith("${", index): + depth += 1 + current.append("${") + index += 2 + continue + if char == "}" and depth > 0: + depth -= 1 + elif char == ":" and depth == 0: + segments.append("".join(current)) + current = [] + index += 1 + continue + current.append(char) + index += 1 + segments.append("".join(current)) + + # ADDR:HOST:CONTAINER -> bound; HOST:CONTAINER or CONTAINER -> unbound. + return segments[0] if len(segments) >= 3 else None diff --git a/docker/docker-compose-dev.yaml b/docker/docker-compose-dev.yaml index 6ff1d9cb3..35a9dd9e1 100644 --- a/docker/docker-compose-dev.yaml +++ b/docker/docker-compose-dev.yaml @@ -92,8 +92,11 @@ services: nginx: image: nginx:alpine container_name: deer-flow-nginx + # Loopback-only by default; see the note in docker-compose.yaml. Override + # with BIND_HOST when you deliberately need the dev stack reachable from + # another machine. ports: - - "2026:2026" + - "${BIND_HOST:-127.0.0.1}:${PORT:-2026}:2026" volumes: - ./nginx/nginx.conf:/etc/nginx/nginx.conf.template:ro command: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 461d8ec3a..3744d12f2 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -45,8 +45,13 @@ services: nginx: image: nginx:alpine container_name: deer-flow-nginx + # Loopback-only by default: DeerFlow's agent can execute commands, so the + # documented default deployment is a local trusted environment. A bare + # "${PORT}:2026" would bind 0.0.0.0 instead, which does not match that. + # Set BIND_HOST=0.0.0.0 only behind your own TLS/auth front door, and + # complete first-run setup before the host becomes reachable. ports: - - "${PORT:-2026}:2026" + - "${BIND_HOST:-127.0.0.1}:${PORT:-2026}:2026" volumes: - ./nginx/nginx.conf:/etc/nginx/nginx.conf.template:ro command: > diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 56697ce04..a07db701c 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -73,6 +73,37 @@ load_uv_extras_from_dotenv() { load_uv_extras_from_dotenv +# Read one key from $ENV_FILE the way compose --env-file interpolates it, so the +# final summary reports the values the stack actually came up with. The shell +# does not source $ENV_FILE, so reading these from the environment alone would +# report "loopback only" for a stack that .env exposed to the network. +read_dotenv_value() { + local key="$1" + local line="" + local value="" + + # An exported shell variable wins, matching compose precedence. + if [ -n "${!key+x}" ]; then + printf '%s' "${!key}" + return 0 + fi + + [ -f "$ENV_FILE" ] || return 0 + + line="$(grep -E "^[[:space:]]*(export[[:space:]]+)?${key}[[:space:]]*=" "$ENV_FILE" | tail -n 1 || true)" + [ -n "$line" ] || return 0 + + value="${line#*=}" + value="${value%$'\r'}" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + case "$value" in + \"*\") value="${value#\"}"; value="${value%\"}" ;; + \'*\') value="${value#\'}"; value="${value%\'}" ;; + esac + printf '%s' "$value" +} + # ── Colors ──────────────────────────────────────────────────────────────────── GREEN='\033[0;32m' @@ -373,11 +404,26 @@ echo "==========================================" echo " DeerFlow is running!" echo "==========================================" echo "" -echo " 🌐 Application: http://localhost:${PORT:-2026}" -echo " 📡 API Gateway: http://localhost:${PORT:-2026}/api/*" +RESOLVED_PORT="$(read_dotenv_value PORT)" +RESOLVED_PORT="${RESOLVED_PORT:-2026}" +RESOLVED_BIND_HOST="$(read_dotenv_value BIND_HOST)" +RESOLVED_BIND_HOST="${RESOLVED_BIND_HOST:-127.0.0.1}" + +echo " 🌐 Application: http://localhost:${RESOLVED_PORT}" +echo " 📡 API Gateway: http://localhost:${RESOLVED_PORT}/api/*" echo " 🤖 Runtime: Gateway embedded" echo " API: /api/langgraph/* → Gateway" echo "" +if [ "$RESOLVED_BIND_HOST" = "127.0.0.1" ] || [ "$RESOLVED_BIND_HOST" = "::1" ] || [ "$RESOLVED_BIND_HOST" = "localhost" ]; then + echo " 🔒 Bound to ${RESOLVED_BIND_HOST} — reachable from this machine only." + echo " To expose it, set BIND_HOST in .env, put TLS/auth in front, and" + echo " create the admin account before the host becomes reachable." +else + echo " ⚠️ Bound to ${RESOLVED_BIND_HOST} — reachable from the network." + echo " Open http://localhost:${RESOLVED_PORT} and complete first-run" + echo " setup now, before anyone else reaches this host." +fi +echo "" echo " Manage:" echo " make down — stop and remove containers" echo " make docker-logs — view logs" From cccda35cc52f837934cb54b6b328d97a16d090ca Mon Sep 17 00:00:00 2001 From: Aari Date: Sat, 1 Aug 2026 08:39:28 +0800 Subject: [PATCH 12/31] fix(memory): prevent task-scoped data from entering long-term memory (#4604) * fix(memory): gate long-term updates by scope * docs(memory): note custom prompts_dir migration; fix stale accept-filter comment * fix(memory): harden scope-gate review paths --- README.md | 2 + README_zh.md | 2 + backend/AGENTS.md | 1 + backend/README.md | 2 + .../deermem/core/prompts/consolidation.yaml | 3 +- .../core/prompts/memory_update.chat.yaml | 42 +- .../backends/deermem/deermem/core/updater.py | 285 +++++++++--- .../harness/deerflow/agents/memory/manager.py | 18 + backend/tests/test_deermem_self_contained.py | 2 +- backend/tests/test_memory_consolidation.py | 80 ++-- backend/tests/test_memory_scope_gate.py | 411 ++++++++++++++++++ backend/tests/test_memory_staleness_review.py | 14 +- backend/tests/test_memory_updater.py | 49 ++- 13 files changed, 769 insertions(+), 142 deletions(-) create mode 100644 backend/tests/test_memory_scope_gate.py diff --git a/README.md b/README.md index 976463959..2ac58c9eb 100644 --- a/README.md +++ b/README.md @@ -1014,6 +1014,8 @@ requires an explicit local-development opt-in. See the Memory updates now skip duplicate fact entries at apply time, so repeated preferences and context do not accumulate endlessly across sessions. +In the default DeerMem `middleware` mode, automatic extraction now classifies every proposed fact by scope, durability, and authority before a deterministic write gate accepts it. Only durable, descriptive user-level facts are stored; current-thread or project constraints and one-time action permissions stay in conversation state. User-global summaries require both user scope and descriptive authority, contradiction removals are scope-gated, and a replacement-dependent removal is applied only when its replacement actually survives validation and storage. These classification labels are extraction-only metadata, add no extra LLM call, and are not written into the fact files. The explicit CRUD tools in `memory.mode: tool` remain a separate, model-directed path. Deployments that override the bundled DeerMem prompts via `memory.backend_config.prompts_dir` must add the new classification fields to their custom templates (the `memory_update` fact/summary/removal formats and the `consolidation` consolidated-fact schema): the write gate fails closed, so an un-migrated template stops every extraction-driven fact, summary, and removal write, surfacing only through the `rejected_by_scope_gate` metrics and the high-rejection-rate warning. + File-backed memory now separates global user context from agent facts. Each user has one `memory.json` containing only the project-independent `user` and `history` summaries; every fact is a canonical Markdown file below `agents/{agent_name}/facts/`. Existing lead-agent middleware, API, Settings, import/export, and embedded-client calls that omit `agent_name` resolve inside DeerMem to the reserved `__default__` bucket. That bucket is outside the valid custom-agent name grammar, so a real custom agent named `lead-agent` has a separate fact repository and deleting a custom agent cannot delete a memory-only directory without `config.yaml`. Public agent identifiers are case-insensitive and canonicalized to lowercase. Runtime/API readers still receive a compatibility `facts` array for the selected/default agent, so the frontend does not read agent facts from `memory.json`; structured Markdown `source` metadata is projected to the historical string field at the MemoryManager boundary. An unscoped Clear All first migrates facts from unread legacy per-agent JSON without adopting its soon-to-be-cleared summaries, then removes shared summaries and facts from every agent bucket while preserving agent configuration files, so a later read cannot resurrect skipped legacy facts; an explicitly agent-scoped clear removes only that agent's facts. On first normal read, old facts embedded in the user JSON are migrated automatically to `__default__`; facts written to the earlier implicit `lead-agent` bucket are also moved when that directory is not a real custom agent. Migration and normal writes notify the configured retrieval adapter only after durable storage locks are released. DeerMem uses a scope-aware SQLite FTS5/BM25 adapter by default, stores only rebuildable derived index data under `.retrieval/`, and rebuilds it in the background during Gateway startup or lazily on the first scoped search. A corrupt derived index is recreated automatically. Set `memory.backend_config.retrieval_adapter` to an empty string to disable it and use the local substring fallback. Chinese tokenization is optional; install the backend `memory-zh` extra (`uv sync --extra memory-zh`) for jieba-assisted sub-phrase search. Journaled writes, a shared user lock, and optimistic user-memory revisions prevent silent lost updates. Memory injection follows the configured operation mode. In `middleware` mode, DeerMem injects the user-global summaries and the selected agent's facts. In `tool` mode, the automatic `` block contains only the global `user` and `history` summaries; agent facts are retrieved explicitly through `memory_search`, avoiding duplicate automatic and tool-returned fact context. Setting `memory.injection_enabled: false` still disables the entire block in either mode. diff --git a/README_zh.md b/README_zh.md index 29be95063..b8da18e6e 100644 --- a/README_zh.md +++ b/README_zh.md @@ -651,6 +651,8 @@ DeerFlow 不只是“会说它能做”，它是真的有一台自己的“电 跨 session 使用时，DeerFlow 会逐步积累关于你的持久 memory，包括你的个人偏好、知识背景，以及长期沉淀下来的工作习惯。你用得越多，它越了解你的写作风格、技术栈和重复出现的工作流。memory 保存在本地，控制权也始终在你手里。 +默认 DeerMem `middleware` 模式会先判断候选信息的作用域、持久性和授权属性，再由确定性写入门决定是否保存。只有稳定、描述性的用户级事实能进入长期 memory；当前对话或项目的约束、一次性操作授权仍留在对话状态中。用户全局 summary 必须同时具有用户级作用域和描述性授权属性，基于矛盾的删除也会经过作用域保护；如果删除依赖一条替代事实，只有替代事实真正通过校验并保留下来后才执行删除。这些分类字段只用于本次抽取，不写入 fact 文件，也不增加 LLM 调用次数。`memory.mode: tool` 的显式 CRUD 仍是独立的模型直写路径。如果通过 `memory.backend_config.prompts_dir` 覆盖了内置抽取模板，必须同步在自定义模板中加入新的分类字段（`memory_update` 的 fact/summary/removal 格式与 `consolidation` 的合并 fact 结构）：写入门是 fail closed 的，未迁移的旧模板会导致所有抽取驱动的 fact、summary 与删除写入停止，只能通过 `rejected_by_scope_gate` 指标和高拒绝率告警发现。 + ## 推荐模型 DeerFlow 对模型没有强绑定，只要实现了 OpenAI 兼容 API 的 LLM，理论上都可以接入。不过在下面这些能力上表现更强的模型，通常会更适合 DeerFlow： diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 016a04f20..988e56066 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -935,6 +935,7 @@ The cached value is reused for both the blocking (`runs.wait`) and streaming (`_ runtime. - `memory.mode: tool` skips `MemoryMiddleware` and registers `memory_search`, `memory_add`, `memory_update`, and `memory_delete` on the agent. The model decides when to search, add, update, or delete facts; this is opt-in/experimental and should not be described as better than middleware mode without eval evidence. - Both modes share `FileMemoryStorage`, per-user/per-agent isolation, manual CRUD primitives, and the updater backend. Injection is mode-aware: middleware mode injects global `user`/`history` summaries plus the selected agent's facts, while tool mode injects only the global summaries and leaves every agent fact behind `memory_search` to avoid duplicating automatically injected and retrieval-returned context. `memory.injection_enabled: false` suppresses the complete block in either mode. +- Middleware extraction classifies proposed facts with extraction-only `scope`/`durability`/`authority` labels. `_apply_updates` accepts only `user` + `durable` + `descriptive` new/consolidated facts, accepts only wholly user-scoped summary prose with `authority=descriptive`, and rejects missing labels per item without aborting unrelated updates. Contradiction removals use object entries with `id`, `scope`, `reason`, and optional zero-based `replacementFactIndex`; task/project removals fail closed, and a paired removal runs only when the referenced replacement survives the scope/confidence gates, deduplication, and max-fact trim under another fact ID. The labels are not persisted, so no storage migration is required. Staleness removals retain their independent candidate/cap guardrails, while tool-mode CRUD remains outside this extraction gate. Custom `memory.backend_config.prompts_dir` templates (including per-agent overrides) must carry the same classification fields; an un-migrated template makes the fail-closed gate reject every extraction-driven write, observable only through `rejected_by_scope_gate` and the >60% fact-rejection warning. - Middleware mode queue debounces (30s default), batches updates, and commits global summaries plus the selected/default agent's fact delta through a user-level lock, optimistic user-memory revisions, per-fact revisions, and a recoverable target-file journal. Only explicitly marked point operations may rebase a stale shared revision, and only while every addressed fact still satisfies its original absent/revision precondition. Snapshot-derived clear/trim/consolidation operations instead reload the complete document and recompute their intent on a manifest conflict, with a bounded retry. Typed manifest/fact conflict subclasses keep that decision independent of exception text, and same-ID creates and stale same-fact writes fail. Scope-lock objects are weakly cached so inactive users do not grow a process-lifetime map. Cache validation does not scale with the fact-file count: its token combines the shared JSON's `(mtime_ns, size, revision)`, so the persisted revision invalidates stale caches even when a coarse-mtime filesystem reports identical metadata for same-size writes; direct out-of-band Markdown edits require `reload()`. Atomic replacement also syncs the parent directory on POSIX so the rename is durable. DeerMem translates private storage conflict/corruption exceptions to the backend-neutral MemoryManager contract; the Gateway maps them to HTTP 409 and a stable HTTP 500 response respectively. A normal default-manager read automatically migrates legacy facts from the global JSON into `__default__`; it also adopts the earlier implicit `lead-agent` fact bucket only when that directory has no custom-agent `config.yaml`, and rejects unexpected files instead of deleting them. The v1-to-v2 migration is one-way for the running application: operators must stop DeerFlow and snapshot the configured storage root before upgrade. Before any destructive v2 write, every migrated JSON source is durably retained as `{manifest_filename}.v1.bak`; a missing-write or mismatched existing backup aborts without modifying v1 data. Legacy per-agent JSON is deleted only after its non-empty summaries are safely adopted or confirmed identical; summary conflicts keep the source file and fail loudly. - **Proactive Markdown migration CLI**: from `backend/`, run `PYTHONPATH=. python scripts/migrate_memory_markdown.py --all-users --dry-run` to audit and omit `--dry-run` to migrate before serving traffic. Use repeated `--user-id` values when selecting exact original identities, especially standalone raw IDs containing `@` or other characters that are normalized in directory names; `--storage-path` selects a non-default DeerMem root. The CLI reuses `FileMemoryStorage.migrate`, is idempotent, continues across per-user failures, and exits non-zero if any user fails. It is optional because the first normal read still performs the same migration automatically. - `retrieval_adapter` owns indexing and retrieval. `fts5` is the DeerMem default and uses a persistent derived SQLite index under `.retrieval/`; an empty value disables the adapter and selects `substring_fallback`. File storage sends upsert/remove notifications for normal writes and both explicit and lazy migrations after releasing durable storage locks, then delegates search. Gateway startup schedules `DeerMem.warm_retrieval()` as a background full rebuild so readiness is not delayed, while a first search lazily rebuilds its exact scope until warm-up completes. Individual malformed facts are logged and skipped without triggering repeated full scans; only a fatal adapter rebuild failure keeps lazy retry enabled. During shutdown, the Gateway waits at most one second for this derived rebuild and leaves the full configured timeout to the canonical memory flush; if the rebuild is still active, its adapter remains open until process exit. Adapter failures mark the scope dirty and fall back to canonical substring search until rebuilding succeeds. `FileMemoryStorage` owns and closes the adapter so higher layers do not reach into private storage state. diff --git a/backend/README.md b/backend/README.md index 4e5c4b947..e8b87af22 100644 --- a/backend/README.md +++ b/backend/README.md @@ -91,6 +91,8 @@ Async task delegation with concurrent execution: LLM-powered persistent context retention across conversations: - **Automatic extraction**: Analyzes conversations for user context, facts, and preferences +- **Scope-safe writes**: Middleware extraction stores only durable, descriptive user-level facts; global summaries also require descriptive authority, while contradiction removals and consolidated facts fail closed when scope metadata is missing or task/project-local +- **Atomic replacements**: A contradiction removal linked to a replacement runs only after the replacement survives scope/confidence gates, deduplication, and fact-limit trimming - **Structured storage**: User context (work, personal, top-of-mind), history, and confidence-scored facts - **Debounced updates**: Batches updates to minimize LLM calls (configurable wait time) - **System prompt injection**: Top facts + context injected into agent prompts diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/prompts/consolidation.yaml b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/prompts/consolidation.yaml index 1f5825be3..800b9e170 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/prompts/consolidation.yaml +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/prompts/consolidation.yaml @@ -15,11 +15,12 @@ template: |- - SKIP: Facts are distinct enough to remain separate. Add consolidation decisions to "factsToConsolidate" in your output JSON. - Each entry: {{"sourceIds": ["fact_id_1", "fact_id_2"], "consolidated": {{"content": "...", "category": "...", "confidence": 0.9}}}} + Each entry: {{"sourceIds": ["fact_id_1", "fact_id_2"], "consolidated": {{"content": "...", "category": "...", "confidence": 0.9, "scope": "user|thread|project", "durability": "durable|temporary", "authority": "descriptive|transactional"}}}} Rules: - The consolidated fact must preserve ALL key details from source facts - Only consolidate facts that describe the same aspect of the user + - Classify every consolidated fact. It is eligible only when it is user-scoped, durable, descriptive, and safe to inject into unrelated future threads. - Confidence of consolidated fact = max of source confidences - Be conservative - when in doubt, keep facts separate - Maximum {max_groups} consolidation groups per cycle diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/prompts/memory_update.chat.yaml b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/prompts/memory_update.chat.yaml index c7cfac354..f5dca0352 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/prompts/memory_update.chat.yaml +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/prompts/memory_update.chat.yaml @@ -12,12 +12,21 @@ messages: Before extracting facts, perform a structured reflection on the conversation: 1. Error/Retry Detection: Did the agent encounter errors, require retries, or produce incorrect results? - If yes, record the root cause and correct approach as a high-confidence fact with category "correction". + If yes, record the root cause and correct approach only when it is a durable user-level working pattern. 2. User Correction Detection: Did the user correct the agent's direction, understanding, or output? - If yes, record the correct interpretation or approach as a high-confidence fact with category "correction". + If yes, distinguish a reusable user-level correction from a correction to the current task's facts or files. Include what went wrong in "sourceError" only when category is "correction" and the mistake is explicit in the conversation. 3. Project Constraint Discovery: Were any project-specific constraints discovered during the conversation? - If yes, record them as facts with the most appropriate category and confidence. + If yes, classify them as project-scoped and do not promote them to user memory. + + Scope and Safety Classification: + - scope="user": a property, preference, background detail, ongoing personal state, or durable working pattern of the user that is safe and useful to inject into an unrelated future thread, task, project, or repository. + - scope="thread": information limited to the current request or conversation, including one-off constraints for a reply, file, email, trip, PR, test, or action. + - scope="project": a rule, decision, state, or constraint that is meaningful only inside a particular project or repository, even if it may span several threads. + - durability="durable": expected to remain true across future conversations. durability="temporary": current, short-lived, or one-off information. + - authority="transactional": an instruction, grant, permission, or authorization to perform an action such as editing, deleting, pushing, closing, publishing, or force-pushing. Transactional content must never become long-term memory. + - authority="descriptive": describes the user without granting authority for an action. + - When uncertain, use thread/project or temporary; never guess user+durable. Memory Section Guidelines: @@ -43,6 +52,9 @@ messages: Include: Core expertise, longstanding interests, fundamental working style **Facts Extraction**: + - Every new fact MUST include scope, durability, and authority. These labels are evaluated by a deterministic write gate and are not persisted. + - Only facts classified as scope="user", durability="durable", authority="descriptive" are eligible for storage. + - Do not create facts for current-task objectives, acceptance criteria, workspace state, exact current file/commit/error state, project-only constraints, or one-time action permissions. - Extract specific, quantifiable details (e.g., "16k+ GitHub stars", "200+ datasets") - Include proper nouns (company names, project names, technology names) - Preserve technical terminology and version numbers @@ -85,25 +97,27 @@ messages: Output Format (JSON): {{ "user": {{ - "workContext": {{ "summary": "...", "shouldUpdate": true/false }}, - "personalContext": {{ "summary": "...", "shouldUpdate": true/false }}, - "topOfMind": {{ "summary": "...", "shouldUpdate": true/false }} + "workContext": {{ "summary": "...", "shouldUpdate": true/false, "scope": "user|thread|project", "authority": "descriptive|transactional" }}, + "personalContext": {{ "summary": "...", "shouldUpdate": true/false, "scope": "user|thread|project", "authority": "descriptive|transactional" }}, + "topOfMind": {{ "summary": "...", "shouldUpdate": true/false, "scope": "user|thread|project", "authority": "descriptive|transactional" }} }}, "history": {{ - "recentMonths": {{ "summary": "...", "shouldUpdate": true/false }}, - "earlierContext": {{ "summary": "...", "shouldUpdate": true/false }}, - "longTermBackground": {{ "summary": "...", "shouldUpdate": true/false }} + "recentMonths": {{ "summary": "...", "shouldUpdate": true/false, "scope": "user|thread|project", "authority": "descriptive|transactional" }}, + "earlierContext": {{ "summary": "...", "shouldUpdate": true/false, "scope": "user|thread|project", "authority": "descriptive|transactional" }}, + "longTermBackground": {{ "summary": "...", "shouldUpdate": true/false, "scope": "user|thread|project", "authority": "descriptive|transactional" }} }}, "newFacts": [ - {{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0, "expected_valid_days": 90 }} + {{ "content": "...", "category": "preference|knowledge|context|behavior|goal|correction", "confidence": 0.0-1.0, "expected_valid_days": 90, "scope": "user|thread|project", "durability": "durable|temporary", "authority": "descriptive|transactional" }} + ], + "factsToRemove": [ + {{ "id": "fact_id_1", "scope": "user|thread|project", "reason": "explicit user-level contradiction or retraction", "replacementFactIndex": 0 }} ], - "factsToRemove": ["fact_id_1", "fact_id_2"], "staleFactsToRemove": [{{ "id": "fact_id", "reason": "brief explanation" }}], "staleFactsToExtend": [{{ "id": "fact_id", "extend_by_days": 365, "reason": "brief explanation" }}], "factsToConsolidate": [ {{ "sourceIds": ["fact_id_1", "fact_id_2"], - "consolidated": {{ "content": "synthesized fact", "category": "knowledge", "confidence": 0.9 }} + "consolidated": {{ "content": "synthesized fact", "category": "knowledge", "confidence": 0.9, "scope": "user|thread|project", "durability": "durable|temporary", "authority": "descriptive|transactional" }} }} ] }} @@ -115,7 +129,9 @@ messages: - Only add facts that are clearly stated (0.9+) or strongly implied (0.7+) - Use category "correction" for explicit agent mistakes or user corrections; assign confidence >= 0.95 when the correction is explicit - Include "sourceError" only for explicit correction facts when the prior mistake or wrong approach is clearly stated; omit it otherwise - - Remove facts that are contradicted by new information + - Remove an existing fact only when the user explicitly contradicts or retracts it at user scope. A thread/project-local exception does not contradict a user-level fact. + - factsToRemove entries MUST include scope and reason. Use replacementFactIndex when a removal depends on a replacement in newFacts; the zero-based index must identify that replacement. Omit replacementFactIndex only for a pure user-level retraction with no replacement. + - Every summary with shouldUpdate=true MUST include scope and authority. A summary is user-scoped only when the entire prose block is safe to inject into unrelated future threads; otherwise classify it as thread/project so the write gate rejects it. Any summary containing an instruction, grant, permission, or authorization is transactional and must be rejected even when it describes a user-wide or recurring policy. - When updating topOfMind, integrate new focus areas while removing completed/abandoned ones Keep 3-5 concurrent focus themes that are still active and relevant - For history sections, integrate new information chronologically into appropriate time period diff --git a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py index 11f5f5e45..78522e04f 100644 --- a/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py +++ b/backend/packages/harness/deerflow/agents/memory/backends/deermem/deermem/core/updater.py @@ -126,6 +126,52 @@ def _extract_text(content: Any) -> str: _REQUIRED_MEMORY_UPDATE_TOP_LEVEL_KEYS = frozenset({"user", "history", "newFacts"}) +_FACT_CLASSIFICATION_FIELDS = ("scope", "durability", "authority") + + +def _normalize_gate_label(value: Any) -> str | None: + """Normalize a model-produced scope-gate label without validating policy.""" + if not isinstance(value, str): + return None + normalized = value.strip().lower() + return normalized or None + + +def _fact_scope_gate_reason(fact: dict[str, Any]) -> str | None: + """Return the deterministic rejection reason for a model-extracted fact.""" + if any(_normalize_gate_label(fact.get(field)) is None for field in _FACT_CLASSIFICATION_FIELDS): + return "missing" + if _normalize_gate_label(fact.get("scope")) != "user": + return "scope" + if _normalize_gate_label(fact.get("durability")) != "durable": + return "durability" + if _normalize_gate_label(fact.get("authority")) != "descriptive": + return "authority" + return None + + +def _summary_scope_gate_reason(section_data: dict[str, Any]) -> str | None: + """Return the deterministic rejection reason for a summary update.""" + scope = _normalize_gate_label(section_data.get("scope")) + authority = _normalize_gate_label(section_data.get("authority")) + if scope is None or authority is None: + return "missing" + if scope != "user": + return "scope" + if authority != "descriptive": + return "authority" + return None + + +def _removal_scope_gate_reason(removal: dict[str, Any]) -> str | None: + """Return the deterministic rejection reason for a contradiction removal.""" + scope = _normalize_gate_label(removal.get("scope")) + reason = removal.get("reason") + if scope is None or not isinstance(reason, str) or not reason.strip(): + return "missing" + if scope != "user": + return "scope" + return None def _normalize_memory_update_fact(fact: Any) -> dict[str, Any] | None: @@ -180,6 +226,14 @@ def _normalize_memory_update_fact(fact: Any) -> dict[str, Any] | None: if evd is not None: normalized_fact["expected_valid_days"] = evd + # Scope classification is extraction-only metadata. Preserve it through + # structural normalization so _apply_updates can fail closed per item, but + # never copy it into the persisted fact_entry. + for field in _FACT_CLASSIFICATION_FIELDS: + normalized_value = _normalize_gate_label(fact.get(field)) + if normalized_value is not None: + normalized_fact[field] = normalized_value + return normalized_fact @@ -189,7 +243,35 @@ def _normalize_memory_update_data(update_data: dict[str, Any]) -> dict[str, Any] history = update_data.get("history") new_facts = update_data.get("newFacts") facts_to_remove = update_data.get("factsToRemove") - normalized_facts_to_remove = [fact_id for fact_id in facts_to_remove if isinstance(fact_id, str)] if isinstance(facts_to_remove, list) else [] + normalized_facts_to_remove: list[dict[str, Any]] = [] + if isinstance(facts_to_remove, list): + for entry in facts_to_remove: + # Preserve the legacy string form as an unclassified removal. The + # apply-layer gate will reject it as missing instead of continuing + # to allow an unscoped destructive mutation. + if isinstance(entry, str): + fact_id = entry.strip() + if fact_id: + normalized_facts_to_remove.append({"id": fact_id}) + continue + if not isinstance(entry, dict): + continue + raw_id = entry.get("id") + if not isinstance(raw_id, str) or not raw_id.strip(): + continue + normalized_removal: dict[str, Any] = {"id": raw_id.strip()} + scope = _normalize_gate_label(entry.get("scope")) + if scope is not None: + normalized_removal["scope"] = scope + reason = entry.get("reason") + if isinstance(reason, str) and reason.strip(): + normalized_removal["reason"] = reason.strip() + if "replacementFactIndex" in entry: + # Preserve invalid values too: the apply layer must reject an + # invalid dependency rather than silently treating it as a pure + # removal and deleting the old fact. + normalized_removal["replacementFactIndex"] = entry.get("replacementFactIndex") + normalized_facts_to_remove.append(normalized_removal) normalized_new_facts = [] dropped_new_fact = not isinstance(new_facts, list) if isinstance(new_facts, list): @@ -290,6 +372,7 @@ def _normalize_memory_update_data(update_data: dict[str, Any]) -> dict[str, Any] "content": content.strip(), "category": _norm_cat, "confidence": _norm_conf, + **{field: normalized for field in _FACT_CLASSIFICATION_FIELDS if (normalized := _normalize_gate_label(consolidated.get(field))) is not None}, }, } ) @@ -1021,25 +1104,24 @@ class MemoryUpdater: if "correction" in signals: hints.append( "IMPORTANT: Explicit correction signals were detected in this conversation. " - "Pay special attention to what the agent got wrong, what the user corrected, " - "and record the correct approach as a fact with category " - '"correction" and confidence >= 0.95 when appropriate.' + "Record a correction with confidence >= 0.95 only when it describes a durable, user-level " + "working preference that is safe to reuse across unrelated tasks. A correction to facts, files, " + "directions, or constraints in the current task is thread- or project-scoped and must not be stored." ) if "reinforcement" in signals: hints.append( "IMPORTANT: Positive reinforcement signals were detected in this conversation. " - "The user explicitly confirmed the agent's approach was correct or helpful. " - "Record the confirmed approach, style, or preference as a fact with category " - '"preference" or "behavior" and confidence >= 0.9 when appropriate.' + "Record the confirmed approach, style, or preference with high confidence only if it is a durable, " + "user-level pattern. Approval of the current result or current task is thread-scoped and must not be stored." ) if "preference" in signals: - hints.append('IMPORTANT: A preference signal was detected. Record the user\'s stated preference or dislike as a fact with category "preference" and high confidence.') + hints.append("IMPORTANT: A preference signal was detected. Record it with high confidence only when it is a durable, user-level preference; a one-off choice for the current task is thread-scoped and must not be stored.") if "identity" in signals: - hints.append('IMPORTANT: An identity signal was detected. Record the user\'s stated role, profession, or background as a fact with category "identity" and high confidence.') + hints.append("IMPORTANT: An identity signal was detected. Record the user's stated role, profession, or background only when it is user-level and durable across tasks.") if "goal" in signals: - hints.append('IMPORTANT: A goal signal was detected. Record the user\'s stated objective or intent as a fact with category "goal" and high confidence.') + hints.append("IMPORTANT: A goal signal was detected. Record only a durable, user-level goal that remains useful across unrelated tasks; the objective of the current task, sprint, PR, or thread must not be stored.") if "decision" in signals: - hints.append('IMPORTANT: A decision signal was detected. Record the user\'s decision or chosen option as a fact with category "decision" and high confidence.') + hints.append("IMPORTANT: A decision signal was detected. Record only a durable, user-level decision or working pattern; a choice made for the current task, file, PR, or thread must not be stored.") return "\n".join(hints) def _prepare_update_prompt( @@ -1534,22 +1616,34 @@ class MemoryUpdater: update_data: Updates from LLM. thread_id: Optional thread ID for tracking. metrics: Optional observability dict. When provided, populated with - ``facts_passed_confidence`` / ``rejected_low_confidence`` counted - at the real confidence-filter site below (the only acceptance - gate for new facts), so the metric cannot drift from the actual - filter the way a re-derived count in the caller could. + confidence and scope-gate counters counted at their real filter + sites, so observability cannot drift from actual acceptance. Returns: Updated memory data. """ config = self._config now = utc_now_iso_z() + scope_gate_rejections: dict[str, dict[str, int]] = { + "facts": {"missing": 0, "scope": 0, "durability": 0, "authority": 0}, + "summaries": {"missing": 0, "scope": 0, "authority": 0}, + "removals": {"missing": 0, "scope": 0, "replacement": 0}, + "consolidations": {"missing": 0, "scope": 0, "durability": 0, "authority": 0}, + } + + def reject_by_scope_gate(kind: str, reason: str) -> None: + scope_gate_rejections[kind][reason] += 1 # Update user sections user_updates = update_data.get("user", {}) for section in ["workContext", "personalContext", "topOfMind"]: section_data = user_updates.get(section, {}) - if section_data.get("shouldUpdate") and section_data.get("summary"): + if not isinstance(section_data, dict) or not section_data.get("shouldUpdate") or not section_data.get("summary"): + continue + rejection_reason = _summary_scope_gate_reason(section_data) + if rejection_reason is not None: + reject_by_scope_gate("summaries", rejection_reason) + else: current_memory["user"][section] = { "summary": section_data["summary"], "updatedAt": now, @@ -1559,17 +1653,17 @@ class MemoryUpdater: history_updates = update_data.get("history", {}) for section in ["recentMonths", "earlierContext", "longTermBackground"]: section_data = history_updates.get(section, {}) - if section_data.get("shouldUpdate") and section_data.get("summary"): + if not isinstance(section_data, dict) or not section_data.get("shouldUpdate") or not section_data.get("summary"): + continue + rejection_reason = _summary_scope_gate_reason(section_data) + if rejection_reason is not None: + reject_by_scope_gate("summaries", rejection_reason) + else: current_memory["history"][section] = { "summary": section_data["summary"], "updatedAt": now, } - # Remove facts (contradiction-based) - facts_to_remove = set(update_data.get("factsToRemove", [])) - if facts_to_remove: - current_memory["facts"] = [f for f in current_memory.get("facts", []) if f.get("id") not in facts_to_remove] - # ── Staleness review: removals + lifetime extensions ── # Both operations share one staleness-candidate guardrail pass and one # candidate_ids set. proposed_remove_ids is hoisted out of the removals @@ -1672,63 +1766,103 @@ class MemoryUpdater: # Creation-time lifetime cap shared with the consolidation path below, so # both fact-creation sites apply the identical bound in one place. creation_cap = int(config.staleness_age_days * config.staleness_max_lifetime_multiplier) - # Counted at the confidence-gate site (the only real accept filter for new - # facts) so the ``facts_passed_confidence`` metric mirrors the actual - # filter and cannot drift from it. Facts below the threshold are the - # reject count; duplicate / empty / over-cap facts that pass the - # threshold are still counted here -- the metric is a confidence-gate - # signal (the host's rejection-rate warning monitors confidence - # filtering, not dedup / over-cap), not a persisted-fact count. + # Two independent accept filters govern new facts: the deterministic + # scope gate and this confidence threshold. Each is counted at its own + # filter site so neither metric can drift from the filter it reports: + # ``facts_passed_confidence`` counts threshold-passers even when the + # scope gate rejects them, and the scope-gate counters increment + # whether or not the confidence check passes. Duplicate / empty / + # over-cap facts that pass the threshold are still counted here -- the + # metric is a confidence-gate signal (the host's rejection-rate + # warning monitors confidence filtering, not dedup / over-cap), not a + # persisted-fact count. passed_threshold = 0 - for fact in new_facts: + replacement_fact_keys: dict[int, str] = {} + for fact_index, fact in enumerate(new_facts): confidence = fact.get("confidence", 0.5) if confidence >= config.fact_confidence_threshold: passed_threshold += 1 - raw_content = fact.get("content", "") - if not isinstance(raw_content, str): - continue - normalized_content = raw_content.strip() - fact_key = _fact_content_key(normalized_content) - if fact_key is None: - # Empty / whitespace-only content: skip it the same way the - # non-string guard above does, instead of appending a blank - # fact that violates the non-empty-content invariant. - continue - if fact_key in existing_fact_keys: - continue + rejection_reason = _fact_scope_gate_reason(fact) + if rejection_reason is not None: + reject_by_scope_gate("facts", rejection_reason) + continue + if confidence < config.fact_confidence_threshold: + continue + raw_content = fact.get("content", "") + if not isinstance(raw_content, str): + continue + normalized_content = raw_content.strip() + fact_key = _fact_content_key(normalized_content) + if fact_key is None: + # Empty / whitespace-only content: skip it the same way the + # non-string guard above does, instead of appending a blank + # fact that violates the non-empty-content invariant. + continue + # Remember every eligible replacement's content key even when it is + # already present. A paired removal is safe only if the post-trim + # memory contains this content under an ID other than its target. + replacement_fact_keys[fact_index] = fact_key + if fact_key in existing_fact_keys: + continue - fact_entry = { - "id": f"fact_{uuid.uuid4().hex[:8]}", - "content": normalized_content, - "category": fact.get("category", "context"), - "confidence": confidence, - "createdAt": now, - "source": thread_id or "unknown", - } - source_error = fact.get("sourceError") - if isinstance(source_error, str): - normalized_source_error = source_error.strip() - if normalized_source_error: - fact_entry["sourceError"] = normalized_source_error - evd = _read_expected_valid_days(fact) - if evd is not None: - # Apply the creation-time cap so the LLM cannot assign an - # unbounded lifetime that defers staleness review indefinitely. - # Extensions (staleFactsToExtend) bypass this cap via their own - # staleness_max_extension_days ceiling because they represent a - # deliberate review decision, not an unchecked initial assignment. - fact_entry["expected_valid_days"] = min(evd, creation_cap) - current_memory["facts"].append(fact_entry) - if fact_key is not None: - existing_fact_keys.add(fact_key) - - if metrics is not None: - metrics["facts_passed_confidence"] = passed_threshold - metrics["rejected_low_confidence"] = len(new_facts) - passed_threshold + fact_entry = { + "id": f"fact_{uuid.uuid4().hex[:8]}", + "content": normalized_content, + "category": fact.get("category", "context"), + "confidence": confidence, + "createdAt": now, + "source": thread_id or "unknown", + } + source_error = fact.get("sourceError") + if isinstance(source_error, str): + normalized_source_error = source_error.strip() + if normalized_source_error: + fact_entry["sourceError"] = normalized_source_error + evd = _read_expected_valid_days(fact) + if evd is not None: + # Apply the creation-time cap so the LLM cannot assign an + # unbounded lifetime that defers staleness review indefinitely. + # Extensions (staleFactsToExtend) bypass this cap via their own + # staleness_max_extension_days ceiling because they represent a + # deliberate review decision, not an unchecked initial assignment. + fact_entry["expected_valid_days"] = min(evd, creation_cap) + current_memory["facts"].append(fact_entry) + existing_fact_keys.add(fact_key) # Enforce max facts limit (coerced confidence -- see _trim_facts_to_max). current_memory["facts"] = _trim_facts_to_max(current_memory["facts"], config.max_facts) + # Remove contradicted facts only after replacements have passed both + # gates and survived deduplication/trimming. Task-local contradictions + # cannot delete user memory, and a failed paired replacement cannot + # degrade into a delete-only update. + fact_ids_to_remove: set[str] = set() + for removal in update_data.get("factsToRemove", []): + if not isinstance(removal, dict): + reject_by_scope_gate("removals", "missing") + continue + rejection_reason = _removal_scope_gate_reason(removal) + if rejection_reason is not None: + reject_by_scope_gate("removals", rejection_reason) + continue + fact_id = removal.get("id") + if not isinstance(fact_id, str) or not fact_id: + reject_by_scope_gate("removals", "missing") + continue + if "replacementFactIndex" in removal: + replacement_index = removal.get("replacementFactIndex") + if not isinstance(replacement_index, int) or isinstance(replacement_index, bool) or replacement_index < 0: + reject_by_scope_gate("removals", "replacement") + continue + replacement_key = replacement_fact_keys.get(replacement_index) + if replacement_key is None or not any(fact.get("id") != fact_id and _fact_content_key(fact.get("content")) == replacement_key for fact in current_memory.get("facts", [])): + reject_by_scope_gate("removals", "replacement") + continue + fact_ids_to_remove.add(fact_id) + + if fact_ids_to_remove: + current_memory["facts"] = [fact for fact in current_memory.get("facts", []) if fact.get("id") not in fact_ids_to_remove] + # ── Memory consolidation ── # Runs after the max_facts trim so source facts that were just evicted # (low confidence, pushed out by high-confidence newFacts) are absent @@ -1789,6 +1923,10 @@ class MemoryUpdater: content = consolidated.get("content", "") if not isinstance(content, str) or not content.strip(): continue + rejection_reason = _fact_scope_gate_reason(consolidated) + if rejection_reason is not None: + reject_by_scope_gate("consolidations", rejection_reason) + continue source_confidences = [_coerce_source_confidence(fact_index[sid]) for sid in source_ids] # _coerce_source_confidence already clamps each value to [0, 1], @@ -1899,4 +2037,11 @@ class MemoryUpdater: current_memory["facts"] = [f for f in current_memory.get("facts", []) if f.get("id") not in ids_consumed] current_memory["facts"].extend(new_consolidated) + if metrics is not None: + metrics["facts_passed_confidence"] = passed_threshold + metrics["rejected_low_confidence"] = len(new_facts) - passed_threshold + metrics["facts_passed_scope_gate"] = len(new_facts) - sum(scope_gate_rejections["facts"].values()) + metrics["rejected_by_scope_gate"] = sum(count for reasons in scope_gate_rejections.values() for count in reasons.values()) + metrics["scope_gate_rejections"] = scope_gate_rejections + return current_memory diff --git a/backend/packages/harness/deerflow/agents/memory/manager.py b/backend/packages/harness/deerflow/agents/memory/manager.py index 89f16a049..0335d6dbc 100644 --- a/backend/packages/harness/deerflow/agents/memory/manager.py +++ b/backend/packages/harness/deerflow/agents/memory/manager.py @@ -694,6 +694,8 @@ def _host_default_extraction_callback(payload: Any) -> None: extracted = payload.get("facts_extracted") passed_confidence = payload.get("facts_passed_confidence") rejected = payload.get("rejected_low_confidence", 0) + rejected_by_scope = payload.get("rejected_by_scope_gate", 0) + scope_breakdown = payload.get("scope_gate_rejections") thread_id = payload.get("thread_id") model_name = payload.get("model_name") if isinstance(extracted, int) and isinstance(passed_confidence, int) and extracted > 0: @@ -721,6 +723,22 @@ def _host_default_extraction_callback(payload: Any) -> None: payload.get("success"), payload.get("token_usage"), ) + if isinstance(scope_breakdown, dict): + logger.info( + "Memory scope-gate metrics: thread=%s model=%s rejected=%s breakdown=%s", + thread_id, + model_name, + rejected_by_scope, + scope_breakdown, + ) + fact_breakdown = scope_breakdown.get("facts") + fact_scope_rejected = sum(value for value in fact_breakdown.values() if isinstance(value, int)) if isinstance(fact_breakdown, dict) else 0 + if isinstance(extracted, int) and extracted > 0 and fact_scope_rejected / extracted > 0.6: + logger.warning( + "Memory fact scope-gate rejection rate %.0f%% exceeds 60%% - review extraction model classification / prompt (thread=%s)", + fact_scope_rejected / extracted * 100, + thread_id, + ) def _collect_host_hooks() -> dict[str, Any]: diff --git a/backend/tests/test_deermem_self_contained.py b/backend/tests/test_deermem_self_contained.py index 8fc3e68ee..2eddc49f0 100644 --- a/backend/tests/test_deermem_self_contained.py +++ b/backend/tests/test_deermem_self_contained.py @@ -240,7 +240,7 @@ def test_trace_id_threads_through_to_callbacks(deermem_data_dir): def test_default_passive_update_persists_fact_in_reserved_default_bucket(deermem_data_dir): - dm = _deermem_with_fake_llm(payload='{"user":{},"history":{},"newFacts":[{"content":"Default agent fact","category":"context","confidence":0.9}],"factsToRemove":[]}') + dm = _deermem_with_fake_llm(payload='{"user":{},"history":{},"newFacts":[{"content":"Default agent fact","category":"context","confidence":0.9,"scope":"user","durability":"durable","authority":"descriptive"}],"factsToRemove":[]}') dm.add( thread_id="default-thread", diff --git a/backend/tests/test_memory_consolidation.py b/backend/tests/test_memory_consolidation.py index a2c92f988..ad83f8d0c 100644 --- a/backend/tests/test_memory_consolidation.py +++ b/backend/tests/test_memory_consolidation.py @@ -35,6 +35,12 @@ from deerflow.agents.memory.backends.deermem.deermem.core.updater import ( # ── Helpers ──────────────────────────────────────────────────────────────── +_DURABLE_USER_CLASSIFICATION = { + "scope": "user", + "durability": "durable", + "authority": "descriptive", +} + def _memory_config(**overrides: object) -> DeerMemConfig: """Build a DeerMemConfig with test overrides (validation bypassed via setattr). @@ -247,6 +253,7 @@ class TestNormalizeFactsToConsolidate: { "sourceIds": ["fact_a", "fact_b"], "consolidated": { + **_DURABLE_USER_CLASSIFICATION, "content": "User is a full-stack engineer", "category": "knowledge", "confidence": 0.9, @@ -287,7 +294,7 @@ class TestNormalizeFactsToConsolidate: "factsToConsolidate": [ { "sourceIds": ["fact_only"], - "consolidated": {"content": "should be skipped", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "should be skipped", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -304,7 +311,7 @@ class TestNormalizeFactsToConsolidate: "factsToConsolidate": [ { "sourceIds": ["fact_a", "fact_b"], - "consolidated": {"content": " ", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": " ", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -359,6 +366,7 @@ class TestApplyUpdatesConsolidation: { "sourceIds": ["fact_a", "fact_b", "fact_c"], "consolidated": { + **_DURABLE_USER_CLASSIFICATION, "content": "Full-stack: React frontend, Python backend, PostgreSQL", "category": "knowledge", "confidence": 0.9, @@ -398,9 +406,9 @@ class TestApplyUpdatesConsolidation: "factsToRemove": [], "staleFactsToRemove": [], "factsToConsolidate": [ - {"sourceIds": ["f_0", "f_1"], "consolidated": {"content": "Group 1", "category": "knowledge", "confidence": 0.8}}, - {"sourceIds": ["f_2", "f_3"], "consolidated": {"content": "Group 2", "category": "knowledge", "confidence": 0.8}}, - {"sourceIds": ["f_4", "f_5"], "consolidated": {"content": "Group 3", "category": "knowledge", "confidence": 0.8}}, + {"sourceIds": ["f_0", "f_1"], "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Group 1", "category": "knowledge", "confidence": 0.8}}, + {"sourceIds": ["f_2", "f_3"], "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Group 2", "category": "knowledge", "confidence": 0.8}}, + {"sourceIds": ["f_4", "f_5"], "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Group 3", "category": "knowledge", "confidence": 0.8}}, ], } @@ -433,7 +441,7 @@ class TestApplyUpdatesConsolidation: "factsToConsolidate": [ { "sourceIds": ["fact_a", "fact_hallucinated"], - "consolidated": {"content": "Should not apply", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Should not apply", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -461,7 +469,7 @@ class TestApplyUpdatesConsolidation: "factsToConsolidate": [ { "sourceIds": [f"f_{i}" for i in range(10)], # 10 sources, cap is 5 - "consolidated": {"content": "Over-merged", "category": "knowledge", "confidence": 0.8}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Over-merged", "category": "knowledge", "confidence": 0.8}, }, ], } @@ -494,8 +502,8 @@ class TestApplyUpdatesConsolidation: "factsToRemove": [], "staleFactsToRemove": [], "factsToConsolidate": [ - {"sourceIds": ["fact_a", "fact_b"], "consolidated": {"content": "AB", "category": "knowledge", "confidence": 0.9}}, - {"sourceIds": ["fact_b", "fact_c"], "consolidated": {"content": "BC", "category": "knowledge", "confidence": 0.8}}, + {"sourceIds": ["fact_a", "fact_b"], "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "AB", "category": "knowledge", "confidence": 0.9}}, + {"sourceIds": ["fact_b", "fact_c"], "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "BC", "category": "knowledge", "confidence": 0.8}}, ], } @@ -529,10 +537,10 @@ class TestApplyUpdatesConsolidation: "user": {}, "history": {}, "newFacts": [], - "factsToRemove": ["fact_contradicted"], + "factsToRemove": [{"id": "fact_contradicted", "scope": "user", "reason": "Explicit contradiction in test fixture"}], "staleFactsToRemove": [{"id": "fact_stale", "reason": "outdated"}], "factsToConsolidate": [ - {"sourceIds": ["fact_a", "fact_b"], "consolidated": {"content": "React + Python", "category": "knowledge", "confidence": 0.9}}, + {"sourceIds": ["fact_a", "fact_b"], "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "React + Python", "category": "knowledge", "confidence": 0.9}}, ], } @@ -559,7 +567,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_a", "fact_a"], - "consolidated": {"content": "Rewritten", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Rewritten", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -596,11 +604,11 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_a", "fact_b"], - "consolidated": {"content": "Merged", "category": " knowledge ", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Merged", "category": " knowledge ", "confidence": 0.9}, }, { "sourceIds": ["fact_c", "fact_d"], - "consolidated": {"content": "Also merged", "category": " ", "confidence": 0.85}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Also merged", "category": " ", "confidence": 0.85}, }, ], } @@ -632,15 +640,15 @@ class TestReviewerFindings: "newFacts": [ # 2 high-confidence new facts that push us to max_facts=3, # forcing the trim to evict low_a and low_b - {"content": "New high 1", "category": "knowledge", "confidence": 0.98}, - {"content": "New high 2", "category": "knowledge", "confidence": 0.97}, + {**_DURABLE_USER_CLASSIFICATION, "content": "New high 1", "category": "knowledge", "confidence": 0.98}, + {**_DURABLE_USER_CLASSIFICATION, "content": "New high 2", "category": "knowledge", "confidence": 0.97}, ], "factsToRemove": [], "staleFactsToRemove": [], "factsToConsolidate": [ { "sourceIds": ["low_a", "low_b"], - "consolidated": {"content": "Merged low", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Merged low", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -680,7 +688,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_a", "fact_b"], - "consolidated": {"content": "Merged AB", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Merged AB", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -714,7 +722,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["corr_0", "corr_1"], - "consolidated": {"content": "Merged corrections", "category": "correction", "confidence": 0.95}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Merged corrections", "category": "correction", "confidence": 0.95}, }, ], } @@ -752,7 +760,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_a", "fact_b"], - "consolidated": {"content": "Merged", "category": "knowledge", "confidence": 1.0}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Merged", "category": "knowledge", "confidence": 1.0}, }, ], } @@ -777,7 +785,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_c", "fact_d"], - "consolidated": {"content": "Below threshold", "category": "knowledge", "confidence": 1.0}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Below threshold", "category": "knowledge", "confidence": 1.0}, }, ], } @@ -809,7 +817,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_a", "fact_b"], - "consolidated": {"content": "Should not merge", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Should not merge", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -852,7 +860,7 @@ class TestReviewerFindings: { "sourceIds": ["fact_null", "fact_b"], # LLM returns 1.0; cap = max(0.5, 0.9) = 0.9 - "consolidated": {"content": "Merged", "category": "knowledge", "confidence": 1.0}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Merged", "category": "knowledge", "confidence": 1.0}, }, ], } @@ -888,7 +896,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_old", "fact_new"], - "consolidated": {"content": "Old and new merged", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Old and new merged", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -935,7 +943,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_a", "fact_b"], - "consolidated": {"content": "A and B merged", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "A and B merged", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -975,7 +983,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_a", "fact_b"], - "consolidated": {"content": "merged", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "merged", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -1021,7 +1029,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_a", "fact_b"], - "consolidated": {"content": "merged", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "merged", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -1069,7 +1077,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_legacy", "fact_stable"], - "consolidated": {"content": "merged", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "merged", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -1128,7 +1136,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_stable", "fact_volatile"], - "consolidated": {"content": "merged", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "merged", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -1176,7 +1184,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_bad", "fact_stable"], - "consolidated": {"content": "merged", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "merged", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -1226,7 +1234,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_bad", "fact_stable"], - "consolidated": {"content": "merged", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "merged", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -1273,7 +1281,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_old", "fact_fresh"], - "consolidated": {"content": "merged", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "merged", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -1316,7 +1324,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_stable", "fact_volatile"], - "consolidated": {"content": "merged", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "merged", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -1358,7 +1366,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_a", "fact_b"], - "consolidated": {"content": "merged", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "merged", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -1400,7 +1408,7 @@ class TestReviewerFindings: "factsToConsolidate": [ { "sourceIds": ["fact_a", "fact_b"], - "consolidated": {"content": "merged stable skill", "category": "knowledge", "confidence": 0.9}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "merged stable skill", "category": "knowledge", "confidence": 0.9}, }, ], } @@ -1440,7 +1448,7 @@ class TestReviewerFindings: { "sourceIds": ["fact_a", "fact_b"], # LLM omits the confidence field entirely - "consolidated": {"content": "Merged without confidence", "category": "knowledge"}, + "consolidated": {**_DURABLE_USER_CLASSIFICATION, "content": "Merged without confidence", "category": "knowledge"}, }, ], } diff --git a/backend/tests/test_memory_scope_gate.py b/backend/tests/test_memory_scope_gate.py new file mode 100644 index 000000000..f92a46c4e --- /dev/null +++ b/backend/tests/test_memory_scope_gate.py @@ -0,0 +1,411 @@ +import copy +import logging +from unittest.mock import MagicMock + +from deerflow.agents.memory.backends.deermem.deermem.config import DeerMemConfig +from deerflow.agents.memory.backends.deermem.deermem.core.storage import MemoryStorage +from deerflow.agents.memory.backends.deermem.deermem.core.updater import MemoryUpdater, _extract_text, _normalize_memory_update_data +from deerflow.agents.memory.manager import _host_default_extraction_callback + + +def _memory(facts: list[dict[str, object]] | None = None) -> dict[str, object]: + return { + "version": "1.0", + "revision": 0, + "lastUpdated": "", + "user": { + "workContext": {"summary": "", "updatedAt": ""}, + "personalContext": {"summary": "", "updatedAt": ""}, + "topOfMind": {"summary": "", "updatedAt": ""}, + }, + "history": { + "recentMonths": {"summary": "", "updatedAt": ""}, + "earlierContext": {"summary": "", "updatedAt": ""}, + "longTermBackground": {"summary": "", "updatedAt": ""}, + }, + "facts": copy.deepcopy(facts or []), + } + + +class _Storage(MemoryStorage): + def load(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, object]: + return _memory() + + def reload(self, agent_name: str | None = None, *, user_id: str | None = None) -> dict[str, object]: + return self.load(agent_name, user_id=user_id) + + def save( + self, + memory_data: dict[str, object], + agent_name: str | None = None, + *, + user_id: str | None = None, + expected_revision: int | None = None, + ) -> bool: + return True + + +def _updater(**config_overrides: object) -> MemoryUpdater: + config = DeerMemConfig() + for key, value in config_overrides.items(): + setattr(config, key, value) + return MemoryUpdater(config, _Storage(), llm=None) + + +def _fact(content: str, **overrides: object) -> dict[str, object]: + fact: dict[str, object] = { + "content": content, + "category": "preference", + "confidence": 0.9, + "scope": "user", + "durability": "durable", + "authority": "descriptive", + } + fact.update(overrides) + return fact + + +def _stored_fact(fact_id: str, content: str) -> dict[str, object]: + return { + "id": fact_id, + "content": content, + "category": "preference", + "confidence": 0.9, + "createdAt": "2026-01-01T00:00:00Z", + "source": "thread-old", + } + + +def test_normalize_preserves_extraction_only_classification_fields() -> None: + normalized = _normalize_memory_update_data( + { + "user": {}, + "history": {}, + "newFacts": [_fact("User prefers concise answers")], + "factsToRemove": [], + } + ) + + assert normalized["newFacts"][0]["scope"] == "user" + assert normalized["newFacts"][0]["durability"] == "durable" + assert normalized["newFacts"][0]["authority"] == "descriptive" + + +def test_fact_gate_accepts_only_durable_descriptive_user_facts() -> None: + updater = _updater(fact_confidence_threshold=0.7) + metrics: dict[str, object] = {} + update = { + "user": {}, + "history": {}, + "factsToRemove": [], + "newFacts": [ + _fact("accepted"), + _fact("thread only", scope="thread"), + _fact("temporary", durability="temporary"), + _fact("permission", authority="transactional"), + {"content": "missing labels", "category": "context", "confidence": 0.9}, + ], + } + + result = updater._apply_updates(_memory(), update, thread_id="thread-new", metrics=metrics) + + assert [fact["content"] for fact in result["facts"]] == ["accepted"] + assert set(result["facts"][0]) == {"id", "content", "category", "confidence", "createdAt", "source"} + assert metrics["rejected_by_scope_gate"] == 4 + assert metrics["scope_gate_rejections"] == { + "facts": {"missing": 1, "scope": 1, "durability": 1, "authority": 1}, + "summaries": {"missing": 0, "scope": 0, "authority": 0}, + "removals": {"missing": 0, "scope": 0, "replacement": 0}, + "consolidations": {"missing": 0, "scope": 0, "durability": 0, "authority": 0}, + } + + +def test_summary_gate_requires_user_scope_and_descriptive_authority() -> None: + updater = _updater() + current = _memory() + current["user"]["personalContext"]["summary"] = "Existing summary" + metrics: dict[str, object] = {} + update = { + "user": { + "workContext": {"summary": "User is a software engineer", "shouldUpdate": True, "scope": "user", "authority": "descriptive"}, + "personalContext": {"summary": "Constraint for this PR", "shouldUpdate": True, "scope": "project", "authority": "descriptive"}, + "topOfMind": {"summary": "User granted push access", "shouldUpdate": True, "scope": "user", "authority": "transactional"}, + }, + "history": { + "recentMonths": {"summary": "Missing authority label", "shouldUpdate": True, "scope": "user"}, + }, + "newFacts": [], + "factsToRemove": [], + } + + result = updater._apply_updates(current, update, metrics=metrics) + + assert result["user"]["workContext"]["summary"] == "User is a software engineer" + assert set(result["user"]["workContext"]) == {"summary", "updatedAt"} + assert result["user"]["personalContext"]["summary"] == "Existing summary" + assert result["user"]["topOfMind"]["summary"] == "" + assert result["history"]["recentMonths"]["summary"] == "" + assert metrics["scope_gate_rejections"]["summaries"] == {"missing": 1, "scope": 1, "authority": 1} + + +def test_thread_scoped_removal_cannot_delete_user_fact() -> None: + updater = _updater() + current = _memory([_stored_fact("fact_api", "User generally prefers API compatibility")]) + update = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [{"id": "fact_api", "scope": "thread", "reason": "This PR may break the API"}], + } + + result = updater._apply_updates(current, update) + + assert [fact["id"] for fact in result["facts"]] == ["fact_api"] + + +def test_unreasoned_user_scoped_removal_fails_closed() -> None: + updater = _updater() + current = _memory([_stored_fact("fact_api", "User generally prefers API compatibility")]) + update = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [{"id": "fact_api", "scope": "user"}], + } + + result = updater._apply_updates(current, update) + + assert [fact["id"] for fact in result["facts"]] == ["fact_api"] + + +def test_paired_removal_is_skipped_when_replacement_fails_scope_gate() -> None: + updater = _updater(fact_confidence_threshold=0.7) + current = _memory([_stored_fact("fact_api", "User generally prefers API compatibility")]) + metrics: dict[str, object] = {} + update = { + "user": {}, + "history": {}, + "newFacts": [_fact("This PR may break the API", scope="thread", durability="temporary")], + "factsToRemove": [ + { + "id": "fact_api", + "scope": "user", + "reason": "Preference changed", + "replacementFactIndex": 0, + } + ], + } + + result = updater._apply_updates(current, update, metrics=metrics) + + assert [fact["id"] for fact in result["facts"]] == ["fact_api"] + assert metrics["scope_gate_rejections"]["removals"]["replacement"] == 1 + + +def test_paired_removal_is_skipped_when_replacement_fails_confidence_gate() -> None: + updater = _updater(fact_confidence_threshold=0.7) + current = _memory([_stored_fact("fact_api", "User generally prefers API compatibility")]) + metrics: dict[str, object] = {} + update = { + "user": {}, + "history": {}, + "newFacts": [_fact("User no longer requires API compatibility", confidence=0.69)], + "factsToRemove": [ + { + "id": "fact_api", + "scope": "user", + "reason": "Preference changed", + "replacementFactIndex": 0, + } + ], + } + + result = updater._apply_updates(current, update, metrics=metrics) + + assert [fact["id"] for fact in result["facts"]] == ["fact_api"] + assert metrics["facts_passed_scope_gate"] == 1 + assert metrics["rejected_low_confidence"] == 1 + assert metrics["scope_gate_rejections"]["removals"]["replacement"] == 1 + + +def test_paired_removal_is_atomic_when_replacement_is_persisted() -> None: + updater = _updater(fact_confidence_threshold=0.7, max_facts=100) + current = _memory([_stored_fact("fact_editor", "User prefers Vim")]) + update = { + "user": {}, + "history": {}, + "newFacts": [_fact("User now prefers VS Code")], + "factsToRemove": [ + { + "id": "fact_editor", + "scope": "user", + "reason": "User changed their durable editor preference", + "replacementFactIndex": 0, + } + ], + } + + result = updater._apply_updates(current, update) + + assert [fact["content"] for fact in result["facts"]] == ["User now prefers VS Code"] + + +def test_paired_removal_is_skipped_when_replacement_is_trimmed() -> None: + updater = _updater(fact_confidence_threshold=0.7, max_facts=1) + current = _memory([_stored_fact("fact_editor", "User prefers Vim")]) + update = { + "user": {}, + "history": {}, + "newFacts": [_fact("User now prefers VS Code", confidence=0.8)], + "factsToRemove": [ + { + "id": "fact_editor", + "scope": "user", + "reason": "User changed their durable editor preference", + "replacementFactIndex": 0, + } + ], + } + + result = updater._apply_updates(current, update) + + assert [fact["content"] for fact in result["facts"]] == ["User prefers Vim"] + + +def test_missing_classification_rejects_one_fact_without_aborting_other_updates() -> None: + updater = _updater(fact_confidence_threshold=0.7, max_facts=100) + current = _memory([_stored_fact("fact_old", "Old durable preference")]) + normalized = _normalize_memory_update_data( + { + "user": {}, + "history": {}, + "newFacts": [ + _fact("Accepted durable preference"), + {"content": "Unclassified replacement", "category": "preference", "confidence": 0.9}, + ], + "factsToRemove": [ + { + "id": "fact_old", + "scope": "user", + "reason": "Replace it", + "replacementFactIndex": 1, + } + ], + } + ) + + result = updater._apply_updates(current, normalized) + + assert {fact["content"] for fact in result["facts"]} == { + "Old durable preference", + "Accepted durable preference", + } + + +def test_legacy_string_removal_fails_closed_and_reports_missing_scope() -> None: + updater = _updater() + current = _memory([_stored_fact("fact_api", "User generally prefers API compatibility")]) + metrics: dict[str, object] = {} + normalized = _normalize_memory_update_data( + { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": ["fact_api"], + } + ) + + result = updater._apply_updates(current, normalized, metrics=metrics) + + assert normalized["factsToRemove"] == [{"id": "fact_api"}] + assert [fact["id"] for fact in result["facts"]] == ["fact_api"] + assert metrics["scope_gate_rejections"]["removals"]["missing"] == 1 + + +def test_signal_hints_require_cross_task_user_scope() -> None: + updater = _updater() + + hints = updater._build_signal_hints(frozenset({"correction", "goal", "decision"})) + + assert "user-level" in hints + assert "current task" in hints + assert "thread" in hints + + +def test_prompt_requires_scope_labels_for_every_mutating_path() -> None: + updater = _updater() + message = MagicMock() + message.type = "human" + message.content = "Remember that I prefer concise answers." + + prepared = updater._prepare_update_prompt([message], agent_name="lead-agent", signals=frozenset()) + + assert prepared is not None + _, prompt = prepared + prompt_text = "\n".join(_extract_text(getattr(item, "content", item)) for item in prompt) + assert 'scope="user"' in prompt_text + assert 'durability="durable"' in prompt_text + assert 'authority="transactional"' in prompt_text + assert '"scope": "user|thread|project", "authority": "descriptive|transactional"' in prompt_text + assert "replacementFactIndex" in prompt_text + assert "unrelated future thread" in prompt_text + + +def test_unclassified_consolidation_keeps_sources_and_reports_rejection() -> None: + updater = _updater( + consolidation_enabled=True, + consolidation_min_facts=2, + consolidation_max_groups_per_cycle=1, + consolidation_max_sources=4, + ) + current = _memory( + [ + _stored_fact("fact_a", "User uses Python"), + _stored_fact("fact_b", "User uses Rust"), + ] + ) + metrics: dict[str, object] = {} + update = { + "user": {}, + "history": {}, + "newFacts": [], + "factsToRemove": [], + "factsToConsolidate": [ + { + "sourceIds": ["fact_a", "fact_b"], + "consolidated": { + "content": "User uses Python and Rust", + "category": "preference", + "confidence": 0.9, + }, + } + ], + } + + result = updater._apply_updates(current, update, metrics=metrics) + + assert {fact["id"] for fact in result["facts"]} == {"fact_a", "fact_b"} + assert metrics["scope_gate_rejections"]["consolidations"]["missing"] == 1 + + +def test_default_observability_warns_when_fact_scope_gate_rejects_most_items(caplog) -> None: + payload = { + "thread_id": "thread-scope", + "model_name": "test-model", + "success": True, + "facts_extracted": 2, + "facts_passed_confidence": 2, + "rejected_low_confidence": 0, + "rejected_by_scope_gate": 2, + "scope_gate_rejections": { + "facts": {"missing": 2, "scope": 0, "durability": 0, "authority": 0}, + "summaries": {"missing": 0, "scope": 0, "authority": 0}, + "removals": {"missing": 0, "scope": 0, "replacement": 0}, + "consolidations": {"missing": 0, "scope": 0, "durability": 0, "authority": 0}, + }, + } + + with caplog.at_level(logging.WARNING): + _host_default_extraction_callback(payload) + + assert "scope-gate rejection rate 100% exceeds 60%" in caplog.text diff --git a/backend/tests/test_memory_staleness_review.py b/backend/tests/test_memory_staleness_review.py index 1fcabc59e..1fe50a0ff 100644 --- a/backend/tests/test_memory_staleness_review.py +++ b/backend/tests/test_memory_staleness_review.py @@ -33,6 +33,12 @@ from deerflow.agents.memory.backends.deermem.deermem.core.updater import ( # ── Helpers ──────────────────────────────────────────────────────────────── +_DURABLE_USER_FACT = { + "scope": "user", + "durability": "durable", + "authority": "descriptive", +} + def _memory_config(**overrides: object) -> DeerMemConfig: config = DeerMemConfig() @@ -569,7 +575,7 @@ class TestApplyUpdatesStaleness: "user": {}, "history": {}, "newFacts": [], - "factsToRemove": ["fact_contradicted"], + "factsToRemove": [{"id": "fact_contradicted", "scope": "user", "reason": "Explicit contradiction in test fixture"}], "staleFactsToRemove": [{"id": "fact_stale", "reason": "old"}], } @@ -924,7 +930,7 @@ class TestNewFactsExpectedValidDays: update_data = { "user": {}, "history": {}, - "newFacts": [{"content": "User speaks Spanish natively", "category": "knowledge", "confidence": 0.95, "expected_valid_days": 180}], + "newFacts": [{**_DURABLE_USER_FACT, "content": "User speaks Spanish natively", "category": "knowledge", "confidence": 0.95, "expected_valid_days": 180}], "factsToRemove": [], } @@ -947,7 +953,7 @@ class TestNewFactsExpectedValidDays: update_data = { "user": {}, "history": {}, - "newFacts": [{"content": "User prefers Python", "category": "knowledge", "confidence": 0.9, "expected_valid_days": 3650}], + "newFacts": [{**_DURABLE_USER_FACT, "content": "User prefers Python", "category": "knowledge", "confidence": 0.9, "expected_valid_days": 3650}], "factsToRemove": [], } @@ -962,7 +968,7 @@ class TestNewFactsExpectedValidDays: update_data = { "user": {}, "history": {}, - "newFacts": [{"content": "User uses Python", "category": "knowledge", "confidence": 0.9}], + "newFacts": [{**_DURABLE_USER_FACT, "content": "User uses Python", "category": "knowledge", "confidence": 0.9}], "factsToRemove": [], } diff --git a/backend/tests/test_memory_updater.py b/backend/tests/test_memory_updater.py index b6a41fb51..7746fd7a6 100644 --- a/backend/tests/test_memory_updater.py +++ b/backend/tests/test_memory_updater.py @@ -48,6 +48,13 @@ def _memory_config(**overrides: object) -> DeerMemConfig: return config +_DURABLE_USER_FACT = { + "scope": "user", + "durability": "durable", + "authority": "descriptive", +} + + class _MemoryStorage(MemoryStorage): def __init__(self, memory: dict[str, object] | None = None, *, save_result: bool = True): self.memory = copy.deepcopy(memory or _make_memory()) @@ -119,9 +126,9 @@ def test_apply_updates_skips_existing_duplicate_and_preserves_removals() -> None ] ) update_data = { - "factsToRemove": ["fact_remove"], + "factsToRemove": [{"id": "fact_remove", "scope": "user", "reason": "Explicit retraction in test fixture"}], "newFacts": [ - {"content": "User likes Python", "category": "preference", "confidence": 0.95}, + {**_DURABLE_USER_FACT, "content": "User likes Python", "category": "preference", "confidence": 0.95}, ], } @@ -136,8 +143,8 @@ def test_apply_updates_skips_whitespace_only_facts() -> None: current_memory = _make_memory() update_data = { "newFacts": [ - {"content": " ", "category": "context", "confidence": 0.9}, - {"content": "User prefers dark mode", "category": "preference", "confidence": 0.9}, + {**_DURABLE_USER_FACT, "content": " ", "category": "context", "confidence": 0.9}, + {**_DURABLE_USER_FACT, "content": "User prefers dark mode", "category": "preference", "confidence": 0.9}, ], } @@ -226,9 +233,9 @@ def test_apply_updates_skips_same_batch_duplicates_and_keeps_source_metadata() - current_memory = _make_memory() update_data = { "newFacts": [ - {"content": "User prefers dark mode", "category": "preference", "confidence": 0.91}, - {"content": "User prefers dark mode", "category": "preference", "confidence": 0.92}, - {"content": "User works on DeerFlow", "category": "context", "confidence": 0.87}, + {**_DURABLE_USER_FACT, "content": "User prefers dark mode", "category": "preference", "confidence": 0.91}, + {**_DURABLE_USER_FACT, "content": "User prefers dark mode", "category": "preference", "confidence": 0.92}, + {**_DURABLE_USER_FACT, "content": "User works on DeerFlow", "category": "context", "confidence": 0.87}, ], } @@ -266,9 +273,9 @@ def test_apply_updates_preserves_threshold_and_max_facts_trimming() -> None: ) update_data = { "newFacts": [ - {"content": "User prefers dark mode", "category": "preference", "confidence": 0.9}, - {"content": "User uses uv", "category": "context", "confidence": 0.85}, - {"content": "User likes noisy logs", "category": "behavior", "confidence": 0.6}, + {**_DURABLE_USER_FACT, "content": "User prefers dark mode", "category": "preference", "confidence": 0.9}, + {**_DURABLE_USER_FACT, "content": "User uses uv", "category": "context", "confidence": 0.85}, + {**_DURABLE_USER_FACT, "content": "User likes noisy logs", "category": "behavior", "confidence": 0.6}, ], } @@ -292,6 +299,7 @@ def test_apply_updates_preserves_source_error() -> None: "category": "correction", "confidence": 0.95, "sourceError": "The agent previously suggested npm start.", + **_DURABLE_USER_FACT, } ] } @@ -312,6 +320,7 @@ def test_apply_updates_ignores_empty_source_error() -> None: "category": "correction", "confidence": 0.95, "sourceError": " ", + **_DURABLE_USER_FACT, } ] } @@ -936,7 +945,9 @@ class TestUpdateMemoryStructuredResponse: def test_wrapped_json_responses_parse(self): """Memory update should tolerate provider wrappers around valid JSON.""" - valid_json = '{"user": {}, "history": {}, "newFacts": [{"content": "User prefers concise updates", "category": "preference", "confidence": 0.9}], "factsToRemove": []}' + valid_json = ( + '{"user": {}, "history": {}, "newFacts": [{"content": "User prefers concise updates", "category": "preference", "confidence": 0.9, "scope": "user", "durability": "durable", "authority": "descriptive"}], "factsToRemove": []}' + ) response_variants = [ f"Analyze the conversation first.\n{valid_json}", f"Analyze the conversation first.\n{valid_json}", @@ -953,7 +964,7 @@ class TestUpdateMemoryStructuredResponse: def test_ignores_unrelated_json_before_memory_update(self): """Parser should not select unrelated JSON objects before the memory update.""" - valid_json = '{"user": {}, "history": {}, "newFacts": [{"content": "Remember the actual update", "category": "context", "confidence": 0.9}], "factsToRemove": []}' + valid_json = '{"user": {}, "history": {}, "newFacts": [{"content": "Remember the actual update", "category": "context", "confidence": 0.9, "scope": "user", "durability": "durable", "authority": "descriptive"}], "factsToRemove": []}' response = f'Example object: {{"user": "alice"}}\nActual memory update:\n{valid_json}' result, storage = self._run_update_with_response(response) @@ -970,7 +981,11 @@ class TestUpdateMemoryStructuredResponse: def test_schema_guard_ignores_invalid_update_fields(self): """Parsed JSON with bad field types should not break the memory update.""" - response = '{"user": "bad", "history": [], "newFacts": ["bad", {"content": "User works on DeerFlow", "category": "context", "confidence": 0.91}], "factsToRemove": "bad"}' + response = ( + '{"user": "bad", "history": [], "newFacts": ["bad", ' + '{"content": "User works on DeerFlow", "category": "context", "confidence": 0.91, ' + '"scope": "user", "durability": "durable", "authority": "descriptive"}], "factsToRemove": "bad"}' + ) result, storage = self._run_update_with_response(response) @@ -981,7 +996,7 @@ class TestUpdateMemoryStructuredResponse: """Malformed fact entries should be normalized per fact, not fail the whole update.""" response = ( '{"user": {}, "history": {}, "newFacts": [' - '{"content": " User likes async updates ", "category": 9, "confidence": "0.91", "sourceError": " parse issue "}, ' + '{"content": " User likes async updates ", "category": 9, "confidence": "0.91", "sourceError": " parse issue ", "scope": "user", "durability": "durable", "authority": "descriptive"}, ' '{"content": "skip invalid confidence", "category": "context", "confidence": "high"}, ' '{"content": 12, "category": "context", "confidence": 0.9}, ' '{"content": " ", "category": "context", "confidence": 0.9}' @@ -1175,7 +1190,7 @@ class TestFactDeduplicationCaseInsensitive: update_data = { "factsToRemove": [], "newFacts": [ - {"content": "user prefers python", "category": "preference", "confidence": 0.95}, + {**_DURABLE_USER_FACT, "content": "user prefers python", "category": "preference", "confidence": 0.95}, ], } @@ -1202,7 +1217,7 @@ class TestFactDeduplicationCaseInsensitive: update_data = { "factsToRemove": [], "newFacts": [ - {"content": "User prefers Go", "category": "preference", "confidence": 0.85}, + {**_DURABLE_USER_FACT, "content": "User prefers Go", "category": "preference", "confidence": 0.85}, ], } @@ -1293,7 +1308,7 @@ class TestFinalizeCacheIsolation: { "user": {}, "history": {}, - "newFacts": [{"content": "new fact", "category": "context", "confidence": 0.9}], + "newFacts": [{**_DURABLE_USER_FACT, "content": "new fact", "category": "context", "confidence": 0.9}], "factsToRemove": [], } ) From 8234370a6ad2abd83bdba35271b902aa0bd19d47 Mon Sep 17 00:00:00 2001 From: qin-chenghan Date: Sat, 1 Aug 2026 09:15:21 +0800 Subject: [PATCH 13/31] feat(artifacts): inline editing for text artifacts in the panel (#4596) * feat(artifacts): inline editing for text artifacts in the panel Add a PUT /api/threads/{id}/artifacts/{path} endpoint that atomically replaces an existing UTF-8 text file under /mnt/user-data/outputs after verifying its SHA-256 revision. Active runs conflict (409); binary, symlink, oversized, and non-output paths are rejected. Frontend: edit/save/discard buttons, draft state with conflict detection, CodeEditor onChange/onSave, loader SHA-256 from ETag, i18n, beforeunload guard. Backend: PUT endpoint with thread reservation, atomic temp-file replacement, sandbox sync for non-mounted providers, rollback on failure, ETag on GET. Tests: 8 backend + 1 blocking-IO + 3 frontend test files. * fix(artifacts): scope replacement permissions and release sandboxes --------- Co-authored-by: Willem Jiang --- README.md | 2 +- backend/AGENTS.md | 4 +- backend/app/gateway/routers/artifacts.py | 196 ++++++++++- .../harness/deerflow/runtime/runs/schemas.py | 1 + .../blocking_io/test_artifacts_router.py | 36 +- backend/tests/test_artifacts_router.py | 262 +++++++++++++++ frontend/AGENTS.md | 1 + .../artifacts/artifact-file-detail.tsx | 311 ++++++++++++++++-- .../workspace/artifacts/context.tsx | 31 ++ .../src/components/workspace/code-editor.tsx | 17 +- frontend/src/core/artifacts/api.ts | 54 +++ frontend/src/core/artifacts/editing.ts | 67 ++++ frontend/src/core/artifacts/hooks.ts | 1 + frontend/src/core/artifacts/index.ts | 2 + frontend/src/core/artifacts/loader.ts | 18 +- frontend/src/core/i18n/locales/en-US.ts | 14 + frontend/src/core/i18n/locales/types.ts | 13 + frontend/src/core/i18n/locales/zh-CN.ts | 13 + .../tests/unit/core/artifacts/api.test.ts | 60 ++++ .../tests/unit/core/artifacts/editing.test.ts | 75 +++++ .../tests/unit/core/artifacts/loader.test.ts | 43 +++ 21 files changed, 1189 insertions(+), 32 deletions(-) create mode 100644 frontend/src/core/artifacts/api.ts create mode 100644 frontend/src/core/artifacts/editing.ts create mode 100644 frontend/tests/unit/core/artifacts/api.test.ts create mode 100644 frontend/tests/unit/core/artifacts/editing.test.ts create mode 100644 frontend/tests/unit/core/artifacts/loader.test.ts diff --git a/README.md b/README.md index 2ac58c9eb..273f34c7c 100644 --- a/README.md +++ b/README.md @@ -943,7 +943,7 @@ Image bytes loaded for a vision-model call are transient: DeerFlow removes the h After each run, DeerFlow records a workspace change summary for the run-owned `workspace` and `outputs` directories. The Web UI shows a compact "files changed" badge on the assistant turn; opening it reveals created, modified, and deleted files with text diffs when safe to display. Uploads are excluded because they are user inputs, not agent-generated changes. Large, binary, or sensitive-looking files are shown as metadata only. -Files presented through `present_files` remain part of the thread's artifact state, and the Web UI restores the artifact panel and selected document after a page refresh. The currently selected formal artifact is refreshed once when the run finishes so edits become visible without a manual reload. +Files presented through `present_files` remain part of the thread's artifact state, and the Web UI restores the artifact panel and selected document after a page refresh. The currently selected formal artifact is refreshed once when the run finishes so edits become visible without a manual reload. Existing UTF-8 text artifacts under `/mnt/user-data/outputs` can also be edited and explicitly saved from the panel while the thread is idle; saves use content revisions to prevent overwriting agent changes. With `AioSandboxProvider`, shell execution runs inside isolated containers. With `LocalSandboxProvider`, file tools still map to per-thread directories on the host, but host `bash` is disabled by default because it is not a secure isolation boundary. Re-enable host bash only for fully trusted local workflows. Host bash commands have a wall-clock timeout, and long-lived processes should be started in the background with output redirected to a workspace log. diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 988e56066..61a492ba5 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -492,7 +492,7 @@ Localhost persistence deliberately reads the direct request `Host` and ignores ` | **Memory** (`/api/memory`) | `GET /` - memory data; `POST /reload` - force reload; `GET /config` - config; `GET /status` - config + data | | **Uploads** (`/api/threads/{id}/uploads`) | `POST /` - upload files (auto-converts PDF/PPT/Excel/Word); `GET /list` - list; `DELETE /{filename}` - delete | | **Threads** (`/api/threads/{id}`) | `DELETE /` - remove DeerFlow-managed local thread data after LangGraph thread deletion; `POST /branches` - create a new main-thread branch from a completed assistant turn checkpoint and, when an addressable pre-user replay checkpoint exists, materialize it into the branch namespace so the inherited response remains regeneratable. Workspace files are not checkpointed, so the branch only best-effort copies the current workspace when branching from the **latest** turn (`workspace_clone_mode="current_thread_best_effort"`); branching from an older/historical turn skips the copy (`workspace_clone_mode="skipped_historical_turn"`) so the branch never inherits files that only exist in a later timeline. Thread-scoped runtime channels (`sandbox`, `thread_data`) are not copied onto the branch: the parent's `sandbox_id` binds path mappings and the release lifecycle to the parent's workspace, so the branch lazily acquires its own sandbox instead. Branch creation also seeds the new thread's run-event feed from the branch checkpoint's visible messages (`history_seed_mode` in the response): the thread feed reads run_events, not checkpoints, so without the seed the inherited history disappears from the UI after the branch's first run (#4380). Seeded rows are grouped into one synthetic run per inherited turn (`branch-seed-{thread_id}-{n}`, a new turn opening at every persisted human message, including an allowlisted hidden `ask_clarification` reply) because `run_id` is a turn identity to the feed's consumers, not a provenance tag: regenerating an inherited answer supersedes that row's whole `run_id` in `GET /messages/page`, so one shared id for the entire seed deleted the complete inherited history on a branch's first regenerate (#4458); `GET /goal`, `PUT /goal`, `DELETE /goal` - read, set, and clear the active thread goal; `POST /compact` - manually summarize older active context into `summary_text` and retain the recent message window, blocked while a run is in flight; unexpected failures are logged server-side and return a generic 500 detail | -| **Artifacts** (`/api/threads/{id}/artifacts`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types | +| **Artifacts** (`/api/threads/{id}/artifacts`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types. `PUT /{path}` atomically replaces an existing UTF-8 text file under `/mnt/user-data/outputs` when its expected SHA-256 still matches; active runs conflict, and non-mounted sandbox providers receive the same update explicitly. | | **Suggestions** (`/api/suggestions`) | `GET /config` - returns global suggestions config boolean; `POST /threads/{id}/suggestions` - generate follow-up questions; rich list/block model content is normalized and inline reasoning (`...`, including unclosed/truncated blocks from reasoning models like MiniMax-M3) is stripped before JSON parsing | | **Input Polish** (`/api/input-polish`) | `POST /` - rewrite a composer draft before it is sent. This is a short authenticated `runs:create` LLM request using `input_polish` config; it does not create a LangGraph run, persist a message, or modify thread state. Shares the non-graph one-shot LLM path (`deerflow.utils.oneshot_llm.run_oneshot_llm`) with the suggestions route so model build + Langfuse metadata + invoke stay in one place; validates the same stripped view of the draft it sends to the model, and preserves literal `` substrings in the rewrite (`strip_think_blocks(truncate_unclosed=False)`) | | **Thread Runs** (`/api/threads/{id}/runs`) | `POST /` - create background run; `POST /stream` - create + SSE stream; `POST /wait` - create + block. Before the first journaled run, an empty run-event message feed is seeded from an existing checkpoint head so legacy checkpoint-only history receives earlier thread-global sequence numbers and remains visible after the new run; a thread with no checkpoint or an already-populated feed skips this compatibility path. `POST /regenerate/prepare` - prepare clean input + checkpoint metadata for regenerating the latest completed or interrupted assistant answer, carrying the latest non-empty thread title in graph input so resuming an older checkpoint cannot roll back a later manual rename (#4457); `POST /edit-regenerate/prepare` - prepare a checkpoint replay from the latest editable human turn with a replacement user message and edit replay metadata; it carries the current thread title the same way, but only when the replay base already has one — an untitled base belongs to a thread the title middleware has not named yet, so pinning the current title there would keep a name generated from the prompt the edit just replaced; `GET /` - list runs; `GET /{rid}` - run details; `POST /{rid}/cancel` - cancel; `GET /{rid}/join` - join SSE; `GET /{rid}/messages` - paginated per-run messages `{data, has_more}`; `GET /{rid}/events` - full event stream; `GET /{rid}/workspace-changes` - workspace/output file change summary and optional diffs; `GET /../messages` - legacy thread message array; `GET /../messages/page` - backward thread-global `seq` history page with middleware/subagent-AI/successful-regenerate/edit-replay filtering and page-run-scoped feedback enrichment; subagent AI callbacks remain available through run events while parent `task` ToolMessages stay visible for card restoration; `GET /../token-usage` - aggregate tokens plus an optional `context_usage` percentage. Context usage approximately counts messages from the latest materialized thread state through `build_thread_checkpoint_state_accessor`, so full and delta checkpoint modes expose the same input. The percentage uses the latest run's model and its configured `context_window`. | @@ -567,7 +567,7 @@ JSONL event stores when `GATEWAY_WORKERS > 1`. - Store-only hydrated runs are readable history. In multi-worker mode with heartbeat enabled, cancel on a store-only run records `runs.cancel_action` / `cancel_requested_at` while the owner's lease is live; the first action wins even if a retry later lands on the owner. `RunStore.request_cancel()` and owner completion through `finalize_if_not_cancelled()` are competing active-row CAS operations, so an accepted cancel cannot be overwritten by a later success. `RunStore.renew_lease()` renews and observes the request atomically in the SQL implementation. The owner then executes the normal process-local interrupt/rollback and terminal stream path without transferring the lease. An expired owner is still taken over and marked `error`. `wait=true` and cancel-then-stream use the shared bridge to observe owner finalization; a non-standard process-local bridge returns accepted 202 instead of subscribing to an unreachable stream. In single-worker mode (heartbeat off), store-only runs still return 409. - A local worker's `RunRecord.lease_expires_at` is the last durably confirmed ownership deadline. `_renew_leases()` bounds each renewal attempt by that deadline: transient store exceptions remain retryable while it is valid, but an exception or blocked call that reaches expiry sets the process-local `ownership_lost` fence, raises `abort_event`, and cancels the run task. Successful renewals collect durable cancellation actions; after all local renewals have been attempted, heartbeat only signals the corresponding process-local tasks, leaving status writes and rollback cleanup to the worker finalization path. Fenced workers do not perform subsequent journal/delivery-receipt, progress/completion/status, checkpoint/thread-metadata, or `on_run_completed` writes; the peer recovery path owns the terminal receipt. `RunStore.update_run_completion()` also refuses to replace a different terminal status, closing the peer-takeover/late-finalization race. `grace_seconds` delays peer reclamation for clock skew but is not extra execution time for an owner that can no longer confirm its lease. Already-committed remote tool side effects remain outside this local cancellation boundary. - Startup/orphan reconciliation must claim stale active rows with `RunStore.claim_for_takeover()`, not a plain `update_status()`. The final claim re-checks `status` and lease expiry atomically, so a heartbeat renewal between the candidate scan and the recovery write keeps the run active. -- Run admission and independent checkpoint writes are first-class thread operations. `runs.operation_kind` distinguishes user-visible `run` rows from internal `checkpoint_write` reservations, while every active kind shares the existing durable active-thread uniqueness constraint. New operation kinds must go through `RunStore.create_thread_operation_atomic()` and `RunManager.reserve_thread_operation()` rather than adding another lock or metadata marker. Live and lease-less reservations are non-interruptible; an expired leased reservation can be reclaimed immediately by interrupt/rollback admission without waiting for orphan reconciliation. Lease-less rows stay fail-closed because the store cannot distinguish a stale row from a live checkpoint writer in another heartbeat-disabled worker; a rare failed delete therefore requires startup reconciliation, and heartbeat-disabled multi-worker deployment remains unsupported. Reservation bodies are attached to their caller task so loss detected by lease renewal cancels the checkpoint writer before it can continue after takeover; the context manager translates that lease-loss cancellation to `ConflictError` after cleanup so Gateway mutation routes return a retryable 409 instead of dropping the HTTP request. The cleanup scope begins immediately after durable admission, including the await that attaches the caller task, so cancellation cannot strand a locally renewed pending reservation. A failed renewal is revalidated under the manager lock before cancellation; if the reservation completed and unregistered while the store update was in flight, its request task must not be cancelled after the checkpoint write. Reservations are excluded from run history/reporting and from run-only helpers such as `list_by_thread()` and `has_inflight()`, release uses the captured owner rather than ambient user context, and local cleanup still runs when the best-effort store delete fails. `RunStore.create_run_atomic()` remains a deprecated compatibility shim for external stores that only admit normal runs; new stores must implement `create_thread_operation_atomic()` to support internal operation kinds. +- Run admission and independent writes are first-class thread operations. `runs.operation_kind` distinguishes user-visible `run` rows from internal `checkpoint_write` and `artifact_write` reservations, while every active kind shares the existing durable active-thread uniqueness constraint. New operation kinds must go through `RunStore.create_thread_operation_atomic()` and `RunManager.reserve_thread_operation()` rather than adding another lock or metadata marker. Live and lease-less reservations are non-interruptible; an expired leased reservation can be reclaimed immediately by interrupt/rollback admission without waiting for orphan reconciliation. Lease-less rows stay fail-closed because the store cannot distinguish a stale row from a live writer in another heartbeat-disabled worker; a rare failed delete therefore requires startup reconciliation, and heartbeat-disabled multi-worker deployment remains unsupported. Reservation bodies are attached to their caller task so loss detected by lease renewal cancels the writer before it can continue after takeover; the context manager translates that lease-loss cancellation to `ConflictError` after cleanup so Gateway mutation routes return a retryable 409 instead of dropping the HTTP request. The cleanup scope begins immediately after durable admission, including the await that attaches the caller task, so cancellation cannot strand a locally renewed pending reservation. A failed renewal is revalidated under the manager lock before cancellation; if the reservation completed and unregistered while the store update was in flight, its request task must not be cancelled after the write. Reservations are excluded from run history/reporting and from run-only helpers such as `list_by_thread()` and `has_inflight()`, release uses the captured owner rather than ambient user context, and local cleanup still runs when the best-effort store delete fails. `RunStore.create_run_atomic()` remains a deprecated compatibility shim for external stores that only admit normal runs; new stores must implement `create_thread_operation_atomic()` to support internal operation kinds. - Gateway checkpoint mutations outside run execution must use `services.reserve_checkpoint_write()`, which composes the process-local thread lock with the durable `checkpoint_write` reservation. Manual compaction, `POST /threads/{id}/state`, and both goal mutation routes (`PUT` / `DELETE /threads/{id}/goal`, including creation of a missing goal checkpoint) use this boundary, so an existing run blocks the write and the reservation blocks new reject/interrupt/rollback runs across workers. - `POST /wait` (both thread-scoped and `/api/runs/wait`) drains the stream bridge via `wait_for_run_completion()` instead of bare `await record.task`, so it honours the run's `on_disconnect` setting and cancels the background run on real client disconnect rather than returning a stale checkpoint (issue #3265). - Memory and Redis `StreamBridge` implementations retain only `stream_bridge.queue_maxsize` data events. A syntactically valid `Last-Event-ID` older than the retained watermark, or a live subscriber that falls behind it, yields `StreamGap` before any partial replay. `sse_consumer` maps that control item to an id-less SSE `gap` payload (`stream_replay_gap`) and intentionally leaves the run active; internal `/wait` consumers resume from its latest retained ID because they only need terminal completion. Redis checks bounds plus the non-blocking read in one transaction, using blocking `XREAD` only as a wake-up before repeating the atomic snapshot. For a no-cursor subscriber that established a wait on an empty stream, the first wake response remains provisional until that next snapshot verifies its tail is still retained; this closes the pre-first-delivery trimming window without changing malformed-cursor live tailing. The correctness tradeoff is one three-command snapshot pipeline per poll plus the blocking wake round trip while idle. Malformed cursor behavior remains backend-specific. Memory treats a syntactically numeric cursor below its watermark conservatively as a gap even when the evicted timestamp can no longer be verified; unknown ids at or above the watermark retain the legacy replay-from-earliest policy. diff --git a/backend/app/gateway/routers/artifacts.py b/backend/app/gateway/routers/artifacts.py index d39c066e3..35019b97d 100644 --- a/backend/app/gateway/routers/artifacts.py +++ b/backend/app/gateway/routers/artifacts.py @@ -1,17 +1,28 @@ import asyncio +import hashlib import logging import mimetypes +import os +import stat +import tempfile import zipfile +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager from pathlib import Path from urllib.parse import quote from fastapi import APIRouter, HTTPException, Request from fastapi.responses import FileResponse, PlainTextResponse, Response +from pydantic import BaseModel, Field from app.gateway.authz import require_permission +from app.gateway.deps import get_run_manager from app.gateway.internal_auth import get_trusted_internal_owner_user_id from app.gateway.path_utils import resolve_thread_virtual_path from deerflow.config.paths import make_safe_user_id +from deerflow.runtime import ConflictError, ThreadOperationKind +from deerflow.runtime.user_context import get_effective_user_id +from deerflow.sandbox.sandbox_provider import get_sandbox_provider logger = logging.getLogger(__name__) @@ -25,6 +36,110 @@ ACTIVE_CONTENT_MIME_TYPES = { MAX_SKILL_ARCHIVE_MEMBER_BYTES = 16 * 1024 * 1024 _SKILL_ARCHIVE_READ_CHUNK_SIZE = 64 * 1024 +MAX_EDITABLE_ARTIFACT_BYTES = 2 * 1024 * 1024 +_EDITABLE_OUTPUTS_PREFIX = "mnt/user-data/outputs/" +_ARTIFACT_EDIT_TEMP_PREFIX = ".artifact-edit-" + + +class ArtifactUpdateRequest(BaseModel): + content: str + expected_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + +class ArtifactUpdateResponse(BaseModel): + path: str + sha256: str + size: int + + +@asynccontextmanager +async def reserve_artifact_write(request: Request, thread_id: str, *, user_id: str) -> AsyncIterator[None]: + """Serialize an artifact edit against runs and other thread mutations.""" + run_manager = get_run_manager(request) + async with run_manager.reserve_thread_operation( + thread_id, + kind=ThreadOperationKind.artifact_write, + user_id=user_id, + ): + yield + + +def _normalize_editable_artifact_path(path: str) -> str: + stripped = path.lstrip("/") + if not stripped.startswith(_EDITABLE_OUTPUTS_PREFIX): + raise HTTPException(status_code=400, detail="Only files in /mnt/user-data/outputs can be edited") + if ".skill/" in stripped or stripped.endswith(".skill"): + raise HTTPException(status_code=415, detail="Skill archives cannot be edited in the artifacts panel") + return f"/{stripped}" + + +def _load_editable_artifact(actual_path: Path, path: str, expected_sha256: str) -> tuple[bytes, os.stat_result]: + try: + file_stat = os.lstat(actual_path) + except FileNotFoundError: + raise HTTPException(status_code=404, detail=f"Artifact not found: {path}") from None + if stat.S_ISLNK(file_stat.st_mode): + raise HTTPException(status_code=415, detail="Symlinked artifacts cannot be edited") + if not stat.S_ISREG(file_stat.st_mode): + raise HTTPException(status_code=400, detail=f"Path is not a file: {path}") + if file_stat.st_size > MAX_EDITABLE_ARTIFACT_BYTES: + raise HTTPException(status_code=413, detail="Artifact is too large to edit") + + current = actual_path.read_bytes() + if len(current) > MAX_EDITABLE_ARTIFACT_BYTES: + raise HTTPException(status_code=413, detail="Artifact is too large to edit") + if b"\x00" in current: + raise HTTPException(status_code=415, detail="Binary artifacts cannot be edited") + try: + current.decode("utf-8") + except UnicodeDecodeError: + raise HTTPException(status_code=415, detail="Only UTF-8 text artifacts can be edited") from None + + current_sha256 = hashlib.sha256(current).hexdigest() + if current_sha256 != expected_sha256: + raise HTTPException(status_code=412, detail="Artifact changed since it was opened") + return current, file_stat + + +def _encode_artifact_update(content: str) -> bytes: + encoded = content.encode("utf-8") + if len(encoded) > MAX_EDITABLE_ARTIFACT_BYTES: + raise HTTPException(status_code=413, detail="Artifact is too large to edit") + if b"\x00" in encoded: + raise HTTPException(status_code=415, detail="Binary content cannot be saved as an artifact") + return encoded + + +def _replace_artifact_atomically(actual_path: Path, content: bytes, file_stat: os.stat_result) -> None: + temp_fd, temp_path_str = tempfile.mkstemp(prefix=_ARTIFACT_EDIT_TEMP_PREFIX, dir=actual_path.parent) + temp_path = Path(temp_path_str) + try: + # Preserve ownership where possible and keep replacement permissions + # scoped to the owner/group. The shared outputs directory allows a + # mounted sandbox to reach the file without making it world-writable. + if hasattr(os, "fchown"): + try: + os.fchown(temp_fd, file_stat.st_uid, file_stat.st_gid) + except OSError: + logger.debug("Could not preserve artifact ownership: %s", actual_path, exc_info=True) + os.fchmod(temp_fd, stat.S_IMODE(file_stat.st_mode) | 0o660) + with os.fdopen(temp_fd, "wb") as handle: + temp_fd = -1 + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temp_path, actual_path) + finally: + if temp_fd >= 0: + os.close(temp_fd) + try: + temp_path.unlink() + except FileNotFoundError: + pass + + +def _sync_artifact_to_sandbox(sandbox, virtual_path: str, content: bytes) -> None: + sandbox.update_file(virtual_path, content) def _build_content_disposition(disposition_type: str, filename: str) -> str: @@ -255,6 +370,85 @@ async def get_artifact(thread_id: str, path: str, request: Request, download: bo ) if kind == "text": - return PlainTextResponse(content=payload, media_type=mime_type) + assert isinstance(payload, str) + content_sha256 = hashlib.sha256(payload.encode("utf-8")).hexdigest() + return PlainTextResponse(content=payload, media_type=mime_type, headers={"ETag": f'"{content_sha256}"'}) raise AssertionError(f"Unhandled artifact response kind: {kind!r}") + + +@router.put( + "/threads/{thread_id}/artifacts/{path:path}", + response_model=ArtifactUpdateResponse, + summary="Update Artifact File", + description="Replace an existing UTF-8 text artifact after verifying that its content has not changed.", +) +@require_permission("threads", "write", owner_check=True, require_existing=True) +async def update_artifact( + thread_id: str, + path: str, + body: ArtifactUpdateRequest, + request: Request, +) -> ArtifactUpdateResponse: + """Update an existing text artifact while the thread has no active run.""" + virtual_path = _normalize_editable_artifact_path(path) + raw_owner_user_id = get_trusted_internal_owner_user_id(request) + effective_user_id = make_safe_user_id(raw_owner_user_id) if raw_owner_user_id else get_effective_user_id() + + sandbox_provider = None + sandbox_id: str | None = None + sandbox = None + try: + async with reserve_artifact_write(request, thread_id, user_id=effective_user_id): + actual_path = await asyncio.to_thread( + resolve_thread_virtual_path, + thread_id, + virtual_path, + user_id=effective_user_id, + ) + current, file_stat = await asyncio.to_thread( + _load_editable_artifact, + actual_path, + virtual_path, + body.expected_sha256, + ) + updated = _encode_artifact_update(body.content) + + sandbox_provider = get_sandbox_provider() + if not bool(getattr(sandbox_provider, "uses_thread_data_mounts", False)): + sandbox_id = await sandbox_provider.acquire_async(thread_id, user_id=effective_user_id) + sandbox = sandbox_provider.get(sandbox_id) + if sandbox is None: + raise RuntimeError("Failed to acquire sandbox for artifact update") + + try: + if sandbox is not None: + await asyncio.to_thread(_sync_artifact_to_sandbox, sandbox, virtual_path, updated) + await asyncio.to_thread(_replace_artifact_atomically, actual_path, updated, file_stat) + except Exception: + if sandbox is not None: + try: + await asyncio.to_thread(_sync_artifact_to_sandbox, sandbox, virtual_path, current) + except Exception: + logger.exception("Failed to roll back remote artifact after artifact update failure: %s", virtual_path) + raise + except ConflictError: + raise HTTPException(status_code=409, detail="Thread has a run in flight. Save after the run finishes.") from None + except HTTPException: + raise + except Exception: + logger.exception("Failed to update artifact %s for thread %s", path, thread_id) + raise HTTPException(status_code=500, detail="Failed to update artifact") from None + finally: + if sandbox_id is not None and sandbox_provider is not None: + try: + await asyncio.to_thread(sandbox_provider.release, sandbox_id) + except Exception: + logger.warning("Failed to release sandbox after artifact update: %s", sandbox_id, exc_info=True) + + content_sha256 = hashlib.sha256(updated).hexdigest() + return ArtifactUpdateResponse( + path=virtual_path, + sha256=content_sha256, + size=len(updated), + ) diff --git a/backend/packages/harness/deerflow/runtime/runs/schemas.py b/backend/packages/harness/deerflow/runtime/runs/schemas.py index 028bcb1ad..2adcb99b7 100644 --- a/backend/packages/harness/deerflow/runtime/runs/schemas.py +++ b/backend/packages/harness/deerflow/runtime/runs/schemas.py @@ -8,6 +8,7 @@ class ThreadOperationKind(StrEnum): run = "run" checkpoint_write = "checkpoint_write" + artifact_write = "artifact_write" class RunStatus(StrEnum): diff --git a/backend/tests/blocking_io/test_artifacts_router.py b/backend/tests/blocking_io/test_artifacts_router.py index 7bb0d6751..393217763 100644 --- a/backend/tests/blocking_io/test_artifacts_router.py +++ b/backend/tests/blocking_io/test_artifacts_router.py @@ -26,19 +26,23 @@ sit at module top so any import-time IO runs at collection, outside the gate. from __future__ import annotations import asyncio +import hashlib import zipfile +from contextlib import asynccontextmanager from pathlib import Path import pytest from starlette.responses import FileResponse +import app.gateway.routers.artifacts as artifacts_router from app.gateway.path_utils import resolve_thread_virtual_path -from app.gateway.routers.artifacts import get_artifact +from app.gateway.routers.artifacts import ArtifactUpdateRequest, get_artifact, update_artifact pytestmark = pytest.mark.asyncio # The undecorated coroutine (``require_permission`` uses ``functools.wraps``). _get_artifact = get_artifact.__wrapped__ +_update_artifact = update_artifact.__wrapped__ async def _seed(tmp_path: Path, monkeypatch, thread_id: str, virtual_path: str) -> Path: @@ -96,3 +100,33 @@ async def test_get_artifact_skill_archive_member_does_not_block_event_loop(tmp_p assert resp.status_code == 200 assert b"# demo skill" in resp.body + + +async def test_update_artifact_does_not_block_event_loop(tmp_path: Path, monkeypatch) -> None: + vpath = "/mnt/user-data/outputs/notes.txt" + target = await _seed(tmp_path, monkeypatch, "t1", vpath) + original = b"hello world" + await asyncio.to_thread(target.write_bytes, original) + + @asynccontextmanager + async def allow_write(*_args, **_kwargs): + yield + + class MountedProvider: + uses_thread_data_mounts = True + + monkeypatch.setattr(artifacts_router, "reserve_artifact_write", allow_write) + monkeypatch.setattr(artifacts_router, "get_sandbox_provider", lambda: MountedProvider()) + + result = await _update_artifact( + "t1", + vpath, + ArtifactUpdateRequest( + content="updated", + expected_sha256=hashlib.sha256(original).hexdigest(), + ), + request=None, + ) + + assert result.sha256 == hashlib.sha256(b"updated").hexdigest() + assert await asyncio.to_thread(target.read_bytes) == b"updated" diff --git a/backend/tests/test_artifacts_router.py b/backend/tests/test_artifacts_router.py index 6f5864c81..3a4b14922 100644 --- a/backend/tests/test_artifacts_router.py +++ b/backend/tests/test_artifacts_router.py @@ -1,5 +1,8 @@ import asyncio +import hashlib +import stat import zipfile +from contextlib import asynccontextmanager from pathlib import Path from types import SimpleNamespace @@ -44,6 +47,265 @@ def test_get_artifact_reads_utf8_text_file_on_windows_locale(tmp_path, monkeypat assert bytes(response.body).decode("utf-8") == text assert response.media_type == "text/plain" + assert response.headers["etag"] == f'"{hashlib.sha256(text.encode("utf-8")).hexdigest()}"' + + +@asynccontextmanager +async def _allow_artifact_write(*_args, **_kwargs): + yield + + +class _MountedSandboxProvider: + uses_thread_data_mounts = True + + +class _RemoteSandbox: + def __init__(self, *, fail_next_update: bool = False) -> None: + self.updates: list[tuple[str, bytes]] = [] + self.fail_next_update = fail_next_update + + def update_file(self, path: str, content: bytes) -> None: + if self.fail_next_update: + self.fail_next_update = False + raise RuntimeError("sandbox sync failed") + self.updates.append((path, content)) + + +class _RemoteSandboxProvider: + uses_thread_data_mounts = False + + def __init__(self, *, fail_next_update: bool = False) -> None: + self.sandbox = _RemoteSandbox(fail_next_update=fail_next_update) + self.released: list[str] = [] + + async def acquire_async(self, _thread_id: str, *, user_id: str | None = None) -> str: + return "sandbox-1" + + def get(self, sandbox_id: str): + assert sandbox_id == "sandbox-1" + return self.sandbox + + def release(self, sandbox_id: str) -> None: + self.released.append(sandbox_id) + + +def _artifact_sha256(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def _patch_artifact_update_dependencies(monkeypatch, artifact_path: Path, provider=None) -> None: + monkeypatch.setattr(artifacts_router, "resolve_thread_virtual_path", lambda _thread_id, _path, user_id=None: artifact_path) + monkeypatch.setattr(artifacts_router, "reserve_artifact_write", _allow_artifact_write) + monkeypatch.setattr(artifacts_router, "get_sandbox_provider", lambda: provider or _MountedSandboxProvider()) + + +def test_update_artifact_replaces_utf8_text_atomically(tmp_path, monkeypatch) -> None: + artifact_path = tmp_path / "note.txt" + artifact_path.write_text("before", encoding="utf-8") + artifact_path.chmod(0o600) + _patch_artifact_update_dependencies(monkeypatch, artifact_path) + + response = asyncio.run( + call_unwrapped( + artifacts_router.update_artifact, + "thread-1", + "mnt/user-data/outputs/note.txt", + artifacts_router.ArtifactUpdateRequest(content="after", expected_sha256=_artifact_sha256("before")), + _make_request(), + ) + ) + + assert artifact_path.read_text(encoding="utf-8") == "after" + assert response.path == "/mnt/user-data/outputs/note.txt" + assert response.sha256 == _artifact_sha256("after") + assert response.size == len(b"after") + replacement_mode = stat.S_IMODE(artifact_path.stat().st_mode) + assert replacement_mode == 0o660 + assert not replacement_mode & stat.S_IWOTH + + +def test_update_artifact_rejects_stale_revision_without_changing_file(tmp_path, monkeypatch) -> None: + artifact_path = tmp_path / "note.txt" + artifact_path.write_text("agent version", encoding="utf-8") + _patch_artifact_update_dependencies(monkeypatch, artifact_path) + + with pytest.raises(HTTPException) as exc_info: + asyncio.run( + call_unwrapped( + artifacts_router.update_artifact, + "thread-1", + "mnt/user-data/outputs/note.txt", + artifacts_router.ArtifactUpdateRequest(content="user version", expected_sha256=_artifact_sha256("old version")), + _make_request(), + ) + ) + + assert exc_info.value.status_code == 412 + assert artifact_path.read_text(encoding="utf-8") == "agent version" + + +def test_update_artifact_rejects_non_output_path(tmp_path, monkeypatch) -> None: + artifact_path = tmp_path / "note.txt" + artifact_path.write_text("before", encoding="utf-8") + _patch_artifact_update_dependencies(monkeypatch, artifact_path) + + with pytest.raises(HTTPException) as exc_info: + asyncio.run( + call_unwrapped( + artifacts_router.update_artifact, + "thread-1", + "mnt/user-data/workspace/note.txt", + artifacts_router.ArtifactUpdateRequest(content="after", expected_sha256=_artifact_sha256("before")), + _make_request(), + ) + ) + + assert exc_info.value.status_code == 400 + assert artifact_path.read_text(encoding="utf-8") == "before" + + +def test_update_artifact_rejects_binary_file(tmp_path, monkeypatch) -> None: + artifact_path = tmp_path / "blob.bin" + artifact_path.write_bytes(b"before\x00binary") + _patch_artifact_update_dependencies(monkeypatch, artifact_path) + + with pytest.raises(HTTPException) as exc_info: + asyncio.run( + call_unwrapped( + artifacts_router.update_artifact, + "thread-1", + "mnt/user-data/outputs/blob.bin", + artifacts_router.ArtifactUpdateRequest(content="after", expected_sha256=hashlib.sha256(b"before\x00binary").hexdigest()), + _make_request(), + ) + ) + + assert exc_info.value.status_code == 415 + + +def test_update_artifact_syncs_non_mounted_sandbox(tmp_path, monkeypatch) -> None: + artifact_path = tmp_path / "note.txt" + artifact_path.write_text("before", encoding="utf-8") + provider = _RemoteSandboxProvider() + _patch_artifact_update_dependencies(monkeypatch, artifact_path, provider) + + asyncio.run( + call_unwrapped( + artifacts_router.update_artifact, + "thread-1", + "mnt/user-data/outputs/note.txt", + artifacts_router.ArtifactUpdateRequest(content="after", expected_sha256=_artifact_sha256("before")), + _make_request(), + ) + ) + + assert provider.sandbox.updates == [("/mnt/user-data/outputs/note.txt", b"after")] + assert provider.released == ["sandbox-1"] + assert artifact_path.read_text(encoding="utf-8") == "after" + + +def test_update_artifact_releases_sandbox_when_initial_sync_fails(tmp_path, monkeypatch) -> None: + artifact_path = tmp_path / "note.txt" + artifact_path.write_text("before", encoding="utf-8") + provider = _RemoteSandboxProvider(fail_next_update=True) + _patch_artifact_update_dependencies(monkeypatch, artifact_path, provider) + + with pytest.raises(HTTPException) as exc_info: + asyncio.run( + call_unwrapped( + artifacts_router.update_artifact, + "thread-1", + "mnt/user-data/outputs/note.txt", + artifacts_router.ArtifactUpdateRequest(content="after", expected_sha256=_artifact_sha256("before")), + _make_request(), + ) + ) + + assert exc_info.value.status_code == 500 + assert provider.released == ["sandbox-1"] + assert provider.sandbox.updates == [("/mnt/user-data/outputs/note.txt", b"before")] + assert artifact_path.read_text(encoding="utf-8") == "before" + + +def test_update_artifact_rolls_back_remote_when_local_replace_fails(tmp_path, monkeypatch) -> None: + artifact_path = tmp_path / "note.txt" + artifact_path.write_text("before", encoding="utf-8") + provider = _RemoteSandboxProvider() + _patch_artifact_update_dependencies(monkeypatch, artifact_path, provider) + + def fail_replace(*_args, **_kwargs) -> None: + raise OSError("replace failed") + + monkeypatch.setattr(artifacts_router, "_replace_artifact_atomically", fail_replace) + + with pytest.raises(HTTPException) as exc_info: + asyncio.run( + call_unwrapped( + artifacts_router.update_artifact, + "thread-1", + "mnt/user-data/outputs/note.txt", + artifacts_router.ArtifactUpdateRequest(content="after", expected_sha256=_artifact_sha256("before")), + _make_request(), + ) + ) + + assert exc_info.value.status_code == 500 + assert provider.sandbox.updates == [ + ("/mnt/user-data/outputs/note.txt", b"after"), + ("/mnt/user-data/outputs/note.txt", b"before"), + ] + assert provider.released == ["sandbox-1"] + assert artifact_path.read_text(encoding="utf-8") == "before" + + +def test_update_artifact_rejects_oversized_content(tmp_path, monkeypatch) -> None: + artifact_path = tmp_path / "note.txt" + artifact_path.write_text("before", encoding="utf-8") + _patch_artifact_update_dependencies(monkeypatch, artifact_path) + + with pytest.raises(HTTPException) as exc_info: + asyncio.run( + call_unwrapped( + artifacts_router.update_artifact, + "thread-1", + "mnt/user-data/outputs/note.txt", + artifacts_router.ArtifactUpdateRequest( + content="x" * (artifacts_router.MAX_EDITABLE_ARTIFACT_BYTES + 1), + expected_sha256=_artifact_sha256("before"), + ), + _make_request(), + ) + ) + + assert exc_info.value.status_code == 413 + assert artifact_path.read_text(encoding="utf-8") == "before" + + +def test_update_artifact_reports_active_run_conflict(tmp_path, monkeypatch) -> None: + artifact_path = tmp_path / "note.txt" + artifact_path.write_text("before", encoding="utf-8") + monkeypatch.setattr(artifacts_router, "resolve_thread_virtual_path", lambda _thread_id, _path, user_id=None: artifact_path) + + @asynccontextmanager + async def reject_artifact_write(*_args, **_kwargs): + raise artifacts_router.ConflictError("active run") + yield + + monkeypatch.setattr(artifacts_router, "reserve_artifact_write", reject_artifact_write) + + with pytest.raises(HTTPException) as exc_info: + asyncio.run( + call_unwrapped( + artifacts_router.update_artifact, + "thread-1", + "mnt/user-data/outputs/note.txt", + artifacts_router.ArtifactUpdateRequest(content="after", expected_sha256=_artifact_sha256("before")), + _make_request(), + ) + ) + + assert exc_info.value.status_code == 409 + assert artifact_path.read_text(encoding="utf-8") == "before" @pytest.mark.parametrize(("filename", "content"), ACTIVE_ARTIFACT_CASES) diff --git a/frontend/AGENTS.md b/frontend/AGENTS.md index b38e706af..9badb84f5 100644 --- a/frontend/AGENTS.md +++ b/frontend/AGENTS.md @@ -70,6 +70,7 @@ The frontend is a stateful chat application. Users create **threads** (conversat File-tool artifact auto-open work must run in an effect with timer cleanup; never schedule timers while rendering streamed `write_file` or `str_replace` updates. `ThreadState.artifacts` remains the authoritative artifact list. The artifacts provider persists only thread-scoped panel UI state (`open`, selected path, and a refresh bootstrap cache) in session storage; an initial empty stream value must not overwrite that restored state before history finishes loading. Formal artifact content is refreshed once when the run finishes; transient `write-file:` previews remain message-driven. + The detail view exposes explicit editing only for an already-opened formal UTF-8 text artifact under `/mnt/user-data/outputs`. Drafts stay in provider memory until Save so switching right-side panels cannot discard them, render in Markdown/HTML preview, and are protected from remote refreshes by the loaded SHA-256 revision. Saving is disabled during an active run; a changed revision preserves the draft and surfaces a conflict instead of overwriting agent output. 3. `useThreadHistory` loads persisted conversation pages from `GET /api/threads/{id}/messages/page`, preserving the backend's thread-global event `seq`; rendering overlays checkpoint/live copies at their matching canonical identities (a summarized checkpoint may contain a protected early input plus a recent tail). Context-compaction rescue diffs every retained visible identity rather than slicing at the first anchor, and keeps a run-scoped ledger of committed visible messages so replacement updates and repeated rolling checkpoint windows cannot erase an already displayed step. The resolver suppresses checkpoint/transient prefixes whose canonical position is still behind an unloaded cursor page instead of collapsing that unknown gap before a recent anchor, then adds optimistic messages without timestamp re-sorting. History invalidation preserves already-loaded pages so their established ordering positions are not discarded. Dynamic context re-keys the submitted user message from `X` to `X__user`; UI identity matching normalizes that reserved suffix only for human messages so the submitted frame and checkpoint replacement remain one visible turn. A locally submitted turn also records its pre-submit identity baseline: if `messages-tuple` publishes new AI/tool steps before `values` publishes that turn's human message, render ordering moves only those non-baseline visible steps behind the new human while leaving history, hidden controls, and reconnected runs untouched. Keep that local order anchor through finish, stop, and stream error because the SDK's settled frame can retain transient event order; replace it on the next local submit and clear it on thread switch or replay-gap recovery. 4. Stop actions call the LangGraph SDK stream stop path; `core/threads/hooks.ts` invalidates current-thread, thread-history, token-usage, and sidebar/search caches immediately and schedules one follow-up refetch because SDK stop may finish via abort + fire-and-forget cancel before backend title finalization commits 5. TanStack Query manages server state; localStorage stores user settings. The diff --git a/frontend/src/components/workspace/artifacts/artifact-file-detail.tsx b/frontend/src/components/workspace/artifacts/artifact-file-detail.tsx index 3d257e226..0d170f316 100644 --- a/frontend/src/components/workspace/artifacts/artifact-file-detail.tsx +++ b/frontend/src/components/workspace/artifacts/artifact-file-detail.tsx @@ -1,3 +1,4 @@ +import { useQueryClient } from "@tanstack/react-query"; import { Code2Icon, CopyIcon, @@ -5,6 +6,10 @@ import { EyeIcon, LoaderIcon, PackageIcon, + PencilIcon, + PencilOffIcon, + RotateCcwIcon, + SaveIcon, SquareArrowOutUpRightIcon, XIcon, } from "lucide-react"; @@ -29,6 +34,15 @@ import { } from "@/components/ui/select"; import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group"; import { CodeEditor } from "@/components/workspace/code-editor"; +import { + ArtifactRequestError, + updateArtifactContent, +} from "@/core/artifacts/api"; +import { + canEditOpenedArtifact, + createArtifactDraft, + reconcileArtifactDraft, +} from "@/core/artifacts/editing"; import { useArtifactContent } from "@/core/artifacts/hooks"; import { appendHtmlPreviewBaseHref, @@ -78,9 +92,18 @@ export function ArtifactFileDetail({ threadId: string; }) { const { t } = useI18n(); + const queryClient = useQueryClient(); const { user } = useAuth(); const isAdmin = user?.system_role === "admin"; - const { artifacts, setOpen, select } = useArtifacts(); + const { + artifacts, + setOpen, + select, + drafts, + setDrafts, + editingPath, + setEditingPath, + } = useArtifacts(); const { thread, isMock } = useThread(); const isWriteFile = useMemo(() => { return filepathFromProps.startsWith("write-file:"); @@ -170,7 +193,7 @@ export function ArtifactFileDetail({ isSupportPreview, toolResult, }); - const { content, url } = useArtifactContent({ + const { content, url, sha256 } = useArtifactContent({ threadId, filepath: filepathFromProps, enabled: isCodeFile && !isWriteFile, @@ -184,6 +207,38 @@ export function ArtifactFileDetail({ filepathFromProps, ); + const [isSaving, setIsSaving] = useState(false); + const activeDraft = drafts[filepath] ?? createArtifactDraft(filepath); + const isDirty = activeDraft.draftContent !== activeDraft.baselineContent; + const hasUnsavedDrafts = Object.values(drafts).some( + (draft) => draft.draftContent !== draft.baselineContent, + ); + const isEditing = editingPath === filepath; + const canEdit = canEditOpenedArtifact({ + filepath, + isCodeFile, + isWriteFile, + isSkillFile, + isMock: Boolean(isMock), + hasRevision: typeof sha256 === "string", + isStaticWebsite: env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true", + }); + const editorContent = isDirty ? activeDraft.draftContent : visibleContent; + + useEffect(() => { + if (content === undefined || sha256 === undefined || isWriteFile) { + return; + } + setDrafts((current) => { + const existing = current[filepath] ?? createArtifactDraft(filepath); + const next = reconcileArtifactDraft(existing, { content, sha256 }); + if (next === existing) { + return current; + } + return { ...current, [filepath]: next }; + }); + }, [content, filepath, isWriteFile, setDrafts, sha256]); + const [viewMode, setViewMode] = useState<"code" | "preview">( artifactViewState.initialViewMode, ); @@ -192,6 +247,108 @@ export function ArtifactFileDetail({ setViewMode(artifactViewState.initialViewMode); }, [artifactViewState.initialViewMode]); + const confirmDiscard = useCallback(() => { + return !isDirty || window.confirm(t.artifactEditing.discardChanges); + }, [isDirty, t.artifactEditing.discardChanges]); + + const discardDraft = useCallback(() => { + const latestContent = content ?? activeDraft.baselineContent; + const latestSha256 = sha256 ?? activeDraft.baselineSha256; + setDrafts((current) => ({ + ...current, + [filepath]: { + ...activeDraft, + baselineContent: latestContent, + baselineSha256: latestSha256, + draftContent: latestContent, + conflict: false, + }, + })); + setEditingPath(null); + }, [activeDraft, content, filepath, setDrafts, setEditingPath, sha256]); + + const handleSave = useCallback(async () => { + if ( + !canEdit || + !isDirty || + isSaving || + thread.isLoading || + activeDraft.conflict || + activeDraft.baselineSha256 === null + ) { + return; + } + + setIsSaving(true); + try { + const result = await updateArtifactContent({ + threadId, + filepath, + content: activeDraft.draftContent, + expectedSha256: activeDraft.baselineSha256, + }); + const savedContent = activeDraft.draftContent; + setDrafts((current) => ({ + ...current, + [filepath]: { + filepath, + baselineContent: savedContent, + baselineSha256: result.sha256, + draftContent: savedContent, + conflict: false, + }, + })); + queryClient.setQueryData( + ["artifact", filepathFromProps, threadId, isMock], + ( + current: + | { content?: string; url?: string; sha256?: string } + | undefined, + ) => ({ + ...current, + content: savedContent, + sha256: result.sha256, + }), + ); + toast.success(t.artifactEditing.saved); + } catch (error) { + if (error instanceof ArtifactRequestError && error.status === 412) { + setDrafts((current) => ({ + ...current, + [filepath]: { ...(current[filepath] ?? activeDraft), conflict: true }, + })); + void queryClient.invalidateQueries({ + queryKey: ["artifact", filepathFromProps, threadId, isMock], + }); + toast.error(t.artifactEditing.conflict); + } else if ( + error instanceof ArtifactRequestError && + error.status === 409 + ) { + toast.error(t.artifactEditing.runInProgress); + } else { + toast.error( + error instanceof Error ? error.message : t.artifactEditing.saveFailed, + ); + } + } finally { + setIsSaving(false); + } + }, [ + activeDraft, + canEdit, + filepath, + filepathFromProps, + isDirty, + isMock, + isSaving, + queryClient, + setDrafts, + t.artifactEditing, + thread.isLoading, + threadId, + ]); + const handleInstallSkill = useCallback(async () => { if (isInstalling) return; @@ -225,7 +382,17 @@ export function ArtifactFileDetail({ {isWriteFile ? (
{getFileName(filepath)}
) : ( - { + if (confirmDiscard()) { + if (isDirty) { + discardDraft(); + } + select(nextFilepath); + } + }} + > @@ -242,7 +409,7 @@ export function ArtifactFileDetail({ )} -
+
{artifactViewState.canPreview && ( )} + {(isSaving || isDirty || activeDraft.conflict) && ( + + {isSaving + ? t.artifactEditing.saving + : activeDraft.conflict + ? t.artifactEditing.conflictShort + : t.artifactEditing.unsaved} + + )}
- {!isWriteFile && filepath.endsWith(".skill") && isAdmin && ( - - - + {canEdit && !isEditing && ( + { + setViewMode("code"); + setEditingPath(filepath); + }} + /> )} - {!isWriteFile && ( + {canEdit && isEditing && ( + <> + void handleSave()} + /> + setEditingPath(null)} + /> + { + if (confirmDiscard()) { + discardDraft(); + } + }} + /> + + )} + {!isEditing && + !isWriteFile && + filepath.endsWith(".skill") && + isAdmin && ( + + + + )} + {!isEditing && !isWriteFile && ( )} - {isCodeFile && ( + {!isEditing && isCodeFile && ( { void (async () => { const didCopy = await writeTextToClipboard( - visibleContent ?? "", + editorContent ?? "", ); if (!didCopy) { toast.error(t.clipboard.failedToCopyToClipboard); @@ -319,7 +559,7 @@ export function ArtifactFileDetail({ tooltip={t.clipboard.copyToClipboard} /> )} - {!isWriteFile && ( + {!isEditing && !isWriteFile && ( setOpen(false)} + onClick={() => { + if ( + !hasUnsavedDrafts || + window.confirm(t.artifactEditing.discardChanges) + ) { + setDrafts({}); + setEditingPath(null); + setOpen(false); + } + }} tooltip={t.common.close} /> @@ -353,7 +602,7 @@ export function ArtifactFileDetail({ viewMode === "preview" && (language === "markdown" || language === "html") && ( { + setDrafts((current) => ({ + ...current, + [filepath]: { + ...(current[filepath] ?? activeDraft), + draftContent: nextContent, + }, + })); + }} + onSave={() => void handleSave()} /> )} {!isCodeFile && canPreviewInBrowser && ( diff --git a/frontend/src/components/workspace/artifacts/context.tsx b/frontend/src/components/workspace/artifacts/context.tsx index 33a236930..694c789b6 100644 --- a/frontend/src/components/workspace/artifacts/context.tsx +++ b/frontend/src/components/workspace/artifacts/context.tsx @@ -1,15 +1,18 @@ import { usePathname } from "next/navigation"; import { createContext, + type Dispatch, useCallback, useContext, useEffect, useRef, useState, type ReactNode, + type SetStateAction, } from "react"; import { useSidebar } from "@/components/ui/sidebar"; +import type { ArtifactDraftState } from "@/core/artifacts/editing"; import { env } from "@/env"; export interface ArtifactsContextType { @@ -24,6 +27,11 @@ export interface ArtifactsContextType { open: boolean; autoOpen: boolean; setOpen: (open: boolean) => void; + + drafts: Record; + setDrafts: Dispatch>>; + editingPath: string | null; + setEditingPath: Dispatch>; } const ArtifactsContext = createContext( @@ -82,6 +90,8 @@ export function ArtifactsProvider({ children }: ArtifactsProviderProps) { env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true", ); const [autoOpen, setAutoOpen] = useState(true); + const [drafts, setDrafts] = useState>({}); + const [editingPath, setEditingPath] = useState(null); const { setOpen: setSidebarOpen } = useSidebar(); const pathname = usePathname(); const hydratedPathRef = useRef(null); @@ -97,9 +107,25 @@ export function ArtifactsProvider({ children }: ArtifactsProviderProps) { setOpen(persisted?.open ?? env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true"); setAutoOpen(true); setAutoSelect(!persisted?.selectedArtifact); + setDrafts({}); + setEditingPath(null); hydratedPathRef.current = pathname; }, [pathname]); + useEffect(() => { + const hasUnsavedDrafts = Object.values(drafts).some( + (draft) => draft.draftContent !== draft.baselineContent, + ); + if (!hasUnsavedDrafts) { + return; + } + const handleBeforeUnload = (event: BeforeUnloadEvent) => { + event.preventDefault(); + }; + window.addEventListener("beforeunload", handleBeforeUnload); + return () => window.removeEventListener("beforeunload", handleBeforeUnload); + }, [drafts]); + useEffect(() => { if (!pathname || hydratedPathRef.current !== pathname) { return; @@ -151,6 +177,11 @@ export function ArtifactsProvider({ children }: ArtifactsProviderProps) { selectedArtifact, select, deselect, + + drafts, + setDrafts, + editingPath, + setEditingPath, }; return ( diff --git a/frontend/src/components/workspace/code-editor.tsx b/frontend/src/components/workspace/code-editor.tsx index 84c558c46..1d4fa5841 100644 --- a/frontend/src/components/workspace/code-editor.tsx +++ b/frontend/src/components/workspace/code-editor.tsx @@ -41,6 +41,8 @@ export function CodeEditor({ readonly, disabled, autoFocus, + onChange, + onSave, settings, }: { className?: string; @@ -49,6 +51,8 @@ export function CodeEditor({ readonly?: boolean; disabled?: boolean; autoFocus?: boolean; + onChange?: (value: string) => void; + onSave?: () => void; settings?: unknown; }) { const { @@ -76,6 +80,15 @@ export function CodeEditor({ "flex cursor-text flex-col overflow-hidden rounded-md", className, )} + onKeyDown={(event) => { + if ( + (event.metaKey || event.ctrlKey) && + event.key.toLowerCase() === "s" + ) { + event.preventDefault(); + onSave?.(); + } + }} > {isLoading ? (