Felix Shi d53ec767af
fix(persistence): keep LF bytes in skill and agent text writes (#7501)
* fix(persistence): keep LF bytes in skill and agent text writes

Text mode translates a newline into os.linesep on write, so on Windows every
caller-supplied LF reached disk as CRLF while readers use universal newlines or
decode bytes explicitly. The difference stayed invisible: the API returned LF,
the file held CRLF, and a skill history record's prev_content described a
revision no API view could return (#7174).

Pass newline="" at the ten writers that persist or judge caller text: the local
and user-scoped skill writers, the skill history JSONL, the user skill-states
JSON, the SOUL.md/config.yaml staging in FileAgentStore, the managed-subagent
store, setup_agent's SOUL.md publish, the USER.md route, and the two temp copies
the frontmatter validator and the static scanner judge.

The tests emulate the Windows translation on io.open, so the regression runs on
the Linux CI shards instead of only on a Windows host: 10 failed before this
commit, 10 pass after.

* fix(skills): preserve candidate newlines during static scan

---------

Co-authored-by: Felix Shi <204130665+shi-felix@users.noreply.github.com>
Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
2026-10-26 18:45:49 +08:00

273 lines
13 KiB
Python

"""Filesystem-backed agent store — today's per-user layout, behaviour-preserving.
The read methods are the pre-refactor bodies of ``load_agent_config`` /
``load_agent_soul`` / ``list_custom_agents`` (so the free functions in
:mod:`deerflow.config.agents_config` dispatch here without changing behaviour).
Writes use a staged temp-file + atomic ``os.replace`` commit — the crash-safety
the ``update_agent`` tool already had, applied uniformly to create/update.
Path/user resolution is done through the :mod:`deerflow.config.agents_config`
module object (``_ac.get_paths`` / ``_ac.get_effective_user_id``) rather than
direct imports, so it honours the same monkeypatch seams the existing agent
tests target.
"""
from __future__ import annotations
import logging
import shutil
import tempfile
from collections.abc import Hashable
from pathlib import Path
from typing import Any
import yaml
from deerflow.config import agents_config as _ac
from deerflow.config.agents_config import (
SOUL_FILENAME,
AgentConfig,
resolve_agent_dir,
validate_agent_name,
)
from deerflow.persistence.agents.base import (
AgentDeleteOutcome,
AgentExistsError,
AgentStore,
parse_agent_config,
)
from deerflow.runtime.user_context import DEFAULT_USER_ID
from deerflow.utils.atomic_write import cleanup_failed_atomic_write
logger = logging.getLogger(__name__)
class FileAgentStore(AgentStore):
def get(self, name: str, *, user_id: str | None = None) -> AgentConfig:
name = validate_agent_name(name)
agent_dir = resolve_agent_dir(name, user_id=user_id)
config_file = agent_dir / "config.yaml"
if not agent_dir.exists():
raise FileNotFoundError(f"Agent directory not found: {agent_dir}")
if not config_file.exists():
raise FileNotFoundError(f"Agent config not found: {config_file}")
try:
with open(config_file, encoding="utf-8") as f:
data: dict[str, Any] = yaml.safe_load(f) or {}
except yaml.YAMLError as e:
raise ValueError(f"Failed to parse agent config {config_file}: {e}") from e
return parse_agent_config(data, name)
def exists(self, name: str, *, user_id: str | None = None) -> bool:
name = validate_agent_name(name)
paths = _ac.get_paths()
effective_user = user_id or _ac.get_effective_user_id()
return paths.user_agent_dir(effective_user, name).exists() or paths.agent_dir(name).exists()
def get_soul(self, name: str, *, user_id: str | None = None) -> str | None:
# Every other read/write method validates the name; without this, a
# ".." segment reaches resolve_agent_dir and escapes the user's tree.
name = validate_agent_name(name)
agent_dir = resolve_agent_dir(name, user_id=user_id)
soul_path = agent_dir / SOUL_FILENAME
# resolve_agent_dir requires config.yaml; SOUL.md loading does not, so
# when the resolver fell back to its default path (no qualifying dir),
# check the per-user and legacy dirs directly (#4135). The config.yaml
# guard keeps this fallback from firing for a properly-resolved per-user
# agent that merely lacks SOUL.md (preserves per-user-shadows-legacy).
if not soul_path.exists() and not (agent_dir / "config.yaml").exists():
paths = _ac.get_paths()
effective_user = user_id or _ac.get_effective_user_id()
for candidate in (
paths.user_agent_dir(effective_user, name),
paths.agent_dir(name),
):
if (candidate / SOUL_FILENAME).exists():
soul_path = candidate / SOUL_FILENAME
break
if not soul_path.exists():
return None
# Match the default-agent read in :func:`deerflow.config.agents_config.
# load_agent_soul`: ``utf-8-sig`` drops the BOM a Windows editor writes,
# which ``str.strip()`` would otherwise leave as the leading character of
# the ``<soul>`` block.
content = soul_path.read_text(encoding="utf-8-sig").strip()
return content or None
def list(self, *, user_id: str | None = None) -> list[AgentConfig]:
paths = _ac.get_paths()
effective_user = user_id or _ac.get_effective_user_id()
seen: set[str] = set()
agents: list[AgentConfig] = []
for root in (paths.user_agents_dir(effective_user), paths.agents_dir):
if not root.exists():
continue
for entry in sorted(root.iterdir()):
if not entry.is_dir() or entry.name in seen:
continue
if not (entry / "config.yaml").exists():
logger.debug("Skipping %s: no config.yaml", entry.name)
continue
try:
agents.append(self.get(entry.name, user_id=effective_user))
seen.add(entry.name)
except Exception as e: # noqa: BLE001 — one bad agent must not hide the rest
logger.warning("Skipping agent '%s': %s", entry.name, e)
agents.sort(key=lambda a: a.name)
return agents
def list_all(self) -> list[tuple[str, AgentConfig]]:
result: list[tuple[str, AgentConfig]] = []
for user_id, name in self._discover():
try:
result.append((user_id, self.get(name, user_id=user_id)))
except Exception as e: # noqa: BLE001
logger.warning("list_all: skipping agent %s/%s: %s", user_id, name, e)
return result
def create(self, name: str, config: dict, soul: str, *, user_id: str | None = None) -> None:
name = validate_agent_name(name)
paths = _ac.get_paths()
effective_user = user_id or _ac.get_effective_user_id()
agent_dir = paths.user_agent_dir(effective_user, name)
# Refuse if a per-user directory OR a legacy shared directory already
# owns the name — the agents router's 409 semantics (a legacy agent must
# not be shadowed; a per-user dir may be memory-only but still blocks).
if agent_dir.exists() or paths.agent_dir(name).exists():
raise AgentExistsError(f"Agent '{name}' already exists for user '{effective_user}'")
try:
agent_dir.mkdir(parents=True, exist_ok=False)
except FileExistsError as e:
# A concurrent create passed the existence check above and reached
# mkdir first. Surface the router's 409 (via AgentExistsError) rather
# than a generic 500 — mirrors SqlAgentStore's IntegrityError path.
raise AgentExistsError(f"Agent '{name}' already exists for user '{effective_user}'") from e
try:
self._write(agent_dir, config, soul)
except Exception:
# The directory was newly created for this call; a failed write
# must not leave an empty/partial agent dir behind.
shutil.rmtree(agent_dir, ignore_errors=True)
raise
def update(self, name: str, config: dict | None, soul: str | None, *, user_id: str | None = None) -> None:
name = validate_agent_name(name)
effective_user = user_id or _ac.get_effective_user_id()
agent_dir = _ac.get_paths().user_agent_dir(effective_user, name)
pre_existing = agent_dir.exists()
agent_dir.mkdir(parents=True, exist_ok=True)
try:
self._write(agent_dir, config, soul)
except Exception:
# Only clean up a directory this call created — never delete a
# pre-existing agent on a failed write.
if not pre_existing:
shutil.rmtree(agent_dir, ignore_errors=True)
raise
def delete(self, name: str, *, user_id: str | None = None) -> AgentDeleteOutcome:
name = validate_agent_name(name)
paths = _ac.get_paths()
effective_user = user_id or _ac.get_effective_user_id()
agent_dir = paths.user_agent_dir(effective_user, name)
if not agent_dir.exists():
# A legacy shared-layout agent is intentionally left in place (the
# write path never targets it); report it distinctly.
return "legacy" if paths.agent_dir(name).exists() else "missing"
if not (agent_dir / "config.yaml").is_file():
# The directory holds memory/facts data but is not a custom agent
# (no config.yaml) — preserve it rather than deleting a user's memory
# (#4279). rmtree below would otherwise take the whole tree.
return "not-custom-agent"
# rmtree removes config.yaml, SOUL.md and the co-located memory.json in
# one shot — the historical behaviour.
shutil.rmtree(agent_dir)
return "deleted"
def signature(self) -> Hashable:
sig: list[tuple[str, str, float]] = []
for user_id, name in self._discover():
config = resolve_agent_dir(name, user_id=user_id) / "config.yaml"
try:
sig.append((user_id, name, config.stat().st_mtime))
except OSError:
continue
return tuple(sig)
# -- internals --
def _discover(self) -> list[tuple[str, str]]:
"""Enumerate ``(user_id, name)`` across per-user and legacy layouts.
A legacy shared-layout agent is attributed to ``DEFAULT_USER_ID`` and is
shadowed only by a ``users/default/`` agent of the same name — not by an
agent another user happens to own — matching the GitHub registry's
historical discovery (``load_agent_config(name)`` resolves a legacy agent
under ``DEFAULT_USER_ID``).
"""
paths = _ac.get_paths()
discovered: list[tuple[str, str]] = []
seen: set[tuple[str, str]] = set()
users_root = paths.base_dir / "users"
if users_root.exists():
for user_entry in sorted(users_root.iterdir()):
if not user_entry.is_dir():
continue
agents_root = paths.user_agents_dir(user_entry.name)
if not agents_root.exists():
continue
for entry in sorted(agents_root.iterdir()):
if entry.is_dir() and (entry / "config.yaml").exists():
key = (user_entry.name, entry.name)
discovered.append(key)
seen.add(key)
legacy_root = paths.agents_dir
if legacy_root.exists():
for entry in sorted(legacy_root.iterdir()):
if entry.is_dir() and (entry / "config.yaml").exists() and (DEFAULT_USER_ID, entry.name) not in seen:
discovered.append((DEFAULT_USER_ID, entry.name))
return discovered
@staticmethod
def _write(agent_dir: Path, config: dict | None, soul: str | None) -> None:
"""Write config.yaml and/or SOUL.md, each via an atomic ``os.replace``.
Each part is written only when supplied (``config``/``soul`` non-None),
staged to a temp file then committed with ``os.replace``, so neither file
is ever observed half-written. The two commits are sequential, **not** a
single transaction: a crash between them can leave a freshly-replaced
config.yaml beside a stale SOUL.md (single-node, sub-millisecond window).
The ``db`` backend commits both fields in one transaction; if cross-file
atomicity ever matters here, restore ``update_agent``'s partial-write
reporting.
"""
pending: list[tuple[Path, Path]] = []
staged: list[Path] = []
try:
if config is not None:
config_text = yaml.dump(config, default_flow_style=False, allow_unicode=True, sort_keys=False)
config_tmp = _stage_temp(agent_dir / "config.yaml", config_text)
staged.append(config_tmp)
pending.append((config_tmp, agent_dir / "config.yaml"))
if soul is not None:
soul_tmp = _stage_temp(agent_dir / SOUL_FILENAME, soul)
staged.append(soul_tmp)
pending.append((soul_tmp, agent_dir / SOUL_FILENAME))
for tmp, target in pending:
tmp.replace(target)
staged.remove(tmp)
finally:
for tmp in staged:
# A failing discard must not replace the error that aborted the
# publish, nor the result of a successful one.
cleanup_failed_atomic_write(tmp)
def _stage_temp(target: Path, text: str) -> Path:
# newline="" keeps the operator's/agent's bytes: the default text write
# translates LF into os.linesep, so SOUL.md and config.yaml landed with
# CRLF on Windows while every reader saw LF.
with tempfile.NamedTemporaryFile("w", encoding="utf-8", newline="", dir=target.parent, suffix=".tmp", delete=False) as f:
f.write(text)
return Path(f.name)