diff --git a/backend/packages/harness/deerflow/agents/middlewares/skill_context.py b/backend/packages/harness/deerflow/agents/middlewares/skill_context.py index 75163148e4..414e547cf2 100644 --- a/backend/packages/harness/deerflow/agents/middlewares/skill_context.py +++ b/backend/packages/harness/deerflow/agents/middlewares/skill_context.py @@ -15,7 +15,10 @@ from langchain_core.messages import AIMessage, AnyMessage, ToolMessage from deerflow.agents.thread_state import _SKILL_DESCRIPTION_MAX_CHARS, SkillEntry _SKILL_FILE_NAME = "SKILL.md" -_FRONT_MATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.DOTALL) +# `\ufeff` is the UTF-8 byte-order mark a Windows-saved SKILL.md can carry. The loader and +# validator already tolerate it (see `deerflow.skills.frontmatter._FRONTMATTER_RE`); without it +# here the match fails on the leading marker and the captured description comes back empty. +_FRONT_MATTER_RE = re.compile(r"^\ufeff?---\s*\n(.*?)\n---\s*\n", re.DOTALL) SKILL_CONTEXT_ENTRY_KEY = "skill_context_entry" logger = logging.getLogger(__name__) diff --git a/backend/packages/harness/deerflow/skills/frontmatter.py b/backend/packages/harness/deerflow/skills/frontmatter.py index 4c5f96dacd..b1187a2616 100644 --- a/backend/packages/harness/deerflow/skills/frontmatter.py +++ b/backend/packages/harness/deerflow/skills/frontmatter.py @@ -26,7 +26,10 @@ ALLOWED_FRONTMATTER_PROPERTIES = { "author", } -_FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n?", re.DOTALL) +# `\ufeff` is the UTF-8 byte-order mark that Windows Notepad and PowerShell's +# `Set-Content -Encoding UTF8` prepend to a saved file. It is consumed by the match, so +# neither the front matter nor the body carries it downstream. +_FRONTMATTER_RE = re.compile(r"^\ufeff?---\s*\n(.*?)\n---\s*\n?", re.DOTALL) @dataclass(frozen=True) diff --git a/backend/packages/harness/deerflow/skills/parser.py b/backend/packages/harness/deerflow/skills/parser.py index 7c2aa58048..cc2d71fe4c 100644 --- a/backend/packages/harness/deerflow/skills/parser.py +++ b/backend/packages/harness/deerflow/skills/parser.py @@ -4,6 +4,7 @@ from pathlib import Path import yaml +from .frontmatter import _FRONTMATTER_RE from .types import SKILL_MD_FILE, SecretRequirement, Skill, SkillCategory logger = logging.getLogger(__name__) @@ -217,7 +218,7 @@ def parse_skill_file(skill_file: Path, category: SkillCategory, relative_path: P # Keep parser diagnostics richer than the pure helper's host-path-free # error string; tests and authoring UX depend on the line-specific hint. - front_matter_match = re.match(r"^---\s*\n(.*?)\n---\s*\n?", content, re.DOTALL) + front_matter_match = _FRONTMATTER_RE.match(content) if not front_matter_match: return None front_matter_text = front_matter_match.group(1) diff --git a/backend/tests/test_skill_context.py b/backend/tests/test_skill_context.py index 2564547b89..c4e4c60d09 100644 --- a/backend/tests/test_skill_context.py +++ b/backend/tests/test_skill_context.py @@ -57,6 +57,17 @@ class TestExtractSkills: } assert "ALWAYS_USE_PANDAS_SENTINEL" not in repr(entry) + def test_build_skill_entry_metadata_from_read_captures_description_behind_a_bom(self): + entry = build_skill_entry_metadata_from_read( + "/mnt/skills/public/data-analysis/SKILL.md", + f"\ufeff{_SKILL_BODY}", + skills_root=_ROOT, + ) + assert entry == { + "path": "/mnt/skills/public/data-analysis/SKILL.md", + "description": "Analyze data with pandas and charts.", + } + def test_captures_skill_reference_with_description(self): msgs = [ HumanMessage(content="use the analysis skill"), diff --git a/backend/tests/test_skills_parser.py b/backend/tests/test_skills_parser.py index 90f6dbc062..c128a77ad6 100644 --- a/backend/tests/test_skills_parser.py +++ b/backend/tests/test_skills_parser.py @@ -22,12 +22,12 @@ from deerflow.skills.parser import parse_skill_file # --------------------------------------------------------------------------- -def _write_skill(tmp_path: Path, front_matter: str, body: str = "# My Skill\n") -> Path: +def _write_skill(tmp_path: Path, front_matter: str, body: str = "# My Skill\n", encoding: str = "utf-8") -> Path: """Write a minimal SKILL.md and return the path.""" skill_dir = tmp_path / "my-skill" skill_dir.mkdir() skill_file = skill_dir / "SKILL.md" - skill_file.write_text(f"---\n{front_matter}\n---\n{body}", encoding="utf-8") + skill_file.write_text(f"---\n{front_matter}\n---\n{body}", encoding=encoding) return skill_file @@ -417,3 +417,37 @@ def test_parse_unquoted_colon_value_escapes_regex_in_hint(tmp_path, caplog): assert skill is None combined = "\n".join(rec.getMessage() for rec in caplog.records) assert r'description: "match: \\d+ digits"' in combined + + +# --------------------------------------------------------------------------- +# UTF-8 byte-order mark +# --------------------------------------------------------------------------- + + +def test_parse_skill_file_saved_with_utf8_bom(tmp_path): + """A SKILL.md saved as "UTF-8 with BOM" is a skill, not a broken file. + + Windows Notepad and PowerShell's ``Set-Content -Encoding UTF8`` prepend U+FEFF to + every file they save. The front-matter anchor began at ``^---``, so that mark made + ``parse_skill_file`` return ``None`` and the catalog builder dropped the skill with + no log line and no error - the skill simply never existed as far as DeerFlow was + concerned, while the exact same file edited on Linux loaded fine. + """ + skill_file = _write_skill(tmp_path, "name: my-skill\ndescription: A test skill", encoding="utf-8-sig") + assert skill_file.read_bytes().startswith(b"\xef\xbb\xbf"), "fixture must carry a real BOM" + + skill = parse_skill_file(skill_file, category="custom") + assert skill is not None + assert skill.name == "my-skill" + assert skill.description == "A test skill" + + +def test_parse_skill_file_without_bom_is_unaffected(tmp_path): + """Control: the same document written as plain UTF-8 parses exactly as before.""" + skill_file = _write_skill(tmp_path, "name: my-skill\ndescription: A test skill", encoding="utf-8") + assert not skill_file.read_bytes().startswith(b"\xef\xbb\xbf") + + skill = parse_skill_file(skill_file, category="custom") + assert skill is not None + assert skill.name == "my-skill" + assert skill.description == "A test skill" diff --git a/backend/tests/test_skills_validation.py b/backend/tests/test_skills_validation.py index 070adbcaf9..a759f0e427 100644 --- a/backend/tests/test_skills_validation.py +++ b/backend/tests/test_skills_validation.py @@ -8,13 +8,14 @@ from pathlib import Path import pytest +from deerflow.skills.frontmatter import split_skill_markdown from deerflow.skills.validation import ALLOWED_FRONTMATTER_PROPERTIES, _validate_skill_frontmatter -def _write_skill(tmp_path: Path, content: str) -> Path: +def _write_skill(tmp_path: Path, content: str, encoding: str = "utf-8") -> Path: """Write a SKILL.md file and return its parent directory.""" skill_file = tmp_path / "SKILL.md" - skill_file.write_text(content, encoding="utf-8") + skill_file.write_text(content, encoding=encoding) return tmp_path @@ -296,3 +297,43 @@ class TestValidateSkillFrontmatter: assert valid is True assert msg == "Skill is valid!" assert name == "demo-skill" + + def test_valid_when_saved_with_utf8_bom(self, tmp_path): + """A SKILL.md saved as "UTF-8 with BOM" installs like any other file. + + Windows Notepad and PowerShell's ``Set-Content -Encoding UTF8`` prepend U+FEFF, + which used to fall outside the ``^---`` anchor and report "No YAML frontmatter + found" for a file that is byte-for-byte a valid skill. + """ + skill_dir = _write_skill( + tmp_path, + "---\nname: my-skill\ndescription: A valid skill\n---\n\nBody\n", + encoding="utf-8-sig", + ) + assert (skill_dir / "SKILL.md").read_bytes().startswith(b"\xef\xbb\xbf"), "fixture must carry a real BOM" + + valid, msg, name = _validate_skill_frontmatter(skill_dir) + assert valid is True + assert msg == "Skill is valid!" + assert name == "my-skill" + + +class TestSplitSkillMarkdownBom: + def test_bom_is_consumed_and_reaches_neither_metadata_nor_body(self): + """The mark is swallowed by the anchor instead of leaking into the parsed parts.""" + parts, error = split_skill_markdown("\ufeff---\nname: my-skill\ndescription: A valid skill\n---\nBody\n") + + assert error is None + assert parts is not None + assert parts.metadata["name"] == "my-skill" + assert parts.frontmatter_text == "name: my-skill\ndescription: A valid skill" + assert parts.body == "Body\n" + + def test_control_document_without_bom_is_parsed_identically(self): + parts, error = split_skill_markdown("---\nname: my-skill\ndescription: A valid skill\n---\nBody\n") + + assert error is None + assert parts is not None + assert parts.metadata["name"] == "my-skill" + assert parts.frontmatter_text == "name: my-skill\ndescription: A valid skill" + assert parts.body == "Body\n"