From 1333c440ef925948f2d0f730fbc8eb834a807805 Mon Sep 17 00:00:00 2001 From: haochen <3033940900@qq.com> Date: Mon, 28 Sep 2026 15:25:06 +0800 Subject: [PATCH] fix(config): accept UTF-8 BOM in extensions configuration (#5983) * fix(config): accept UTF-8 BOM in extensions configuration * fix(support): accept BOM in extensions config summary --------- Co-authored-by: Zhangchenhao-111 <285911435+Zhangchenhao-111@users.noreply.github.com> Co-authored-by: Willem Jiang --- README.md | 1 + .../harness/deerflow/config/AGENTS.md | 3 ++ .../deerflow/config/extensions_config.py | 4 +-- .../test_extensions_config_raw_writes.py | 33 ++++++++++++++----- backend/tests/test_support_bundle.py | 12 +++++-- scripts/AGENTS.md | 4 +++ scripts/support_bundle.py | 2 +- 7 files changed, 46 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d86722634b..1399679cf0 100644 --- a/README.md +++ b/README.md @@ -660,6 +660,7 @@ Plugin brand icons are bundled locally. When adding or editing one MCP plugin, a Capability Center > Plugins adds, replaces, and deletes one MCP server at a time through targeted mutations that preserve concurrent sibling changes; deletes use a bodyless URL-addressed request. An invalid stdio command on one server no longer blocks toggling another, while enabling that invalid server remains protected by the command allowlist and surfaces the backend validation message in the UI. Targeted updates accept both DeerFlow's `type` field and the MCP-spec `transport` field for SSE/HTTP servers. Runtime MCP and skill updates replace `extensions_config.json` atomically, so an interrupted write cannot leave the shared configuration truncated or partially written. +`extensions_config.json` accepts UTF-8 with or without a leading byte-order mark (BOM), including files saved as UTF-8 with BOM by an editor. MCP routing hints can also prefer a specific MCP tool for matching requests without forbidding other tools. When `tool_search` defers MCP schemas, matching routing metadata can auto-promote up to `tool_search.auto_promote_top_k` deferred schemas before the model call. OpenViking users can register the official Streamable HTTP endpoint at `/mcp` diff --git a/backend/packages/harness/deerflow/config/AGENTS.md b/backend/packages/harness/deerflow/config/AGENTS.md index d5d2726f9c..aa007a6b20 100644 --- a/backend/packages/harness/deerflow/config/AGENTS.md +++ b/backend/packages/harness/deerflow/config/AGENTS.md @@ -75,6 +75,9 @@ not by the top-level infrastructure reload-boundary registry. MCP servers and skills are configured together in `extensions_config.json` in project root: +Both the runtime loader and raw read-modify-write reader accept UTF-8 with or +without a leading BOM. Writes remain UTF-8 without a BOM. + Docker development mounts the project directory at `/app/project` and points `DEER_FLOW_CONFIG_PATH` / `DEER_FLOW_EXTENSIONS_CONFIG_PATH` into that directory. Keep mutable config files behind a directory bind mount: single-file bind mounts diff --git a/backend/packages/harness/deerflow/config/extensions_config.py b/backend/packages/harness/deerflow/config/extensions_config.py index a6ca569142..3a116e8726 100644 --- a/backend/packages/harness/deerflow/config/extensions_config.py +++ b/backend/packages/harness/deerflow/config/extensions_config.py @@ -514,7 +514,7 @@ class ExtensionsConfig(BaseModel): return cls(mcp_servers={}, skills={}) try: - with open(resolved_path, encoding="utf-8") as f: + with open(resolved_path, encoding="utf-8-sig") as f: config_data = json.load(f) config_data = cls.resolve_env_variables(config_data) return cls.model_validate(config_data) @@ -710,7 +710,7 @@ def read_raw_extensions_config(path: Path) -> dict[str, Any]: that message omits the path so API callers can surface it as-is. """ try: - with open(path, encoding="utf-8") as f: + with open(path, encoding="utf-8-sig") as f: raw_data = json.load(f) except json.JSONDecodeError as e: raise ValueError(f"Extensions configuration is not valid JSON: {e.msg} at line {e.lineno} column {e.colno}") from e diff --git a/backend/tests/test_extensions_config_raw_writes.py b/backend/tests/test_extensions_config_raw_writes.py index f825b2813f..d8b86997eb 100644 --- a/backend/tests/test_extensions_config_raw_writes.py +++ b/backend/tests/test_extensions_config_raw_writes.py @@ -56,8 +56,8 @@ def placeholder_env(monkeypatch): monkeypatch.delenv("DEERFLOW_TEST_UNSET_VAR", raising=False) -def _write_json(path: Path, data: object) -> None: - path.write_text(json.dumps(data, indent=2), encoding="utf-8") +def _write_json(path: Path, data: object, encoding: str = "utf-8") -> None: + path.write_text(json.dumps(data, indent=2), encoding=encoding) # --------------------------------------------------------------------------- @@ -65,19 +65,35 @@ def _write_json(path: Path, data: object) -> None: # --------------------------------------------------------------------------- -def test_read_raw_keeps_placeholders(tmp_path: Path, placeholder_env) -> None: +@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"]) +def test_read_raw_keeps_placeholders(tmp_path: Path, placeholder_env, encoding: str) -> None: config_path = tmp_path / "extensions_config.json" - _write_json(config_path, _raw_config_with_placeholders()) + _write_json(config_path, _raw_config_with_placeholders(), encoding) assert read_raw_extensions_config(config_path) == _raw_config_with_placeholders() -def test_read_raw_rejects_invalid_json(tmp_path: Path) -> None: +@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"]) +def test_from_file_resolves_placeholders(tmp_path: Path, placeholder_env, encoding: str) -> None: config_path = tmp_path / "extensions_config.json" - config_path.write_text("{not json", encoding="utf-8") + _write_json(config_path, _raw_config_with_placeholders(), encoding) + + config = ExtensionsConfig.from_file(str(config_path)) + + assert config.mcp_servers["github"].env == {"GITHUB_TOKEN": SECRET, "OPTIONAL": ""} + assert config.mcp_servers["remote"].headers == {"Authorization": SECRET} + assert config.skills["existing-skill"].enabled is True + + +@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"]) +def test_read_raw_rejects_invalid_json(tmp_path: Path, encoding: str) -> None: + config_path = tmp_path / "extensions_config.json" + config_path.write_text("{not json", encoding=encoding) with pytest.raises(ValueError, match="not valid JSON"): read_raw_extensions_config(config_path) + with pytest.raises(ValueError, match="not valid JSON"): + ExtensionsConfig.from_file(str(config_path)) def test_read_raw_rejects_non_object(tmp_path: Path) -> None: @@ -134,9 +150,10 @@ def _patch_gateway_writer(monkeypatch, config_path: Path, cached: ExtensionsConf monkeypatch.setattr(skills_router, "get_extensions_config", lambda: cached or ExtensionsConfig()) -def test_gateway_skill_toggle_preserves_placeholders(tmp_path: Path, monkeypatch, placeholder_env) -> None: +@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"]) +def test_gateway_skill_toggle_preserves_placeholders(tmp_path: Path, monkeypatch, placeholder_env, encoding: str) -> None: config_path = tmp_path / "extensions_config.json" - _write_json(config_path, _raw_config_with_placeholders()) + _write_json(config_path, _raw_config_with_placeholders(), encoding) _patch_gateway_writer(monkeypatch, config_path) skills_router._write_extensions_skill_state(None, "demo-skill", False, rebuild_public_projection=False) diff --git a/backend/tests/test_support_bundle.py b/backend/tests/test_support_bundle.py index 7bcc721ea1..d8a0d6fc61 100644 --- a/backend/tests/test_support_bundle.py +++ b/backend/tests/test_support_bundle.py @@ -399,7 +399,8 @@ def test_create_support_bundle_masks_hardcoded_env_secret(tmp_path): assert env["PROJECT_REF"] == "$SUPABASE_PROJECT_REF" -def test_create_support_bundle_writes_sanitized_zip(tmp_path): +@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"]) +def test_create_support_bundle_writes_sanitized_zip(tmp_path, encoding): project_root = tmp_path / "project" project_root.mkdir() (project_root / "config.yaml").write_text( @@ -439,7 +440,7 @@ channels: }, } ), - encoding="utf-8", + encoding=encoding, ) output_path = tmp_path / "support.zip" @@ -468,6 +469,13 @@ channels: assert "xoxb-secret" not in all_text assert "mcp-secret" not in all_text + extensions_summary = json.loads(_zip_text(bundle_path, "extensions-summary.json")) + assert extensions_summary["mcpServers"]["private"]["env"]["PRIVATE_TOKEN"] == "" + + triage = json.loads(_zip_text(bundle_path, "triage.json")) + assert triage["signals"]["extensions_config_error"] is False + assert not any("fix `extensions_config.json` syntax" in step for step in triage["maintainer_next_steps"]) + config_summary = json.loads(_zip_text(bundle_path, "config-summary.json")) assert config_summary["models"][0]["api_key"] == "" assert config_summary["tools"][0]["api_key"] == "" diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index 9d31ea1425..5bd376ae70 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -295,3 +295,7 @@ bounded, drop-oldest frame queue. WebSocket clients that request The legacy no-parameter protocol still base64-encodes frames into JSON at the Gateway boundary for backward compatibility. Unknown `frame_format` values receive a JSON error and close code 1008. + +The support bundle's `extensions_config.json` reader accepts UTF-8 with or +without a leading BOM, matching the runtime loader. Preserve redaction and +avoid flagging a valid BOM-prefixed file as a syntax error in triage output. diff --git a/scripts/support_bundle.py b/scripts/support_bundle.py index d73624759e..3d689e97a9 100644 --- a/scripts/support_bundle.py +++ b/scripts/support_bundle.py @@ -168,7 +168,7 @@ def _read_json(path: Path) -> Any: if not path.exists(): return {"present": False} try: - return json.loads(path.read_text(encoding="utf-8")) + return json.loads(path.read_text(encoding="utf-8-sig")) except Exception as exc: return {"present": True, "error": f"{type(exc).__name__}: {exc}"}