fix(models): decode CLI credential files as UTF-8 (#6277)

* fix(models): decode CLI credential files as UTF-8

* fix(doctor): accept BOM CLI credentials consistently

README.md

@@ -311,6 +311,7 @@ For Google's official Gemini OpenAI-compatible endpoint, use the
    - Codex CLI reads `~/.codex/auth.json`
    - The Codex model provider returns completed responses without waiting for the SSE connection to close. Failed or incomplete responses report the provider's error or reason; partial output is not returned as a successful answer. Non-object error details are reported as text.
    - Claude Code accepts `CLAUDE_CODE_OAUTH_TOKEN`, `ANTHROPIC_AUTH_TOKEN`, `CLAUDE_CODE_CREDENTIALS_PATH`, or `~/.claude/.credentials.json`
+   - CLI credential JSON files accept UTF-8 with or without a BOM, independently of the host locale. `make doctor` accepts the same files when checking CLI authentication. Invalid text encoding is treated as an unreadable source; Claude Code can still try its default file after an invalid override.
    - ACP agent entries are separate from model providers — if you configure `acp_agents.codex`, point it at a Codex ACP adapter such as `npx -y @zed-industries/codex-acp`
    - Each ACP agent's `timeout_seconds` (default: 1800) is one shared budget for initialization, session creation, and the prompt, starting after the subprocess launches. On timeout, DeerFlow aborts the invocation and closes the subprocess before returning an error. Workspace/MCP preparation and subprocess cleanup are outside this budget. A `TimeoutError` raised by the SDK before this deadline expires retains its own error message.
    - MiniMax Code speaks ACP directly. Install and authenticate it, then add it as an ACP agent:

backend/packages/harness/deerflow/models/AGENTS.md

@@ -58,6 +58,13 @@ the same policy first so run metadata reports the effective values. Design note:
 `tests/test_reasoning_contract.py`, the contract section of
 `tests/test_model_factory.py`, `tests/test_models_router_reasoning.py`.
 
+### CLI credential files (`packages/harness/deerflow/models/credential_loader.py`)
+
+CLI credential JSON uses `utf-8-sig` for locale-independent reads with optional
+BOM. Decode failures follow the existing unreadable-source path, preserving
+Claude's override-to-default fallback. Doctor's mirrored JSON reader must accept
+the same optional BOM. Tests: `test_credential_file_encoding.py`.
+
 ### Codex tool-call/result serialization (`packages/harness/deerflow/models/openai_codex_provider.py`)
 
 `_convert_messages` uses `_is_valid_call_id` for both assistant tool calls

backend/packages/harness/deerflow/models/credential_loader.py

@@ -92,8 +92,8 @@ def _load_json_file(path: Path, label: str) -> dict[str, Any] | None:
         return None
 
     try:
-        return json.loads(path.read_text())
-    except (json.JSONDecodeError, OSError) as e:
+        return json.loads(path.read_text(encoding="utf-8-sig"))
+    except (json.JSONDecodeError, OSError, UnicodeError) as e:
         logger.warning(f"Failed to read {label}: {e}")
         return None
 

backend/tests/test_credential_file_encoding.py

@@ -0,0 +1,115 @@
+"""CLI credential files must decode independently of the host locale."""
+
+import json
+from pathlib import Path
+
+import doctor
+import pytest
+
+from deerflow.models.credential_loader import load_claude_code_credential, load_codex_cli_credential
+
+
+@pytest.fixture(autouse=True)
+def isolate_credential_sources(tmp_path, monkeypatch):
+    monkeypatch.setenv("HOME", str(tmp_path))
+    monkeypatch.setenv("USERPROFILE", str(tmp_path))
+    for name in (
+        "ANTHROPIC_API_KEY",
+        "CLAUDE_CODE_OAUTH_TOKEN",
+        "ANTHROPIC_AUTH_TOKEN",
+        "CLAUDE_CODE_OAUTH_TOKEN_FILE_DESCRIPTOR",
+        "CLAUDE_CODE_CREDENTIALS_PATH",
+        "CODEX_AUTH_PATH",
+    ):
+        monkeypatch.delenv(name, raising=False)
+
+
+@pytest.fixture(params=["claude", "codex", "codex-legacy"])
+def credential_file(request, tmp_path, monkeypatch):
+    path = tmp_path / "credentials.json"
+    if request.param == "claude":
+        monkeypatch.setenv("CLAUDE_CODE_CREDENTIALS_PATH", str(path))
+        payload = {"claudeAiOauth": {"accessToken": "synthetic-token"}}
+        loader = load_claude_code_credential
+    else:
+        monkeypatch.setenv("CODEX_AUTH_PATH", str(path))
+        tokens = {"access_token": "synthetic-token", "account_id": "synthetic-account"}
+        payload = {"tokens": tokens} if request.param == "codex" else tokens
+        loader = load_codex_cli_credential
+    return path, payload, loader
+
+
+@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"])
+def test_cli_credentials_accept_utf8_with_or_without_bom(credential_file, encoding):
+    path, payload, loader = credential_file
+    path.write_text(json.dumps(payload), encoding=encoding)
+
+    credential = loader()
+
+    assert credential is not None
+    assert credential.access_token == "synthetic-token"
+    if hasattr(credential, "account_id"):
+        assert credential.account_id == "synthetic-account"
+
+
+@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"])
+def test_doctor_accepts_the_same_utf8_credential_files(credential_file, encoding):
+    path, payload, loader = credential_file
+    path.write_text(json.dumps(payload), encoding=encoding)
+    provider = "claude_provider:ClaudeChatModel" if loader is load_claude_code_credential else "openai_codex_provider:CodexChatModel"
+    config = path.parent / "config.yaml"
+    config.write_text(f"config_version: 5\nmodels:\n  - name: synthetic\n    use: deerflow.models.{provider}\n    model: synthetic\n", encoding="utf-8")
+
+    assert loader() is not None
+    results = doctor.check_llm_auth(config)
+
+    assert len(results) == 1
+    assert results[0].status == "ok"
+    assert results[0].detail == str(path)
+    assert "synthetic-token" not in results[0].detail
+
+
+def test_cli_credentials_do_not_use_the_host_text_encoding(credential_file, monkeypatch):
+    path, payload, loader = credential_file
+    # U+201D contains byte 0x9d in UTF-8, which is undefined in cp1252.
+    # A valid JSON file can contain Unicode metadata unrelated to authentication.
+    payload["label"] = "开发者”"
+    path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
+    original_open = Path.open
+
+    def legacy_locale_open(self, mode="r", buffering=-1, encoding=None, errors=None, newline=None):
+        if self == path and "b" not in mode and encoding in (None, "locale"):
+            encoding = "cp1252"
+        return original_open(self, mode, buffering, encoding, errors, newline)
+
+    monkeypatch.setattr(Path, "open", legacy_locale_open)
+
+    credential = loader()
+
+    assert credential is not None
+    assert credential.access_token == "synthetic-token"
+
+
+@pytest.mark.parametrize("content", [b"\xff\xfeinvalid", b'{"access_token": "synthetic-token"}\xff', b"not JSON"])
+def test_unreadable_credential_files_degrade_to_no_credential(credential_file, content, caplog):
+    path, _, loader = credential_file
+    path.write_bytes(content)
+
+    assert loader() is None
+    assert "Failed to read" in caplog.text
+    assert "synthetic-token" not in caplog.text
+
+
+def test_claude_falls_back_after_invalid_utf8_override(tmp_path, monkeypatch):
+    override = tmp_path / "invalid.json"
+    override.write_bytes(b"\xff")
+    monkeypatch.setenv("CLAUDE_CODE_CREDENTIALS_PATH", str(override))
+    default = tmp_path / ".claude" / ".credentials.json"
+    default.parent.mkdir()
+    default.write_text(json.dumps({"claudeAiOauth": {"accessToken": "fallback-token"}}), encoding="utf-8")
+
+    credential = load_claude_code_credential()
+
+    assert credential is not None
+    assert credential.access_token == "fallback-token"
+    assert credential.source == "claude-cli-file"

scripts/AGENTS.md

@@ -54,6 +54,12 @@ Gateway's error, and the config-dependent checks skip. Any failure to import
 the harness is reported, never raised: doctor diagnoses broken environments.
 Pinned by `backend/tests/test_doctor.py::TestMainConfigResolution`.
 
+CLI credential JSON checks accept UTF-8 with or without a leading BOM, matching
+the runtime credential loader. Keep `_load_json_object` on `utf-8-sig`; malformed
+JSON and invalid encoding remain missing/invalid sources without exposing tokens.
+Public doctor/runtime agreement is pinned by
+`backend/tests/test_credential_file_encoding.py`.
+
 Root `make install` runs pre-commit through uv, so uv's tool bin directory
 need not be on `PATH`.
 

scripts/doctor.py

@@ -108,7 +108,7 @@ def _load_json_object(path: Path) -> dict | None:
     if not path.is_file():
         return None
     try:
-        value = json.loads(path.read_text(encoding="utf-8"))
+        value = json.loads(path.read_text(encoding="utf-8-sig"))
     except (json.JSONDecodeError, OSError, UnicodeDecodeError):
         return None
     return value if isinstance(value, dict) else None