From 57a2d7fa70d8e3c5a898232dd0ac99c338da5b0d Mon Sep 17 00:00:00 2001 From: Guo-Yixin Date: Mon, 5 Oct 2026 01:47:14 -0500 Subject: [PATCH] fix(models): decode CLI credential files as UTF-8 (#6277) * fix(models): decode CLI credential files as UTF-8 * fix(doctor): accept BOM CLI credentials consistently --- README.md | 1 + .../harness/deerflow/models/AGENTS.md | 7 ++ .../deerflow/models/credential_loader.py | 4 +- .../tests/test_credential_file_encoding.py | 115 ++++++++++++++++++ scripts/AGENTS.md | 6 + scripts/doctor.py | 2 +- 6 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 backend/tests/test_credential_file_encoding.py diff --git a/README.md b/README.md index 5c7918a9fb..0b7927d9c3 100644 --- a/README.md +++ b/README.md @@ -311,6 +311,7 @@ For Google's official Gemini OpenAI-compatible endpoint, use the - Codex CLI reads `~/.codex/auth.json` - The Codex model provider returns completed responses without waiting for the SSE connection to close. Failed or incomplete responses report the provider's error or reason; partial output is not returned as a successful answer. Non-object error details are reported as text. - Claude Code accepts `CLAUDE_CODE_OAUTH_TOKEN`, `ANTHROPIC_AUTH_TOKEN`, `CLAUDE_CODE_CREDENTIALS_PATH`, or `~/.claude/.credentials.json` + - CLI credential JSON files accept UTF-8 with or without a BOM, independently of the host locale. `make doctor` accepts the same files when checking CLI authentication. Invalid text encoding is treated as an unreadable source; Claude Code can still try its default file after an invalid override. - ACP agent entries are separate from model providers — if you configure `acp_agents.codex`, point it at a Codex ACP adapter such as `npx -y @zed-industries/codex-acp` - Each ACP agent's `timeout_seconds` (default: 1800) is one shared budget for initialization, session creation, and the prompt, starting after the subprocess launches. On timeout, DeerFlow aborts the invocation and closes the subprocess before returning an error. Workspace/MCP preparation and subprocess cleanup are outside this budget. A `TimeoutError` raised by the SDK before this deadline expires retains its own error message. - MiniMax Code speaks ACP directly. Install and authenticate it, then add it as an ACP agent: diff --git a/backend/packages/harness/deerflow/models/AGENTS.md b/backend/packages/harness/deerflow/models/AGENTS.md index 5b96772db6..c42a3391c2 100644 --- a/backend/packages/harness/deerflow/models/AGENTS.md +++ b/backend/packages/harness/deerflow/models/AGENTS.md @@ -58,6 +58,13 @@ the same policy first so run metadata reports the effective values. Design note: `tests/test_reasoning_contract.py`, the contract section of `tests/test_model_factory.py`, `tests/test_models_router_reasoning.py`. +### CLI credential files (`packages/harness/deerflow/models/credential_loader.py`) + +CLI credential JSON uses `utf-8-sig` for locale-independent reads with optional +BOM. Decode failures follow the existing unreadable-source path, preserving +Claude's override-to-default fallback. Doctor's mirrored JSON reader must accept +the same optional BOM. Tests: `test_credential_file_encoding.py`. + ### Codex tool-call/result serialization (`packages/harness/deerflow/models/openai_codex_provider.py`) `_convert_messages` uses `_is_valid_call_id` for both assistant tool calls diff --git a/backend/packages/harness/deerflow/models/credential_loader.py b/backend/packages/harness/deerflow/models/credential_loader.py index 440d38d105..649e0afcb2 100644 --- a/backend/packages/harness/deerflow/models/credential_loader.py +++ b/backend/packages/harness/deerflow/models/credential_loader.py @@ -92,8 +92,8 @@ def _load_json_file(path: Path, label: str) -> dict[str, Any] | None: return None try: - return json.loads(path.read_text()) - except (json.JSONDecodeError, OSError) as e: + return json.loads(path.read_text(encoding="utf-8-sig")) + except (json.JSONDecodeError, OSError, UnicodeError) as e: logger.warning(f"Failed to read {label}: {e}") return None diff --git a/backend/tests/test_credential_file_encoding.py b/backend/tests/test_credential_file_encoding.py new file mode 100644 index 0000000000..777e1dc0ce --- /dev/null +++ b/backend/tests/test_credential_file_encoding.py @@ -0,0 +1,115 @@ +"""CLI credential files must decode independently of the host locale.""" + +import json +from pathlib import Path + +import doctor +import pytest + +from deerflow.models.credential_loader import load_claude_code_credential, load_codex_cli_credential + + +@pytest.fixture(autouse=True) +def isolate_credential_sources(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("USERPROFILE", str(tmp_path)) + for name in ( + "ANTHROPIC_API_KEY", + "CLAUDE_CODE_OAUTH_TOKEN", + "ANTHROPIC_AUTH_TOKEN", + "CLAUDE_CODE_OAUTH_TOKEN_FILE_DESCRIPTOR", + "CLAUDE_CODE_CREDENTIALS_PATH", + "CODEX_AUTH_PATH", + ): + monkeypatch.delenv(name, raising=False) + + +@pytest.fixture(params=["claude", "codex", "codex-legacy"]) +def credential_file(request, tmp_path, monkeypatch): + path = tmp_path / "credentials.json" + if request.param == "claude": + monkeypatch.setenv("CLAUDE_CODE_CREDENTIALS_PATH", str(path)) + payload = {"claudeAiOauth": {"accessToken": "synthetic-token"}} + loader = load_claude_code_credential + else: + monkeypatch.setenv("CODEX_AUTH_PATH", str(path)) + tokens = {"access_token": "synthetic-token", "account_id": "synthetic-account"} + payload = {"tokens": tokens} if request.param == "codex" else tokens + loader = load_codex_cli_credential + return path, payload, loader + + +@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"]) +def test_cli_credentials_accept_utf8_with_or_without_bom(credential_file, encoding): + path, payload, loader = credential_file + path.write_text(json.dumps(payload), encoding=encoding) + + credential = loader() + + assert credential is not None + assert credential.access_token == "synthetic-token" + if hasattr(credential, "account_id"): + assert credential.account_id == "synthetic-account" + + +@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig"]) +def test_doctor_accepts_the_same_utf8_credential_files(credential_file, encoding): + path, payload, loader = credential_file + path.write_text(json.dumps(payload), encoding=encoding) + provider = "claude_provider:ClaudeChatModel" if loader is load_claude_code_credential else "openai_codex_provider:CodexChatModel" + config = path.parent / "config.yaml" + config.write_text(f"config_version: 5\nmodels:\n - name: synthetic\n use: deerflow.models.{provider}\n model: synthetic\n", encoding="utf-8") + + assert loader() is not None + results = doctor.check_llm_auth(config) + + assert len(results) == 1 + assert results[0].status == "ok" + assert results[0].detail == str(path) + assert "synthetic-token" not in results[0].detail + + +def test_cli_credentials_do_not_use_the_host_text_encoding(credential_file, monkeypatch): + path, payload, loader = credential_file + # U+201D contains byte 0x9d in UTF-8, which is undefined in cp1252. + # A valid JSON file can contain Unicode metadata unrelated to authentication. + payload["label"] = "开发者”" + path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + original_open = Path.open + + def legacy_locale_open(self, mode="r", buffering=-1, encoding=None, errors=None, newline=None): + if self == path and "b" not in mode and encoding in (None, "locale"): + encoding = "cp1252" + return original_open(self, mode, buffering, encoding, errors, newline) + + monkeypatch.setattr(Path, "open", legacy_locale_open) + + credential = loader() + + assert credential is not None + assert credential.access_token == "synthetic-token" + + +@pytest.mark.parametrize("content", [b"\xff\xfeinvalid", b'{"access_token": "synthetic-token"}\xff', b"not JSON"]) +def test_unreadable_credential_files_degrade_to_no_credential(credential_file, content, caplog): + path, _, loader = credential_file + path.write_bytes(content) + + assert loader() is None + assert "Failed to read" in caplog.text + assert "synthetic-token" not in caplog.text + + +def test_claude_falls_back_after_invalid_utf8_override(tmp_path, monkeypatch): + override = tmp_path / "invalid.json" + override.write_bytes(b"\xff") + monkeypatch.setenv("CLAUDE_CODE_CREDENTIALS_PATH", str(override)) + default = tmp_path / ".claude" / ".credentials.json" + default.parent.mkdir() + default.write_text(json.dumps({"claudeAiOauth": {"accessToken": "fallback-token"}}), encoding="utf-8") + + credential = load_claude_code_credential() + + assert credential is not None + assert credential.access_token == "fallback-token" + assert credential.source == "claude-cli-file" diff --git a/scripts/AGENTS.md b/scripts/AGENTS.md index c0e0341699..b395770346 100644 --- a/scripts/AGENTS.md +++ b/scripts/AGENTS.md @@ -54,6 +54,12 @@ Gateway's error, and the config-dependent checks skip. Any failure to import the harness is reported, never raised: doctor diagnoses broken environments. Pinned by `backend/tests/test_doctor.py::TestMainConfigResolution`. +CLI credential JSON checks accept UTF-8 with or without a leading BOM, matching +the runtime credential loader. Keep `_load_json_object` on `utf-8-sig`; malformed +JSON and invalid encoding remain missing/invalid sources without exposing tokens. +Public doctor/runtime agreement is pinned by +`backend/tests/test_credential_file_encoding.py`. + Root `make install` runs pre-commit through uv, so uv's tool bin directory need not be on `PATH`. diff --git a/scripts/doctor.py b/scripts/doctor.py index a97de499c5..28470bf72f 100644 --- a/scripts/doctor.py +++ b/scripts/doctor.py @@ -108,7 +108,7 @@ def _load_json_object(path: Path) -> dict | None: if not path.is_file(): return None try: - value = json.loads(path.read_text(encoding="utf-8")) + value = json.loads(path.read_text(encoding="utf-8-sig")) except (json.JSONDecodeError, OSError, UnicodeDecodeError): return None return value if isinstance(value, dict) else None