diff --git a/CHANGELOG.md b/CHANGELOG.md index 01d720fbf6..9d270058d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4602,6 +4602,14 @@ This release closes that milestone with **439 merged pull requests**. coverage stays excluded. A new test pins the concurrent startup and the wait-for-every-shard failure reporting with offline worker doubles. ([#6324]) +- **integrations:** Lark/Feishu CLI output is decoded as UTF-8 instead of with + the host locale. `lark-cli` (a native binary shipped through the + `@larksuite/cli` npm package) and npm both write UTF-8 to a pipe, but every + capture in `lark_cli.py` passed `text=True` without an `encoding`, so a host + whose ANSI code page is not UTF-8 (cp936, cp1252) silently mangled non-ASCII + fields — `auth status --json` returned a garbled `userName`, and an + undecodable byte could kill the reader thread and leave `stdout` as `None`, + reporting a healthy CLI as unavailable. ([#6590]) ## [2.1.0] — 2026-09-24 @@ -9464,3 +9472,4 @@ with **180 merged pull requests** since the first 2.0 milestone tag. [#6580]: https://github.com/bytedance/deer-flow/pull/6580 [#6582]: https://github.com/bytedance/deer-flow/pull/6582 [#6588]: https://github.com/bytedance/deer-flow/pull/6588 +[#6590]: https://github.com/bytedance/deer-flow/pull/6590 diff --git a/CHANGELOG_zh.md b/CHANGELOG_zh.md index 3f04c78d86..e20f236390 100644 --- a/CHANGELOG_zh.md +++ b/CHANGELOG_zh.md @@ -3586,6 +3586,13 @@ `make test-shard` 与 CI 的分片方式不变,live 与阻塞 I/O 测试仍被排除。 新增测试用离线 worker 替身固定了分片并行启动与“等待全部分片再报失败” 的行为。([#6324]) +- **集成:** Lark/Feishu CLI 的输出改为按 UTF-8 解码,不再使用宿主 locale。 + `lark-cli`(通过 `@larksuite/cli` npm 包分发的原生二进制)与 npm 都会向管道 + 写入 UTF-8,但 `lark_cli.py` 中的每一处捕获都只传了 `text=True` 而未指定 + `encoding`,因此在 ANSI 代码页非 UTF-8 的宿主上(cp936、cp1252)非 ASCII + 字段会被静默破坏——`auth status --json` 返回的 `userName` 变成乱码,而无法 + 解码的字节还可能让读取线程异常退出、使 `stdout` 变成 `None`,从而把一个正常 + 的 CLI 报告为不可用。([#6590]) ## [2.1.0] — 2026-09-24 @@ -7622,3 +7629,4 @@ DeerFlow 2.0 是围绕"超级智能体"框架的彻底重写,核心包含子 [#6580]: https://github.com/bytedance/deer-flow/pull/6580 [#6582]: https://github.com/bytedance/deer-flow/pull/6582 [#6588]: https://github.com/bytedance/deer-flow/pull/6588 +[#6590]: https://github.com/bytedance/deer-flow/pull/6590 diff --git a/backend/packages/harness/deerflow/integrations/lark_cli.py b/backend/packages/harness/deerflow/integrations/lark_cli.py index c6a9dd4e20..f218499908 100644 --- a/backend/packages/harness/deerflow/integrations/lark_cli.py +++ b/backend/packages/harness/deerflow/integrations/lark_cli.py @@ -97,6 +97,14 @@ INTEGRATION_ID = "lark-cli" FALLBACK_LARK_CLI_VERSION = "v1.0.65" LARK_CLI_NPM_VERSION = FALLBACK_LARK_CLI_VERSION.removeprefix("v") LARK_CLI_NPM_PACKAGE = "@larksuite/cli" +# ``lark-cli`` is a native binary (installed by the npm package's Node wrapper) +# and npm itself always write UTF-8 to a pipe, so every capture of their output +# must be decoded as UTF-8 rather than with the host locale. On a host whose ANSI +# code page is not UTF-8 (cp936, cp1252) the locale decode silently mangles every +# non-ASCII field -- ``auth status --json`` reports a CJK ``userName``, documents +# titles come back garbled -- and can kill the pipe reader thread outright when a +# byte sequence is invalid in the code page, leaving ``stdout`` as ``None``. +LARK_CLI_SUBPROCESS_ENCODING = "utf-8" LARK_CLI_GITHUB_REPO = "larksuite/cli" LARK_CLI_LATEST_RELEASE_API = f"https://api.github.com/repos/{LARK_CLI_GITHUB_REPO}/releases/latest" LARK_CLI_SOURCE_ARCHIVE_ENV = "DEER_FLOW_LARK_CLI_SKILLS_ARCHIVE" @@ -1534,6 +1542,8 @@ def _probe_lark_cli_at_path(path: str) -> LarkCliProbe: check=False, capture_output=True, text=True, + encoding=LARK_CLI_SUBPROCESS_ENCODING, + errors="replace", timeout=5, ) except Exception as exc: # noqa: BLE001 - probe boundary @@ -1569,6 +1579,8 @@ def probe_lark_auth(user_id: str, *, verify: bool = False) -> LarkAuthProbe: check=False, capture_output=True, text=True, + encoding=LARK_CLI_SUBPROCESS_ENCODING, + errors="replace", timeout=8, env=lark_cli_env(user_id), ) @@ -2339,6 +2351,8 @@ def _install_managed_gateway_lark_cli(version: str) -> LarkCliProbe: check=False, capture_output=True, text=True, + encoding=LARK_CLI_SUBPROCESS_ENCODING, + errors="replace", timeout=LARK_CLI_NPM_INSTALL_TIMEOUT_SECONDS, env={**os.environ, "npm_config_update_notifier": "false"}, ) @@ -2491,6 +2505,8 @@ def _run_lark_config_init(*, app_id: str, app_secret: str, brand: str, env: dict check=False, capture_output=True, text=True, + encoding=LARK_CLI_SUBPROCESS_ENCODING, + errors="replace", timeout=15, env=env, ) @@ -2593,6 +2609,8 @@ def _revoke_lark_auth_from_snapshot(snapshot: Path) -> None: check=False, capture_output=True, text=True, + encoding=LARK_CLI_SUBPROCESS_ENCODING, + errors="replace", timeout=15, env=_lark_cli_env_for_directories(config_dir=snapshot / "config", data_dir=data_dir), ) @@ -2619,6 +2637,8 @@ def _run_lark_cli_json( check=False, capture_output=True, text=True, + encoding=LARK_CLI_SUBPROCESS_ENCODING, + errors="replace", timeout=timeout, env=lark_cli_env(user_id), ) diff --git a/backend/tests/test_lark_cli_subprocess_encoding.py b/backend/tests/test_lark_cli_subprocess_encoding.py new file mode 100644 index 0000000000..d5663a0819 --- /dev/null +++ b/backend/tests/test_lark_cli_subprocess_encoding.py @@ -0,0 +1,182 @@ +"""Every capture of ``lark-cli`` / npm output decodes as UTF-8, not the locale. + +``lark-cli`` is a native binary shipped through the ``@larksuite/cli`` npm +package (``bin`` is the Node wrapper ``scripts/run.js``, which ``execFileSync``es +the downloaded binary), and npm writes its own output as UTF-8. Both therefore +put UTF-8 bytes on the pipe regardless of the host's ANSI code page. Decoding +those bytes with ``locale.getencoding()`` -- cp936 on a Chinese Windows host, +cp1252 on a Western one -- silently corrupts every non-ASCII field the CLI +returns (a CJK ``userName`` from ``auth status --json``, a document title) and +can kill the pipe reader thread, which surfaces as ``stdout is None`` rather +than as an exception. + +The tests drive the real functions with a stand-in CLI that emits non-ASCII +UTF-8, and force the locale decoding path without depending on the host code +page: the module's ``subprocess.run`` reference is wrapped so a text-mode call +that names no codec is decoded with one that mangles UTF-8, while a call that +pins its own ``encoding`` is passed through untouched. That reproduces the +behaviour of the pre-fix call sites on a cp936/cp1252 host even when the test +host itself is UTF-8. +""" + +from __future__ import annotations + +import json +import os +import stat +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +from deerflow.config import paths as paths_module +from deerflow.config.paths import Paths +from deerflow.integrations import lark_cli + +# A code page that cannot represent the payload below and is not UTF-8. +MANGLING_CODEPAGE = "cp1252" + +EXPECTED_USER = "\u5f20\u4f1f" # 张伟 +EXPECTED_TITLE = "\u5b63\u5ea6\u590d\u76d8" # 季度复盘 +EXPECTED_VERSION = "lark-cli v1.0.65 \u5f20\u4f1f" # the CLI's --version payload + +_FAKE_CLI_BODY = f"""\ +import json +import os +import sys + +if "--version" in sys.argv[1:]: + os.write(1, {EXPECTED_VERSION!r}.encode("utf-8")) + raise SystemExit(0) + +payload = {{ + "userName": {EXPECTED_USER!r}, + "title": {EXPECTED_TITLE!r}, + "verification_url": "https://open.feishu.cn/verify", +}} +os.write(1, json.dumps(payload, ensure_ascii=False).encode("utf-8")) +""" + + +@pytest.fixture +def fake_cli(tmp_path: Path) -> tuple[list[str], str]: + """A stand-in CLI emitting UTF-8 non-ASCII JSON, as argv and as a path. + + The interpreter path can contain non-ASCII characters (a checkout under a + CJK directory), so the Windows shim is written in the ANSI code page that + ``cmd.exe`` itself uses to read a ``.cmd`` file; otherwise the shim cannot + find its own interpreter. The Python body is always UTF-8. + """ + body = textwrap.dedent(_FAKE_CLI_BODY) + script = tmp_path / "fake_lark_cli.py" + script.write_text(body, encoding="utf-8") + argv = [sys.executable, str(script)] + + if os.name != "nt": + script.chmod(script.stat().st_mode | stat.S_IEXEC) + launcher = tmp_path / "lark-cli" + launcher.write_text(f"#!{sys.executable}\n{body}", encoding="utf-8") + launcher.chmod(launcher.stat().st_mode | stat.S_IEXEC) + return argv, str(launcher) + + launcher = tmp_path / "lark-cli.cmd" + launcher.write_text(f'@echo off\r\n"{sys.executable}" "{script}" %*\r\n', encoding="mbcs") + return argv, str(launcher) + + +@pytest.fixture +def locale_decoding(monkeypatch: pytest.MonkeyPatch) -> None: + """Make ``lark_cli`` capture text the way a non-UTF-8 host code page would. + + A call that names its own codec is passed through untouched, so a pinned + ``encoding="utf-8"`` still wins; a text-mode call that names none is decoded + with a code page that cannot represent non-ASCII -- exactly what happens on + cp936/cp1252 when the locale supplies the codec. Emulating it here keeps + these tests meaningful on a host (or CI runner) whose own locale is UTF-8, + where the unpatched flip would otherwise pass on the pre-fix code too. + + ``locale.getencoding`` is deliberately not patched: ``lark_cli`` never calls + it, so pinning it would document a redirection that does not happen. + """ + real_run = subprocess.run + + def locale_run(*args, **kwargs): + if (kwargs.get("text") or kwargs.get("universal_newlines")) and "encoding" not in kwargs: + kwargs["encoding"] = MANGLING_CODEPAGE + kwargs["errors"] = "replace" + return real_run(*args, **kwargs) + + monkeypatch.setattr(lark_cli.subprocess, "run", locale_run) + + +@pytest.fixture(autouse=True) +def isolated_paths(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Keep the suite off the live runtime home. + + ``_run_lark_cli_json`` and ``probe_lark_auth`` reach + ``ensure_lark_cli_credential_tree`` through the global ``get_paths()`` + singleton, so without this every run would create and harden a real + ``/.deer-flow/users/alice/integrations/lark-cli/{config,data}`` tree -- + the same directory a live deployment keeps real per-user Lark credentials + in -- in whatever directory pytest runs from. The sibling + ``test_lark_cli_integration.py`` isolates it the same way. + """ + monkeypatch.setattr(paths_module, "_paths", Paths(base_dir=tmp_path / "home")) + + +def test_run_lark_cli_json_decodes_utf8_output_regardless_of_locale(fake_cli, locale_decoding: None) -> None: + """``_run_lark_cli_json`` must return the CLI's non-ASCII fields intact.""" + argv, _launcher = fake_cli + + data = lark_cli._run_lark_cli_json([*argv, "auth", "status", "--json"], user_id="alice", timeout=20) + + assert data["userName"] == EXPECTED_USER + assert data["title"] == EXPECTED_TITLE + + +def test_probe_lark_auth_decodes_utf8_user_name(fake_cli, locale_decoding: None, monkeypatch: pytest.MonkeyPatch) -> None: + """The frequently-polled status probe reports the real CJK account name.""" + _argv, launcher = fake_cli + monkeypatch.setattr(lark_cli, "_resolve_lark_cli_path", lambda: launcher) + monkeypatch.setattr(lark_cli, "read_lark_app_config", lambda _user_id: {"configured": True}) + + probe = lark_cli.probe_lark_auth("alice") + + assert probe.status == "authenticated" + assert probe.user == EXPECTED_USER + + +def test_probe_lark_cli_at_path_decodes_utf8_version_output(fake_cli, locale_decoding: None) -> None: + """The probe's reported version keeps a non-ASCII byte intact. + + Asserting only ``available``/``error`` would not guard the pin: the probe + returns ``available=True, version=output or None`` for any non-empty output + with a zero exit code, so a mojibake version string still passes. Naming the + expected text is what fails on the unpinned call site. + """ + _argv, launcher = fake_cli + + probe = lark_cli._probe_lark_cli_at_path(launcher) + + assert probe.available is True + assert probe.error is None + assert probe.version == EXPECTED_VERSION + + +def test_unpinned_capture_corrupts_the_same_output(fake_cli) -> None: + """Control: without the pin, the locale decode corrupts the payload. + + Asserted directly so the failure mode stays documented on hosts whose own + locale happens to be UTF-8 (where this records that the pin is what keeps + the decode correct elsewhere rather than re-testing the corruption). + """ + argv, _launcher = fake_cli + + unpinned = subprocess.run(argv, check=False, capture_output=True, text=True) + raw = unpinned.stdout or "" + if raw.strip() and json.loads(raw)["userName"] == EXPECTED_USER: + # A UTF-8 host locale: the locale decode happens to be correct here. + return + assert EXPECTED_USER not in raw