mirror of
https://github.com/bytedance/deer-flow.git
synced 2026-10-26 13:16:24 +00:00
fix(channels): pin JSON state encoding across platforms (#7228)
* fix(channels): pin JSON state encoding across platforms * test(channels): check encoding pins at each state I/O call --------- Co-authored-by: huyengiang101086-spec <270684046+huyengiang101086-spec@users.noreply.github.com> Co-authored-by: Willem Jiang <willem.jiang@gmail.com>
This commit is contained in:
parent
d4012c4b24
commit
1ad8147947
@ -145,7 +145,7 @@ class BuzzSeenEventStore:
|
||||
# crash mid-write must never truncate the store (a truncated store
|
||||
# would fail open into replay on the next start, which is
|
||||
# recoverable — but there is no reason to accept even that).
|
||||
fh = tempfile.NamedTemporaryFile(mode="w", dir=path.parent, suffix=".tmp", delete=False, encoding="utf-8")
|
||||
fh = tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", newline="", dir=path.parent, suffix=".tmp", delete=False)
|
||||
json.dump(payload, fh)
|
||||
fh.close()
|
||||
Path(fh.name).replace(path)
|
||||
|
||||
@ -227,7 +227,7 @@ class DiscordChannel(Channel):
|
||||
if not self._thread_store_path.exists():
|
||||
logger.debug("[Discord] no thread mappings file at %s", self._thread_store_path)
|
||||
else:
|
||||
data = json.loads(self._thread_store_path.read_text())
|
||||
data = json.loads(self._thread_store_path.read_text(encoding="utf-8"))
|
||||
self._active_threads.clear()
|
||||
self._active_thread_ids.clear()
|
||||
for channel_id, thread_id in data.items():
|
||||
@ -275,7 +275,7 @@ class DiscordChannel(Channel):
|
||||
try:
|
||||
snapshot = dict(self._active_threads)
|
||||
self._thread_store_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._thread_store_path.write_text(json.dumps(snapshot, indent=2))
|
||||
self._thread_store_path.write_text(json.dumps(snapshot, indent=2), encoding="utf-8", newline="")
|
||||
except Exception:
|
||||
logger.exception("[Discord] failed to persist thread mappings")
|
||||
|
||||
|
||||
@ -162,6 +162,7 @@ class ChannelRuntimeConfigStore:
|
||||
fd = tempfile.NamedTemporaryFile(
|
||||
mode="w",
|
||||
encoding="utf-8",
|
||||
newline="",
|
||||
dir=self._path.parent,
|
||||
prefix=credential_staging_prefix(self._path),
|
||||
suffix=".tmp",
|
||||
|
||||
@ -135,7 +135,7 @@ class JsonChannelStore:
|
||||
|
||||
def _save(self) -> None:
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd = tempfile.NamedTemporaryFile(mode="w", dir=self._path.parent, suffix=".tmp", delete=False)
|
||||
fd = tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", newline="", dir=self._path.parent, suffix=".tmp", delete=False)
|
||||
try:
|
||||
json.dump(self._data, fd, indent=2)
|
||||
fd.close()
|
||||
|
||||
@ -1707,7 +1707,7 @@ class WechatChannel(Channel):
|
||||
return
|
||||
try:
|
||||
self._cursor_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._cursor_path.write_text(json.dumps({"get_updates_buf": self._get_updates_buf}, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
self._cursor_path.write_text(json.dumps({"get_updates_buf": self._get_updates_buf}, ensure_ascii=False, indent=2), encoding="utf-8", newline="")
|
||||
except OSError:
|
||||
logger.warning("[WeChat] failed to persist cursor state to %s", self._cursor_path)
|
||||
|
||||
@ -1774,7 +1774,7 @@ class WechatChannel(Channel):
|
||||
# iLink bot_token is never briefly readable at umask defaults
|
||||
# (mirrors ChannelRuntimeConfigStore._save). NamedTemporaryFile
|
||||
# uses mkstemp, which creates the file at 0o600 from the start.
|
||||
fd = tempfile.NamedTemporaryFile(mode="w", dir=self._auth_path.parent, prefix=credential_staging_prefix(self._auth_path), suffix=".tmp", delete=False, encoding="utf-8")
|
||||
fd = tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", newline="", dir=self._auth_path.parent, prefix=credential_staging_prefix(self._auth_path), suffix=".tmp", delete=False)
|
||||
try:
|
||||
json.dump(data, fd, ensure_ascii=False, indent=2)
|
||||
fd.close()
|
||||
|
||||
@ -112,6 +112,8 @@ must fail rather than receive a fabricated IPv4 answer.
|
||||
The local sandbox's UTF-8 subprocess guard inspects each text-mode call with
|
||||
`ast`, checking both `encoding` and `errors`; module-wide literal counts can
|
||||
hide unpinned calls behind unrelated settings.
|
||||
Channel JSON state coverage likewise checks each writer and Discord reader call
|
||||
within its owning class/method; unrelated encoding literals must not mask a regression.
|
||||
|
||||
## PostgreSQL batch fixtures
|
||||
|
||||
|
||||
59
backend/tests/test_channel_state_encoding.py
Normal file
59
backend/tests/test_channel_state_encoding.py
Normal file
@ -0,0 +1,59 @@
|
||||
"""Pin the cross-platform byte contract for IM channel JSON state writers."""
|
||||
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
CHANNELS = Path(__file__).parents[1] / "app" / "channels"
|
||||
|
||||
|
||||
def _state_io_calls(filename: str, class_name: str, method_name: str, call_name: str) -> list[ast.Call]:
|
||||
module = ast.parse((CHANNELS / filename).read_text(encoding="utf-8"))
|
||||
owner = next(node for node in module.body if isinstance(node, ast.ClassDef) and node.name == class_name)
|
||||
method = next(node for node in owner.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == method_name)
|
||||
calls = [node for node in ast.walk(method) if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and node.func.attr == call_name]
|
||||
assert calls, f"{filename}:{class_name}.{method_name} has no {call_name} call"
|
||||
return calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "class_name", "method_name", "call_name"),
|
||||
[
|
||||
("store.py", "JsonChannelStore", "_save", "NamedTemporaryFile"),
|
||||
("discord.py", "DiscordChannel", "_persist_thread_mappings", "write_text"),
|
||||
("runtime_config_store.py", "ChannelRuntimeConfigStore", "_save", "NamedTemporaryFile"),
|
||||
("buzz_seen_events.py", "BuzzSeenEventStore", "_write_snapshot", "NamedTemporaryFile"),
|
||||
("wechat.py", "WechatChannel", "_save_state", "write_text"),
|
||||
("wechat.py", "WechatChannel", "_save_auth_state", "NamedTemporaryFile"),
|
||||
],
|
||||
)
|
||||
def test_json_channel_state_writers_pin_utf8_and_lf(filename: str, class_name: str, method_name: str, call_name: str) -> None:
|
||||
for call in _state_io_calls(filename, class_name, method_name, call_name):
|
||||
keywords = {keyword.arg: keyword.value for keyword in call.keywords}
|
||||
for name, expected in (("encoding", "utf-8"), ("newline", "")):
|
||||
value = keywords.get(name)
|
||||
assert isinstance(value, ast.Constant) and value.value == expected, f"{filename}:{call.lineno} must pin {name}={expected!r}"
|
||||
|
||||
|
||||
def test_json_channel_store_writes_utf8_lf_bytes(tmp_path: Path) -> None:
|
||||
from app.channels.store import JsonChannelStore
|
||||
|
||||
path = tmp_path / "channels" / "store.json"
|
||||
store = JsonChannelStore(path)
|
||||
store._set_sync("微信:资源", "thread-讨论", "用户")
|
||||
|
||||
text = path.read_bytes().decode("utf-8")
|
||||
assert "\r" not in text
|
||||
assert "\\u5fae\\u4fe1" in text
|
||||
entry = json.loads(text)["微信:资源"]
|
||||
assert entry["thread_id"] == "thread-讨论"
|
||||
assert entry["user_id"] == "用户"
|
||||
|
||||
|
||||
def test_discord_state_read_is_explicitly_utf8() -> None:
|
||||
for call in _state_io_calls("discord.py", "DiscordChannel", "_load_active_threads", "read_text"):
|
||||
keywords = {keyword.arg: keyword.value for keyword in call.keywords}
|
||||
encoding = keywords.get("encoding")
|
||||
assert isinstance(encoding, ast.Constant) and encoding.value == "utf-8"
|
||||
Loading…
x
Reference in New Issue
Block a user