fix(backend): stop classifying UTF-16 markdown files as binary (#3966)

* fix: detect utf16 markdown workspace diffs

* fix: tighten binary detection in workspace diff scanner

* fix: decode utf-8-sig before utf-8 to strip bom from diff content

backend/packages/harness/deerflow/workspace_changes/diff.py

@@ -175,7 +175,7 @@ def _snapshot_text(file: FileSnapshot | None) -> str | None:
         try:
             with open(file.text_path, encoding="utf-8") as cached:
                 return cached.read()
-        except (OSError, UnicodeDecodeError):
+        except OSError:
             return None
     return None
 

backend/packages/harness/deerflow/workspace_changes/scanner.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import fnmatch
 import hashlib
 import os
-import shutil
+from codecs import BOM_UTF16_BE, BOM_UTF16_LE, getincrementaldecoder
 from pathlib import Path
 
 from .types import (
@@ -74,6 +74,7 @@ SENSITIVE_PATH_PATTERNS = (
 )
 
 SAMPLE_BYTES = 4096
+_UTF16_BOMS = (BOM_UTF16_LE, BOM_UTF16_BE)
 
 
 def is_sensitive_workspace_path(path: str) -> bool:
@@ -193,16 +194,19 @@ def _snapshot_file(
         reason = "large"
     elif not should_include_text:
         text = None
-    elif text_cache_dir is not None:
-        text_path = str(_cache_text_file(host_file, virtual_path, text_cache_dir))
     else:
         try:
-            text = host_file.read_text(encoding="utf-8")
+            raw = host_file.read_bytes()
-        except UnicodeDecodeError:
-            binary = True
-            reason = "binary"
         except OSError:
             return None
+        decoded = _decode_text_bytes(raw)
+        if decoded is None:
+            binary = True
+            reason = "binary"
+        elif text_cache_dir is not None:
+            text_path = str(_cache_text_file(decoded, virtual_path, text_cache_dir))
+        else:
+            text = decoded
 
     return FileSnapshot(
         path=virtual_path,
@@ -218,10 +222,10 @@ def _snapshot_file(
     )
 
 
-def _cache_text_file(source: Path, virtual_path: str, cache_dir: Path) -> Path:
+def _cache_text_file(text: str, virtual_path: str, cache_dir: Path) -> Path:
     cache_name = hashlib.sha256(virtual_path.encode("utf-8")).hexdigest()
     target = cache_dir / cache_name
-    shutil.copyfile(source, target)
+    target.write_text(text, encoding="utf-8")
     return target
 
 
@@ -238,11 +242,36 @@ def _sha256_file(path: Path) -> str:
     return digest.hexdigest()
 
 
+def _decode_text_bytes(data: bytes) -> str | None:
+    for encoding in ("utf-8-sig", "utf-8"):
+        try:
+            return data.decode(encoding)
+        except UnicodeDecodeError:
+            continue
+
+    if data.startswith(_UTF16_BOMS):
+        try:
+            return data.decode("utf-16")
+        except UnicodeDecodeError:
+            return None
+
+    return None
+
+
+def _sample_decodes_as_text(sample: bytes, encoding: str) -> bool:
+    try:
+        decoder = getincrementaldecoder(encoding)()
+        decoder.decode(sample, final=False)
+    except UnicodeDecodeError:
+        return False
+    return True
+
+
 def _looks_binary(sample: bytes) -> bool:
+    if sample.startswith(_UTF16_BOMS) and _sample_decodes_as_text(sample, "utf-16"):
+        return False
     if b"\x00" in sample:
         return True
-    try:
+    if _sample_decodes_as_text(sample, "utf-8"):
-        sample.decode("utf-8")
+        return False
-    except UnicodeDecodeError:
+    return True
-        return True
-    return False

backend/tests/test_workspace_changes.py

@@ -18,7 +18,7 @@ from deerflow.workspace_changes import (
     scan_workspace_roots,
 )
 from deerflow.workspace_changes.api import get_workspace_changes_response
-from deerflow.workspace_changes.scanner import is_sensitive_workspace_path
+from deerflow.workspace_changes.scanner import SAMPLE_BYTES, is_sensitive_workspace_path
 
 
 def _roots(tmp_path):
@@ -70,6 +70,98 @@ def test_compare_snapshots_reports_text_file_changes(tmp_path):
     assert changes["/mnt/user-data/workspace/old.txt"].status == "deleted"
 
 
+def test_compare_snapshots_treats_utf16_markdown_as_text(tmp_path):
+    roots = _roots(tmp_path)
+    workspace = roots[0].host_path
+
+    before = scan_workspace_roots(roots)
+    (workspace / "guide.md").write_bytes("# 标题\n\nhello\n".encode("utf-16"))
+    after = scan_workspace_roots(roots)
+
+    result = compare_snapshots(before, after)
+
+    change = result.files[0]
+    assert change.path == "/mnt/user-data/workspace/guide.md"
+    assert change.binary is False
+    assert change.diff_unavailable_reason is None
+    assert "+# 标题" in change.diff
+
+
+def test_compare_snapshots_reads_cached_utf16_markdown_baseline(tmp_path):
+    roots = _roots(tmp_path)
+    workspace = roots[0].host_path
+    cache_dir = tmp_path / "cache"
+
+    (workspace / "guide.md").write_bytes("# 标题\n\nhello\n".encode("utf-16"))
+    before = scan_workspace_roots(roots, text_cache_dir=cache_dir)
+    (workspace / "guide.md").write_bytes("# 标题\n\nupdated\n".encode("utf-16"))
+    after = scan_workspace_roots(roots)
+
+    result = compare_snapshots(before, after)
+
+    change = result.files[0]
+    assert change.binary is False
+    assert change.diff_unavailable_reason is None
+    assert "-hello" in change.diff
+    assert "+updated" in change.diff
+
+
+def test_compare_snapshots_treats_utf8_markdown_crossing_sample_boundary_as_text(
+    tmp_path,
+):
+    roots = _roots(tmp_path)
+    workspace = roots[0].host_path
+
+    before = scan_workspace_roots(roots)
+    content = ("a" * (SAMPLE_BYTES - 1)) + "你" + "\nrest\n"
+    (workspace / "guide.md").write_text(content, encoding="utf-8")
+    after = scan_workspace_roots(roots)
+
+    result = compare_snapshots(before, after)
+
+    change = result.files[0]
+    assert change.path == "/mnt/user-data/workspace/guide.md"
+    assert change.binary is False
+    assert "你" in change.diff
+    assert "+rest" in change.diff
+
+
+def test_compare_snapshots_strips_utf8_bom(tmp_path):
+    roots = _roots(tmp_path)
+    workspace = roots[0].host_path
+
+    before = scan_workspace_roots(roots)
+    content = "# 标题\n\nhello\n".encode()
+    (workspace / "guide.md").write_bytes(b"\xef\xbb\xbf" + content)
+    after = scan_workspace_roots(roots)
+
+    result = compare_snapshots(before, after)
+
+    change = result.files[0]
+    assert change.path == "/mnt/user-data/workspace/guide.md"
+    assert change.binary is False
+    assert change.diff_unavailable_reason is None
+    assert "\ufeff" not in change.diff
+    assert "+# 标题" in change.diff
+
+
+def test_compare_snapshots_keeps_nul_bytes_classified_as_binary(tmp_path):
+    roots = _roots(tmp_path)
+    workspace = roots[0].host_path
+
+    before = scan_workspace_roots(roots)
+    (workspace / "guide.md").write_bytes(b"\x00\x01\x02binary\n")
+    after = scan_workspace_roots(roots)
+
+    result = compare_snapshots(before, after)
+
+    change = result.files[0]
+    assert change.path == "/mnt/user-data/workspace/guide.md"
+    assert change.binary is True
+    assert change.diff == ""
+    assert change.diff_unavailable_reason == "binary"
+
+
 def test_count_diff_lines_ignores_only_real_headers():
     from deerflow.workspace_changes.diff import _count_diff_lines