fix(skills): read and write sub-skill script files as UTF-8 (#561)

html-token-validator.py crashed with UnicodeDecodeError on UTF-8 HTML that
contains bytes cp1252 cannot decode, and generate-slide.py --json turned
non-ASCII text into mojibake, because open()/read_text()/write_text() used the
locale codec (cp1252 on Windows). Pass encoding="utf-8" at every text I/O
call in the sub-skill scripts, add regression tests, and mirror to
cli/assets/skills.
This commit is contained in:
Lupin133 2026-11-07 17:04:11 +01:00 committed by GitHub
parent b7d5353e01
commit 0da327f37b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 176 additions and 16 deletions

View File

@ -64,7 +64,7 @@ def load_brand_colors():
Resolves semantic token references to actual hex values.
"""
try:
with open(TOKENS_PATH) as f:
with open(TOKENS_PATH, encoding='utf-8') as f:
tokens = json.load(f)
colors = tokens.get('primitive', {}).get('color', {})
@ -139,7 +139,7 @@ def load_backgrounds_config():
"""Load background configuration from CSV."""
config = {}
try:
with open(BACKGROUNDS_CSV, newline='') as f:
with open(BACKGROUNDS_CSV, newline='', encoding='utf-8') as f:
reader = csv.DictReader(f)
for row in reader:
config[row['slide_type']] = row

View File

@ -752,7 +752,7 @@ def main():
print(f"Demo deck generated: {output_path}")
elif args.json:
with open(args.json, 'r') as f:
with open(args.json, 'r', encoding='utf-8') as f:
data = json.load(f)
html = generate_deck(data.get('slides', []), data.get('title', 'Presentation'))

View File

@ -111,7 +111,7 @@ def load_css_variables() -> Dict[str, str]:
"""Load CSS variables from design-tokens.css."""
variables = {}
if TOKENS_CSS_PATH.exists():
content = TOKENS_CSS_PATH.read_text()
content = TOKENS_CSS_PATH.read_text(encoding="utf-8")
# Extract --var-name: value patterns
for match in re.finditer(r'(--[\w-]+):\s*([^;]+);', content):
variables[match.group(1)] = match.group(2).strip()
@ -221,7 +221,7 @@ def validate_file(file_path: Path, verbose: bool = False) -> ValidationResult:
result.add_error("File not found")
return result
content = file_path.read_text()
content = file_path.read_text(encoding="utf-8")
return validate_html(content, file_path, verbose)

View File

@ -0,0 +1,76 @@
"""Regression tests: sub-skill scripts must read and write text as UTF-8.
``open()``, ``Path.read_text()`` and ``Path.write_text()`` without an explicit
``encoding`` use the locale codec, which is cp1252 on Windows. A UTF-8 file
containing a byte cp1252 does not map (0x81, as in "ā" or most CJK text) made
html-token-validator.py crash, and generate-slide.py --json silently turned
"Café" into "Café" (mojibake). The subprocess tests below force a non-UTF-8
locale codec (cp1252 on Windows, ASCII on POSIX) so they fail the same way on
any platform.
"""
import json
import os
import subprocess
import sys
from pathlib import Path
SCRIPTS = Path(__file__).resolve().parent.parent
# "ā" is UTF-8 C4 81: byte 0x81 is undefined in cp1252, so reading it with the
# locale codec raises. "Café" is C3 A9: every byte decodes in cp1252, so reading
# it with the locale codec silently yields "Café".
CRASH_TEXT = "ā 你好"
MOJIBAKE_TEXT = "Café"
def _run(*args: str) -> subprocess.CompletedProcess:
env = dict(os.environ)
env.pop("PYTHONIOENCODING", None)
env.update({"PYTHONUTF8": "0", "LC_ALL": "C", "PYTHONCOERCECLOCALE": "0"})
return subprocess.run(
[sys.executable, *args],
capture_output=True,
env=env,
encoding="utf-8",
errors="replace",
)
def test_html_token_validator_reads_utf8_html(tmp_path):
page = tmp_path / "page.html"
page.write_text(
f"<html><body><h1>{CRASH_TEXT}</h1></body></html>", encoding="utf-8"
)
result = _run(str(SCRIPTS / "html-token-validator.py"), str(page))
assert "Traceback" not in result.stderr, result.stderr
# The page is validated and reported (it has no token import), not skipped.
assert result.returncode == 1, result.stderr
assert "HTML DESIGN TOKEN VALIDATION SUMMARY" in result.stdout, result.stdout
assert "page.html" in result.stdout, result.stdout
def test_generate_slide_json_keeps_non_ascii_text(tmp_path):
spec = tmp_path / "deck.json"
spec.write_text(
json.dumps(
{
"title": MOJIBAKE_TEXT,
"slides": [{"type": "title", "title": MOJIBAKE_TEXT}],
},
ensure_ascii=False,
),
encoding="utf-8",
)
out = tmp_path / "deck.html"
result = _run(
str(SCRIPTS / "generate-slide.py"), "--json", str(spec), "--output", str(out)
)
assert result.returncode == 0, result.stderr
html = out.read_text(encoding="utf-8")
assert MOJIBAKE_TEXT in html
assert "Ã" not in html # UTF-8 bytes decoded as cp1252 leave "Ã" behind

View File

@ -49,7 +49,7 @@ class ShadcnInstaller:
return []
try:
with open(self.components_json) as f:
with open(self.components_json, encoding="utf-8") as f:
config = json.load(f)
components_dir = self.project_root / config.get("aliases", {}).get(
@ -69,7 +69,7 @@ class ShadcnInstaller:
pkg_json = self.project_root / "package.json"
if pkg_json.exists():
try:
pkg = json.loads(pkg_json.read_text())
pkg = json.loads(pkg_json.read_text(encoding="utf-8"))
for section in ("dependencies", "devDependencies"):
version = pkg.get(section, {}).get("shadcn")
if version:
@ -128,6 +128,8 @@ class ShadcnInstaller:
cwd=self.project_root,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
check=True,
)
@ -174,6 +176,8 @@ class ShadcnInstaller:
cwd=self.project_root,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
check=True,
)

View File

@ -302,7 +302,7 @@ module.exports = {{
config_content = self.generate_config_string()
self.output_path.write_text(config_content)
self.output_path.write_text(config_content, encoding="utf-8")
return True, f"Configuration written to {self.output_path}"

View File

@ -64,7 +64,7 @@ def load_brand_colors():
Resolves semantic token references to actual hex values.
"""
try:
with open(TOKENS_PATH) as f:
with open(TOKENS_PATH, encoding='utf-8') as f:
tokens = json.load(f)
colors = tokens.get('primitive', {}).get('color', {})
@ -139,7 +139,7 @@ def load_backgrounds_config():
"""Load background configuration from CSV."""
config = {}
try:
with open(BACKGROUNDS_CSV, newline='') as f:
with open(BACKGROUNDS_CSV, newline='', encoding='utf-8') as f:
reader = csv.DictReader(f)
for row in reader:
config[row['slide_type']] = row

View File

@ -752,7 +752,7 @@ def main():
print(f"Demo deck generated: {output_path}")
elif args.json:
with open(args.json, 'r') as f:
with open(args.json, 'r', encoding='utf-8') as f:
data = json.load(f)
html = generate_deck(data.get('slides', []), data.get('title', 'Presentation'))

View File

@ -111,7 +111,7 @@ def load_css_variables() -> Dict[str, str]:
"""Load CSS variables from design-tokens.css."""
variables = {}
if TOKENS_CSS_PATH.exists():
content = TOKENS_CSS_PATH.read_text()
content = TOKENS_CSS_PATH.read_text(encoding="utf-8")
# Extract --var-name: value patterns
for match in re.finditer(r'(--[\w-]+):\s*([^;]+);', content):
variables[match.group(1)] = match.group(2).strip()
@ -221,7 +221,7 @@ def validate_file(file_path: Path, verbose: bool = False) -> ValidationResult:
result.add_error("File not found")
return result
content = file_path.read_text()
content = file_path.read_text(encoding="utf-8")
return validate_html(content, file_path, verbose)

View File

@ -0,0 +1,76 @@
"""Regression tests: sub-skill scripts must read and write text as UTF-8.
``open()``, ``Path.read_text()`` and ``Path.write_text()`` without an explicit
``encoding`` use the locale codec, which is cp1252 on Windows. A UTF-8 file
containing a byte cp1252 does not map (0x81, as in "ā" or most CJK text) made
html-token-validator.py crash, and generate-slide.py --json silently turned
"Café" into "Café" (mojibake). The subprocess tests below force a non-UTF-8
locale codec (cp1252 on Windows, ASCII on POSIX) so they fail the same way on
any platform.
"""
import json
import os
import subprocess
import sys
from pathlib import Path
SCRIPTS = Path(__file__).resolve().parent.parent
# "ā" is UTF-8 C4 81: byte 0x81 is undefined in cp1252, so reading it with the
# locale codec raises. "Café" is C3 A9: every byte decodes in cp1252, so reading
# it with the locale codec silently yields "Café".
CRASH_TEXT = "ā 你好"
MOJIBAKE_TEXT = "Café"
def _run(*args: str) -> subprocess.CompletedProcess:
env = dict(os.environ)
env.pop("PYTHONIOENCODING", None)
env.update({"PYTHONUTF8": "0", "LC_ALL": "C", "PYTHONCOERCECLOCALE": "0"})
return subprocess.run(
[sys.executable, *args],
capture_output=True,
env=env,
encoding="utf-8",
errors="replace",
)
def test_html_token_validator_reads_utf8_html(tmp_path):
page = tmp_path / "page.html"
page.write_text(
f"<html><body><h1>{CRASH_TEXT}</h1></body></html>", encoding="utf-8"
)
result = _run(str(SCRIPTS / "html-token-validator.py"), str(page))
assert "Traceback" not in result.stderr, result.stderr
# The page is validated and reported (it has no token import), not skipped.
assert result.returncode == 1, result.stderr
assert "HTML DESIGN TOKEN VALIDATION SUMMARY" in result.stdout, result.stdout
assert "page.html" in result.stdout, result.stdout
def test_generate_slide_json_keeps_non_ascii_text(tmp_path):
spec = tmp_path / "deck.json"
spec.write_text(
json.dumps(
{
"title": MOJIBAKE_TEXT,
"slides": [{"type": "title", "title": MOJIBAKE_TEXT}],
},
ensure_ascii=False,
),
encoding="utf-8",
)
out = tmp_path / "deck.html"
result = _run(
str(SCRIPTS / "generate-slide.py"), "--json", str(spec), "--output", str(out)
)
assert result.returncode == 0, result.stderr
html = out.read_text(encoding="utf-8")
assert MOJIBAKE_TEXT in html
assert "Ã" not in html # UTF-8 bytes decoded as cp1252 leave "Ã" behind

View File

@ -49,7 +49,7 @@ class ShadcnInstaller:
return []
try:
with open(self.components_json) as f:
with open(self.components_json, encoding="utf-8") as f:
config = json.load(f)
components_dir = self.project_root / config.get("aliases", {}).get(
@ -69,7 +69,7 @@ class ShadcnInstaller:
pkg_json = self.project_root / "package.json"
if pkg_json.exists():
try:
pkg = json.loads(pkg_json.read_text())
pkg = json.loads(pkg_json.read_text(encoding="utf-8"))
for section in ("dependencies", "devDependencies"):
version = pkg.get(section, {}).get("shadcn")
if version:
@ -128,6 +128,8 @@ class ShadcnInstaller:
cwd=self.project_root,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
check=True,
)
@ -174,6 +176,8 @@ class ShadcnInstaller:
cwd=self.project_root,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
check=True,
)

View File

@ -302,7 +302,7 @@ module.exports = {{
config_content = self.generate_config_string()
self.output_path.write_text(config_content)
self.output_path.write_text(config_content, encoding="utf-8")
return True, f"Configuration written to {self.output_path}"