diff --git a/.serena/memories/frontend/translations.md b/.serena/memories/frontend/translations.md index 35bba082d7..5cce5b01fc 100644 --- a/.serena/memories/frontend/translations.md +++ b/.serena/memories/frontend/translations.md @@ -57,14 +57,15 @@ high-coverage support reference, never the base. ## QA before commit -- Run `node ./scripts/check-translations.js -l ` from - `frontend/` (`pnpm run check-translations` covers `ca`): 0 errors +- Run `node ./scripts/translations.js check -l ` from + `frontend/` (no default locale: pass `-l` explicitly): 0 errors required; review warnings by hand. Word lists live in `frontend/scripts/check-translations/words..txt` (`[elision]` `[function]` `[common]` `[ok]` `[brands]`); new valid - words that trip the gate go to `[ok]`; `--self-test` covers the - detector rules. Without a catalog only the universal checks run. - `#, fuzzy` entries are skipped (known-pending, owned elsewhere). + words that trip the gate go to `[ok]`; `check --self-test` + covers the detector rules. Without a catalog only the universal + checks run. `#, fuzzy` entries are skipped (known-pending, + owned elsewhere). - Placeholder parity per entry (singular AND each plural form, also enforced by the script); verify `%s` against the `tr` call site when `en`/`es`/code disagree (a `%s` the code never passes diff --git a/frontend/package.json b/frontend/package.json index d21e1a6945..e39dbf1443 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -27,7 +27,6 @@ "check-fmt:clj": "cljfmt check --parallel=true src/ test/", "check-fmt:js": "prettier -c src/**/*.stories.jsx -c playwright/**/*.js -c scripts/**/*.js -c text-editor/**/*.js", "check-fmt:scss": "prettier -c resources/styles -c src/**/*.scss", - "check-translations": "node ./scripts/check-translations.js -l ca", "fmt:clj": "cljfmt fix --parallel=true src/ test/", "fmt:js": "prettier -c src/**/*.stories.jsx -c playwright/**/*.js -c scripts/**/*.js -c text-editor/**/*.js -w", "fmt:scss": "prettier -c resources/styles -c src/**/*.scss -w", diff --git a/frontend/scripts/check-translations.js b/frontend/scripts/check-translations.js deleted file mode 100644 index 28d646401c..0000000000 --- a/frontend/scripts/check-translations.js +++ /dev/null @@ -1,438 +0,0 @@ -#!/usr/bin/env node -// QA for PO translation files: finds words glued by a missing space, -// broken placeholders and lost plural structures. -// -// Usage (from `frontend/`, like `translations.js`): -// node ./scripts/check-translations.js [-l ] [--self-test] -// -// Exit: 0 with no errors (warnings don't fail), 1 with errors, -// 2 on misuse or unreadable files. -// -// Language data lives in `./scripts/check-translations/words..txt` -// (sections: [elision] [function] [common] [ok] [brands]). Without a -// catalog only the language-independent checks run (placeholders, -// plurals, punctuation, camelCase). - -import getopts from "getopts"; -import { promises as fs } from "node:fs"; -import gt from "gettext-parser"; - -// Brands kept as-is in every locale. -const GENERIC_BRANDS = [ - "GitHub", - "GitLab", - "YouTube", - "InVision", - "innerShadow", - "dropShadow", - "iOS", - "macOS", -]; - -const TOKEN_RE = /[\p{L}\p{M}]+(?:[·'’\-][\p{L}\p{M}]+)*/gu; -const SKIP_RE = /[%{@/\\=<>|#0-9]/; -const PLACEHOLDER_RES = [/%[sd]/g, /\{[^}]*\}/g, /%\([^)]*\)[sd]/g]; -const PUNCT_RE = /[,.:;!?…»)\]]([A-Za-zÀ-Úà-ú«("“‘$])/gu; -const CAMEL_RE = /[a-zàèéíòóúüç·]([A-ZÀÈÉÍÒÓÚÜ][a-zàèéíòóúü]+)/gu; -const PH_GLUED_RE = /%[sd](?=[A-Za-zÀ-Úà-ú])/gu; -const APOSTROPHE_DIGIT_RE = /[a-zàèéíòóúüç·]d['’][0-9]/gu; - -function tokenize(text) { - return [...text.matchAll(TOKEN_RE)].map((m) => m[0].toLowerCase()); -} - -function elisionBase(token, elision) { - if (!elision) return null; - const m = token.match(new RegExp(`^[${elision}]['’](.+)$`)); - return m ? m[1] : null; -} - -function countIn(text, re) { - re.lastIndex = 0; - return [...text.matchAll(re)].length; -} - -// A word is valid as one half of a split. Never the whole token: -// a repeated glue (`del'equip` x3) must not validate itself by frequency. -function validPart(word, words, freq) { - return ( - words.functionWords.has(word) || - words.okWords.has(word) || - words.commonWords.has(word) || - (freq.get(word) ?? 0) >= 1 - ); -} - -// Every (left, right, rule) split of a token, including each -// hyphen-separated segment. -function* splits(tok, words, freq) { - const cands = [tok, ...tok.split("-")]; - const seen = new Set(); - for (const c of cands) { - if (c.length < 3) continue; - for (let k = 1; k < c.length; k++) { - const left = c.slice(0, k); - const right = c.slice(k); - if (left.length < 1) continue; - if (right.length < 2 && !words.functionWords.has(right)) continue; - const key = left + "|" + right; - if (seen.has(key)) continue; - seen.add(key); - const rightBase = elisionBase(right, words.elision) ?? right; - const rightOk = validPart(rightBase, words, freq); - const leftOk = validPart(left, words, freq); - if (words.functionWords.has(left) && rightOk) - yield [left, right, "func-left"]; - else if (words.functionWords.has(right) && right.length <= 4 && leftOk) - yield [left, right, "func-right"]; - else if ( - c.length >= 8 && - left.length >= 3 && - right.length >= 2 && - leftOk && - rightOk - ) - yield [left, right, "content"]; - } - } -} - -const RULE_ORDER = { "func-left": 0, "func-right": 1, content: 2 }; - -function bestSplit(tok, words, freq) { - let best = null; - for (const [left, right, rule] of splits(tok, words, freq)) { - if (left.includes("-")) continue; - if ( - !best || - RULE_ORDER[rule] < RULE_ORDER[best[2]] || - (RULE_ORDER[rule] === RULE_ORDER[best[2]] && left.length > best[0].length) - ) { - best = [left, right, rule]; - } - } - return best; -} - -function folded(s) { - return s.normalize("NFD").replace(/\p{M}/gu, "").toLowerCase(); -} - -function distance(a, b) { - const dp = Array.from({ length: a.length + 1 }, (_, i) => [i]); - for (let j = 1; j <= b.length; j++) dp[0][j] = j; - for (let i = 1; i <= a.length; i++) { - for (let j = 1; j <= b.length; j++) { - dp[i][j] = Math.min( - dp[i - 1][j] + 1, - dp[i][j - 1] + 1, - dp[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1), - ); - } - } - return dp[a.length][b.length]; -} - -// Resemblance to the source: a correct word almost always resembles -// its cognate in `en`/`es`; a glued one never does. A pure affix -// (1-4 letters more or less) doesn't count: that is exactly the -// shape of a glue (`lapolitica` vs `politica`, `desdel` vs `desde`). -function isCognate(tok, refToks, elision) { - const base = tok.replace(new RegExp(`^[${elision || "-"}]['’]`), ""); - const t = folded(base); - const limit = t.length <= 4 ? 0 : 1; - for (const r of refToks) { - const rt = folded(r); - if (Math.abs(rt.length - t.length) > Math.max(limit, 4)) continue; - const d = distance(t, rt); - if (d === 0) return true; - if (d > limit) continue; - if (isAffix(t, rt)) continue; - return true; - } - return false; -} - -function isAffix(t, rt) { - const dif = Math.abs(t.length - rt.length); - if (dif < 1 || dif > 4) return false; - return ( - t.startsWith(rt) || t.endsWith(rt) || rt.startsWith(t) || rt.endsWith(t) - ); -} - -function refToks(...texts) { - const toks = new Set(); - for (const text of texts) for (const t of tokenize(text)) toks.add(t); - return toks; -} - -function cleanContext(frag) { - return !/https?:|www\.|@|\|target:|\.mcp\.json/.test(frag); -} - -function checkPunctuation(text, brands) { - const found = []; - for (const m of text.matchAll(PUNCT_RE)) { - const punct = m[0][0]; - const next = m[1]; - const frag = text.slice(Math.max(0, m.index - 30), m.index + 32); - if (!cleanContext(frag)) continue; - if (m[0] === "](") continue; // markdown link - if (m[0] === ":s" && frag.includes("|target:")) continue; // [x|target:self] - if (/%[sd]\.\(/.test(frag)) continue; // %s.(suffix)... notation - if (punct === "." && !/[A-ZÀÈÉÍÒÓÚÜ«"“(%$]/.test(next)) continue; - found.push({ what: m[0], frag }); - } - for (const m of text.matchAll(CAMEL_RE)) { - const frag = text.slice(Math.max(0, m.index - 30), m.index + 32); - if (brands.some((mk) => frag.includes(mk))) continue; - found.push({ what: m[0], frag }); - } - for (const m of text.matchAll(PH_GLUED_RE)) { - const frag = text.slice(Math.max(0, m.index - 30), m.index + 32); - found.push({ what: m[0], frag }); - } - for (const m of text.matchAll(APOSTROPHE_DIGIT_RE)) { - const frag = text.slice(Math.max(0, m.index - 30), m.index + 32); - found.push({ what: m[0], frag }); - } - return found; -} - -function loadFile(path) { - return fs.readFile(path).then((buf) => gt.po.parse(buf, "utf-8")); -} - -function parseCatalog(text, locale) { - const words = { - functionWords: new Set(), - commonWords: new Set(), - okWords: new Set(), - brands: [], - elision: "", - }; - const sections = { function: 1, common: 1, ok: 1, brands: 1, elision: 1 }; - let current = null; - for (const raw of text.split("\n")) { - const line = raw.trim(); - if (line === "" || line.startsWith("#")) continue; - const sec = line.match(/^\[([a-z]+)\]$/); - if (sec) { - if (!sections[sec[1]]) { - throw new Error(`words.${locale}.txt: unknown section [${sec[1]}]`); - } - current = sec[1]; - continue; - } - if (!current) { - throw new Error(`words.${locale}.txt: word outside any section`); - } - if (current === "elision") { - if (!/^[a-z]+$/.test(line)) { - throw new Error(`words.${locale}.txt: bad [elision] line`); - } - words.elision += line; - } else if (current === "brands") { - words.brands.push(line); - } else { - words[`${current}Words`].add(line.toLowerCase()); - } - } - return words; -} - -async function loadCatalog(locale) { - const path = `./scripts/check-translations/words.${locale}.txt`; - try { - return parseCatalog(await fs.readFile(path, "utf-8"), locale); - } catch (err) { - if (err.code === "ENOENT") return null; - throw err; - } -} - -function isFuzzy(entry) { - return (entry.comments?.flag ?? "").split(/,\s*/).includes("fuzzy"); -} - -async function check(locale, words) { - const [data, dataEn, dataEs] = await Promise.all([ - loadFile(`./translations/${locale}.po`), - loadFile("./translations/en.po"), - locale === "es" ? null : loadFile("./translations/es.po").catch(() => null), - ]); - const entries = data.translations[""]; - const entriesEn = dataEn.translations[""]; - const entriesEs = dataEs ? dataEs.translations[""] : {}; - const brands = [...GENERIC_BRANDS, ...(words?.brands ?? [])]; - const errors = []; - const warnings = []; - - const freq = new Map(); - for (const [msgid, e] of Object.entries(entries)) { - if (msgid === "" || isFuzzy(e)) continue; - for (const t of e.msgstr) - for (const w of tokenize(t)) { - freq.set(w, (freq.get(w) ?? 0) + 1); - } - } - - for (const [msgid, e] of Object.entries(entries)) { - if (msgid === "" || isFuzzy(e)) continue; - const texts = e.msgstr; - const eEn = entriesEn[msgid]; - const textsEn = eEn ? eEn.msgstr : []; - const eEs = entriesEs[msgid]; - const textsEs = eEs ? eEs.msgstr : []; - - if (eEn?.msgid_plural && !e.msgid_plural) { - errors.push( - `${msgid}: source uses msgid_plural but ${locale} lacks msgstr[0]/[1]`, - ); - } - - texts.forEach((text, i) => { - if (!text) return; - const textEn = textsEn[i] ?? ""; - const textEs = textsEs[i] ?? ""; - const refs = refToks(textEn, textEs); - if (textEn) { - for (const re of PLACEHOLDER_RES) { - const nEn = countIn(textEn, re); - const nLoc = countIn(text, re); - if (nEn !== nLoc) { - errors.push( - `${msgid}[${i}]: placeholder mismatch ${re.source}: en=${nEn} ${locale}=${nLoc}`, - ); - } - } - } - for (const t of checkPunctuation(text, brands)) { - errors.push( - `${msgid}: glued punctuation ${JSON.stringify(t.what)} ...${t.frag}...`, - ); - } - if (!words) return; - for (const tok of tokenize(text)) { - if (SKIP_RE.test(tok)) continue; - if (words.okWords.has(tok)) continue; - // No whole-token frequency skip: a repeated glue - // (`del'equip` x3) must never validate itself by frequency. - const base = elisionBase(tok, words.elision); - if (base && validPart(base, words, freq)) continue; - if (isCognate(tok, refs, words.elision)) continue; - const part = bestSplit(tok, words, freq); - if (!part) continue; - const [left, right, rule] = part; - const line = `${msgid}: glued word ${JSON.stringify(tok)} -> ${left} + ${right}`; - if (rule === "content") warnings.push(`${line} (review)`); - else errors.push(line); - } - }); - } - return { errors, warnings }; -} - -const FIXTURES = [ - // [text, expectsError] - ["Els membres del'equip continuaran.", true], - ["Accepteu lapolítica de privadesa.", true], - ["Si necessiteu més informació,contacteu amb nosaltres.", true], - ["Revisions delPenpot disponibles.", true], - ["Seleccioneu Lowercase oCapitalize.", true], - ["Cobreix fins a %seditors nous.", true], - ["Bienvenido acasa nueva.", true], - ["Les biblioteques compartides.", false], - ["Edita el webhook.", false], - ["Emplenament del grup.", false], - ["S'està desant el fitxer.", false], - ["Els components no es poden niar.", false], - ["Gira horitzontalment.", false], - ["Desbloquegeu les funcions.", false], - ["Atributs SVG importats.", false], - ["Commuta la negreta.", false], - ["Bibliotecas compartidas.", false], -]; - -async function selfTest(catalog) { - // Minimal vocabulary for the fixtures. - const freq = new Map( - "els membres continuaran accepteu de privadesa si necessiteu més informació amb nosaltres revisions disponibles seleccioneu lowercase infrequent les biblioteques compartides edita el webhook emplenament del grup està desant fitxer components no es poden niar gira commuta la negreta desbloquegeu les funcions atributs svg importats horitzontalment podeu crear equip política casa bibliotecas compartidas bienvenido nueva" - .split(" ") - .map((w) => [w, 3]), - ); - const words = { - functionWords: catalog.functionWords, - commonWords: new Set(), - okWords: new Set(), - brands: [], - elision: catalog.elision, - }; - let bad = 0; - for (const [text, expectsError] of FIXTURES) { - const found = []; - for (const t of checkPunctuation(text, GENERIC_BRANDS)) - found.push(`punct:${t.what}`); - for (const tok of tokenize(text)) { - if (SKIP_RE.test(tok) || words.functionWords.has(tok)) continue; - if (words.okWords.has(tok)) continue; - const base = elisionBase(tok, words.elision); - if (base && validPart(base, words, freq)) continue; - const part = bestSplit(tok, words, freq); - if (part && part[2] !== "content") found.push(`tok:${tok}`); - } - if (found.length > 0 !== expectsError) { - console.error( - `SELF-TEST FAILED: ${JSON.stringify(text)} expected error=${expectsError}, found=${JSON.stringify(found)}`, - ); - bad++; - } - } - if (bad > 0) process.exit(1); - console.log(`SELF-TEST OK: ${FIXTURES.length} cases`); -} - -const options = getopts(process.argv.slice(2), { - string: ["l"], - boolean: ["self-test", "h"], - alias: { locale: ["l"], help: ["h"] }, -}); - -if (options.h) { - console.log(`PO translation QA. -Usage: node ./scripts/check-translations.js [-l ] [--self-test] - -l: locale to check (default: ca), from frontend/ - --self-test: validate the detector with built-in cases`); - process.exit(0); -} - -if (options["self-test"]) { - const catalog = await loadCatalog("ca"); - if (!catalog) { - console.error("SELF-TEST FAILED: cannot load words.ca.txt"); - process.exit(2); - } - await selfTest(catalog); -} else { - const locale = options.l ?? options.locale ?? "ca"; - const catalog = await loadCatalog(locale); - if (!catalog) { - console.log( - `note: no word catalog for '${locale}', lexical checks skipped`, - ); - } - let result; - try { - result = await check(locale, catalog); - } catch (err) { - console.error(`Could not read translations/${locale}.po: ${err.message}`); - process.exit(2); - } - for (const w of result.warnings) console.log(`warn: ${w}`); - for (const e of result.errors) console.error(`error: ${e}`); - console.log( - `${locale}: ${result.errors.length} errors, ${result.warnings.length} warnings`, - ); - process.exit(result.errors.length > 0 ? 1 : 0); -} diff --git a/frontend/scripts/translations.js b/frontend/scripts/translations.js index 8ee1798567..04b832f332 100755 --- a/frontend/scripts/translations.js +++ b/frontend/scripts/translations.js @@ -1,11 +1,10 @@ #!/usr/bin/env node import getopts from "getopts"; -import { promises as fs, createReadStream } from "fs"; +import { promises as fs, createReadStream } from "node:fs"; import gt from "gettext-parser"; -import l from "lodash"; -import path from "path"; -import readline from "readline"; +import path from "node:path"; +import readline from "node:readline"; const baseLocale = "en"; @@ -24,18 +23,6 @@ async function* getFiles(dir) { } } -async function translationExists(locale) { - const target = path.normalize("./translations/"); - const targetPath = path.join(target, `${locale}.po`); - - try { - const result = await fs.stat(targetPath); - return true; - } catch (cause) { - return false; - } -} - async function readLocaleByPath(path) { const content = await fs.readFile(path); return gt.po.parse(content, "utf-8"); @@ -78,7 +65,7 @@ async function processLocale(options, f) { } else if (locales === undefined) { } else { console.error(`Invalid value found on locales parameter: '${locales}'`); - process.exit(-1); + process.exit(2); } for await (const { name } of scanLocales()) { @@ -336,42 +323,507 @@ async function synchronize(options, ...other) { }); } +// Brands kept as-is in every locale. +const GENERIC_BRANDS = [ + "GitHub", + "GitLab", + "YouTube", + "InVision", + "innerShadow", + "dropShadow", + "iOS", + "macOS", +]; + +const TOKEN_RE = /[\p{L}\p{M}]+(?:[·'’\-][\p{L}\p{M}]+)*/gu; +const SKIP_RE = /[%{@/\\=<>|#0-9]/; +const PLACEHOLDER_RES = [/%[sd]/g, /\{[^}]*\}/g, /%\([^)]*\)[sd]/g]; +const PUNCT_RE = /[,.:;!?…»)\]]([A-Za-zÀ-Úà-ú«("“‘$])/gu; +const CAMEL_RE = /[a-zàèéíòóúüç·]([A-ZÀÈÉÍÒÓÚÜ][a-zàèéíòóúü]+)/gu; +const PH_GLUED_RE = /%[sd](?=[A-Za-zÀ-Úà-ú])/gu; +const APOSTROPHE_DIGIT_RE = /[a-zàèéíòóúüç·]d['’][0-9]/gu; + +function tokenize(text) { + return [...text.matchAll(TOKEN_RE)].map((m) => m[0].toLowerCase()); +} + +function elisionBase(token, elision) { + if (!elision) return null; + const m = token.match(new RegExp(`^[${elision}]['’](.+)$`)); + return m ? m[1] : null; +} + +function countIn(text, re) { + re.lastIndex = 0; + return [...text.matchAll(re)].length; +} + +// A word is valid as one half of a split. Never the whole token: +// a repeated glue (`del'equip` x3) must not validate itself by frequency. +function validPart(word, words, freq) { + return ( + words.functionWords.has(word) || + words.okWords.has(word) || + words.commonWords.has(word) || + (freq.get(word) ?? 0) >= 1 + ); +} + +// Every (left, right, rule) split of a token, including each +// hyphen-separated segment. +function* splits(tok, words, freq) { + const cands = [tok, ...tok.split("-")]; + const seen = new Set(); + for (const c of cands) { + if (c.length < 3) continue; + for (let k = 1; k < c.length; k++) { + const left = c.slice(0, k); + const right = c.slice(k); + if (left.length < 1) continue; + if (right.length < 2 && !words.functionWords.has(right)) continue; + const key = left + "|" + right; + if (seen.has(key)) continue; + seen.add(key); + const rightBase = elisionBase(right, words.elision) ?? right; + const rightOk = validPart(rightBase, words, freq); + const leftOk = validPart(left, words, freq); + if (words.functionWords.has(left) && rightOk) + yield [left, right, "func-left"]; + else if (words.functionWords.has(right) && right.length <= 4 && leftOk) + yield [left, right, "func-right"]; + else if ( + c.length >= 8 && + left.length >= 3 && + right.length >= 2 && + leftOk && + rightOk + ) + yield [left, right, "content"]; + } + } +} + +const RULE_ORDER = { "func-left": 0, "func-right": 1, content: 2 }; + +function bestSplit(tok, words, freq) { + let best = null; + for (const [left, right, rule] of splits(tok, words, freq)) { + if (left.includes("-")) continue; + if ( + !best || + RULE_ORDER[rule] < RULE_ORDER[best[2]] || + (RULE_ORDER[rule] === RULE_ORDER[best[2]] && left.length > best[0].length) + ) { + best = [left, right, rule]; + } + } + return best; +} + +function folded(s) { + return s.normalize("NFD").replace(/\p{M}/gu, "").toLowerCase(); +} + +function distance(a, b) { + const dp = Array.from({ length: a.length + 1 }, (_, i) => [i]); + for (let j = 1; j <= b.length; j++) dp[0][j] = j; + for (let i = 1; i <= a.length; i++) { + for (let j = 1; j <= b.length; j++) { + dp[i][j] = Math.min( + dp[i - 1][j] + 1, + dp[i][j - 1] + 1, + dp[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1), + ); + } + } + return dp[a.length][b.length]; +} + +// Resemblance to the source: a correct word almost always resembles +// its cognate in `en`/`es`; a glued one never does. A pure affix +// (1-4 letters more or less) doesn't count: that is exactly the +// shape of a glue (`lapolitica` vs `politica`, `desdel` vs `desde`). +function isCognate(tok, refToks, elision) { + const base = tok.replace(new RegExp(`^[${elision || "-"}]['’]`), ""); + const t = folded(base); + const limit = t.length <= 4 ? 0 : 1; + for (const r of refToks) { + const rt = folded(r); + if (Math.abs(rt.length - t.length) > Math.max(limit, 4)) continue; + const d = distance(t, rt); + if (d === 0) return true; + if (d > limit) continue; + if (isAffix(t, rt)) continue; + return true; + } + return false; +} + +function isAffix(t, rt) { + const dif = Math.abs(t.length - rt.length); + if (dif < 1 || dif > 4) return false; + return ( + t.startsWith(rt) || t.endsWith(rt) || rt.startsWith(t) || rt.endsWith(t) + ); +} + +function refToks(...texts) { + const toks = new Set(); + for (const text of texts) for (const t of tokenize(text)) toks.add(t); + return toks; +} + +function cleanContext(frag) { + return !/https?:|www\.|@|\|target:|\.mcp\.json/.test(frag); +} + +function checkPunctuation(text, brands) { + const found = []; + for (const m of text.matchAll(PUNCT_RE)) { + const punct = m[0][0]; + const next = m[1]; + const frag = text.slice(Math.max(0, m.index - 30), m.index + 32); + if (!cleanContext(frag)) continue; + if (m[0] === "](") continue; // markdown link + if (m[0] === ":s" && frag.includes("|target:")) continue; // [x|target:self] + if (/%[sd]\.\(/.test(frag)) continue; // %s.(suffix)... notation + if (punct === "." && !/[A-ZÀÈÉÍÒÓÚÜ«"“(%$]/.test(next)) continue; + found.push({ what: m[0], frag }); + } + for (const m of text.matchAll(CAMEL_RE)) { + const frag = text.slice(Math.max(0, m.index - 30), m.index + 32); + if (brands.some((mk) => frag.includes(mk))) continue; + found.push({ what: m[0], frag }); + } + for (const m of text.matchAll(PH_GLUED_RE)) { + const frag = text.slice(Math.max(0, m.index - 30), m.index + 32); + found.push({ what: m[0], frag }); + } + for (const m of text.matchAll(APOSTROPHE_DIGIT_RE)) { + const frag = text.slice(Math.max(0, m.index - 30), m.index + 32); + found.push({ what: m[0], frag }); + } + return found; +} + +function loadFile(path) { + return fs.readFile(path).then((buf) => gt.po.parse(buf, "utf-8")); +} + +function parseCatalog(text, locale) { + const words = { + functionWords: new Set(), + commonWords: new Set(), + okWords: new Set(), + brands: [], + elision: "", + }; + const sections = { function: 1, common: 1, ok: 1, brands: 1, elision: 1 }; + let current = null; + for (const raw of text.split("\n")) { + const line = raw.trim(); + if (line === "" || line.startsWith("#")) continue; + const sec = line.match(/^\[([a-z]+)\]$/); + if (sec) { + if (!sections[sec[1]]) { + throw new Error(`words.${locale}.txt: unknown section [${sec[1]}]`); + } + current = sec[1]; + continue; + } + if (!current) { + throw new Error(`words.${locale}.txt: word outside any section`); + } + if (current === "elision") { + if (!/^[a-z]+$/.test(line)) { + throw new Error(`words.${locale}.txt: bad [elision] line`); + } + words.elision += line; + } else if (current === "brands") { + words.brands.push(line); + } else { + words[`${current}Words`].add(line.toLowerCase()); + } + } + return words; +} + +async function loadCatalog(locale) { + const path = `./scripts/check-translations/words.${locale}.txt`; + try { + return parseCatalog(await fs.readFile(path, "utf-8"), locale); + } catch (err) { + if (err.code === "ENOENT") return null; + throw err; + } +} + +function isFuzzy(entry) { + return (entry.comments?.flag ?? "").split(/,\s*/).includes("fuzzy"); +} + +async function check(locale, words) { + const [data, dataEn, dataEs] = await Promise.all([ + loadFile(`./translations/${locale}.po`), + loadFile("./translations/en.po"), + locale === "es" ? null : loadFile("./translations/es.po").catch(() => null), + ]); + const entries = data.translations[""]; + const entriesEn = dataEn.translations[""]; + const entriesEs = dataEs ? dataEs.translations[""] : {}; + const brands = [...GENERIC_BRANDS, ...(words?.brands ?? [])]; + const errors = []; + const warnings = []; + + const freq = new Map(); + for (const [msgid, e] of Object.entries(entries)) { + if (msgid === "" || isFuzzy(e)) continue; + for (const t of e.msgstr) + for (const w of tokenize(t)) { + freq.set(w, (freq.get(w) ?? 0) + 1); + } + } + + for (const [msgid, e] of Object.entries(entries)) { + if (msgid === "" || isFuzzy(e)) continue; + const texts = e.msgstr; + const eEn = entriesEn[msgid]; + const textsEn = eEn ? eEn.msgstr : []; + const eEs = entriesEs[msgid]; + const textsEs = eEs ? eEs.msgstr : []; + + if (eEn?.msgid_plural && !e.msgid_plural) { + errors.push( + `${msgid}: source uses msgid_plural but ${locale} lacks msgstr[0]/[1]`, + ); + } + + texts.forEach((text, i) => { + if (!text) return; + const textEn = textsEn[i] ?? ""; + const textEs = textsEs[i] ?? ""; + const refs = refToks(textEn, textEs); + if (textEn) { + for (const re of PLACEHOLDER_RES) { + const nEn = countIn(textEn, re); + const nLoc = countIn(text, re); + if (nEn !== nLoc) { + errors.push( + `${msgid}[${i}]: placeholder mismatch ${re.source}: en=${nEn} ${locale}=${nLoc}`, + ); + } + } + } + for (const t of checkPunctuation(text, brands)) { + errors.push( + `${msgid}: glued punctuation ${JSON.stringify(t.what)} ...${t.frag}...`, + ); + } + if (!words) return; + for (const tok of tokenize(text)) { + if (SKIP_RE.test(tok)) continue; + if (words.okWords.has(tok)) continue; + // No whole-token frequency skip: a repeated glue + // (`del'equip` x3) must never validate itself by frequency. + const base = elisionBase(tok, words.elision); + if (base && validPart(base, words, freq)) continue; + if (isCognate(tok, refs, words.elision)) continue; + const part = bestSplit(tok, words, freq); + if (!part) continue; + const [left, right, rule] = part; + const line = `${msgid}: glued word ${JSON.stringify(tok)} -> ${left} + ${right}`; + if (rule === "content") warnings.push(`${line} (review)`); + else errors.push(line); + } + }); + } + return { errors, warnings }; +} + +const FIXTURES = [ + // [text, expectsError] + ["Els membres del'equip continuaran.", true], + ["Accepteu lapolítica de privadesa.", true], + ["Si necessiteu més informació,contacteu amb nosaltres.", true], + ["Revisions delPenpot disponibles.", true], + ["Seleccioneu Lowercase oCapitalize.", true], + ["Cobreix fins a %seditors nous.", true], + ["Bienvenido acasa nueva.", true], + ["Les biblioteques compartides.", false], + ["Edita el webhook.", false], + ["Emplenament del grup.", false], + ["S'està desant el fitxer.", false], + ["Els components no es poden niar.", false], + ["Gira horitzontalment.", false], + ["Desbloquegeu les funcions.", false], + ["Atributs SVG importats.", false], + ["Commuta la negreta.", false], + ["Bibliotecas compartidas.", false], +]; + +async function selfTest(catalog) { + // Minimal vocabulary for the fixtures. + const freq = new Map( + "els membres continuaran accepteu de privadesa si necessiteu més informació amb nosaltres revisions disponibles seleccioneu lowercase infrequent les biblioteques compartides edita el webhook emplenament del grup està desant fitxer components no es poden niar gira commuta la negreta desbloquegeu les funcions atributs svg importats horitzontalment podeu crear equip política casa bibliotecas compartidas bienvenido nueva" + .split(" ") + .map((w) => [w, 3]), + ); + const words = { + functionWords: catalog.functionWords, + commonWords: new Set(), + okWords: new Set(), + brands: [], + elision: catalog.elision, + }; + let bad = 0; + for (const [text, expectsError] of FIXTURES) { + const found = []; + for (const t of checkPunctuation(text, GENERIC_BRANDS)) + found.push(`punct:${t.what}`); + for (const tok of tokenize(text)) { + if (SKIP_RE.test(tok) || words.functionWords.has(tok)) continue; + if (words.okWords.has(tok)) continue; + const base = elisionBase(tok, words.elision); + if (base && validPart(base, words, freq)) continue; + const part = bestSplit(tok, words, freq); + if (part && part[2] !== "content") found.push(`tok:${tok}`); + } + if (found.length > 0 !== expectsError) { + console.error( + `SELF-TEST FAILED: ${JSON.stringify(text)} expected error=${expectsError}, found=${JSON.stringify(found)}`, + ); + bad++; + } + } + if (bad > 0) process.exit(1); + console.log(`SELF-TEST OK: ${FIXTURES.length} cases`); +} + +const HELP_TOP = `PO translation toolkit. +Usage: node ./scripts/translations.js [options] + +Available options (work before or after the command): + + --locale -l : restrict the command to one locale + --verbose -v : verbose output + --help -h : this help, or help for + +Available subcommands (run from \`frontend/\`): +`; + +const COMMANDS = { + rehash: { + args: "", + help: 'Scan ./src for (tr "key") usages and update en.po references.', + run: (options, params) => rehash(options, ...params), + }, + sync: { + args: "[-l ]", + help: "Copy #: references and flags from en.po into each locale.", + run: (options, params) => synchronize(options, ...params), + }, + delete: { + args: " [-l ]", + help: "Delete every entry whose key starts with .", + run: (options, params) => { + if (!params[0]) { + console.error("delete needs a "); + process.exit(2); + } + return deleteByPrefix(options, ...params); + }, + }, + fuzzy: { + args: " [-l ]", + help: "Mark as fuzzy every entry whose key starts with .", + run: (options, params) => { + if (!params[0]) { + console.error("fuzzy needs a "); + process.exit(2); + } + return markFuzzy(options, ...params); + }, + }, + check: { + args: "-l [--self-test]", + help: "QA a locale PO: glued words, placeholders, plurals.", + run: async (options, params) => { + if (options["self-test"]) { + const catalog = await loadCatalog("ca"); + if (!catalog) { + console.error("SELF-TEST FAILED: cannot load words.ca.txt"); + process.exit(2); + } + await selfTest(catalog); + return; + } + const locale = options.l ?? options.locale; + if (!locale) { + console.error("check needs -l "); + process.exit(2); + } + const catalog = await loadCatalog(locale); + if (!catalog) { + console.log( + `note: no word catalog for '${locale}', lexical checks skipped`, + ); + } + let result; + try { + result = await check(locale, catalog); + } catch (err) { + console.error( + `Could not read translations/${locale}.po: ${err.message}`, + ); + process.exit(2); + } + for (const w of result.warnings) console.log(`warn: ${w}`); + for (const e of result.errors) console.error(`error: ${e}`); + console.log( + `${locale}: ${result.errors.length} errors, ${result.warnings.length} warnings`, + ); + process.exit(result.errors.length > 0 ? 1 : 0); + }, + }, +}; + const options = getopts(process.argv.slice(2), { - boolean: ["h", "v"], + boolean: ["h", "v", "self-test"], + string: ["l"], alias: { help: ["h"], locale: ["l"], verbose: ["v"], }, - stopEarly: true, }); const [command, ...params] = options._; -if (command === "rehash") { - await rehash(options, ...params); -} else if (command === "sync") { - await synchronize(options, ...params); -} else if (command === "delete") { - await deleteByPrefix(options, ...params); -} else if (command === "fuzzy") { - await markFuzzy(options, ...params); -} else { - console.log(`Translations manipulation script. -How to use: -./scripts/translation.js - -Available options: - - --locale -l : specify a concrete locale - --verbose -v : enables verbose output - --help -h : prints this help - -Available subcommands: - - rehash : reads and writes all translations files, sorting and validating - sync : synchronize baselocale file with all other locale files - delete : delete all entries that matches the prefix - fuzzy : mark as fuzzy all entries that matches the prefix -`); +function printHelp() { + console.log(HELP_TOP); + for (const [name, cmd] of Object.entries(COMMANDS)) { + console.log(` ${name} ${cmd.args}\n ${cmd.help}\n`); + } +} + +if (!command || options.h || options.help) { + if (command && !COMMANDS[command]) { + console.error(`Unknown command '${command}'.`); + printHelp(); + process.exit(2); + } + if (command) { + const cmd = COMMANDS[command]; + console.log( + `Usage: node ./scripts/translations.js ${command} ${cmd.args}\n\n${cmd.help}`, + ); + } else { + printHelp(); + } +} else if (!COMMANDS[command]) { + console.error(`Unknown command '${command}'.`); + printHelp(); + process.exit(2); +} else { + await COMMANDS[command].run(options, params); }