From 0dd12b09e9d21f30832b09d4be936154bd19da0a Mon Sep 17 00:00:00 2001 From: "alonso.torres" Date: Mon, 19 Oct 2026 18:03:54 +0200 Subject: [PATCH] :sparkles: Add a CJK text layout proof of concept Choose paragraph layout rules from an optional language tag. Pass language to horizontal and vertical shaping, add basic Chinese punctuation rules, and add experimental Korean spaces-only wrapping. Keep text boundaries grapheme-safe and preserve paragraph profiles across editor splits and merges. Add reproducible font fixtures for localized glyphs and regression tests for the shared WASM protocol and layout paths. Validation: renderer, common JVM/JS, frontend and exporter tests pass. Both WASM targets build. Full CI still reports baseline lint and format errors and one backend multipart-upload test failure. AI-assisted-by: GPT-6 --- .../app/common/render_wasm/text_content.cljs | 42 ++++--- common/src/app/common/types/shape/text.cljc | 6 + common/src/app/common/types/text.cljc | 1 + common/test/common_tests/types/text_test.cljc | 13 ++ frontend/src/app/render_wasm/text_editor.cljs | 8 +- .../render_wasm/texts_test.cljs | 43 +++++++ render-wasm/scripts/build-cjk-test-font.py | 67 ++++++++++ render-wasm/src/fonts/cjk-locl-test.ttf | Bin 0 -> 1480 bytes render-wasm/src/render/text.rs | 3 +- render-wasm/src/shapes.rs | 1 + render-wasm/src/shapes/japanese.rs | 20 --- render-wasm/src/shapes/kinsoku.rs | 49 +++++--- render-wasm/src/shapes/text.rs | 58 ++++++++- render-wasm/src/shapes/text_horizontal/mod.rs | 25 +++- .../src/shapes/text_horizontal/ruby.rs | 3 + .../src/shapes/text_horizontal/tests.rs | 118 +++++++++++++++++- render-wasm/src/shapes/text_policy.rs | 85 +++++++++++++ .../src/shapes/text_vertical/annotations.rs | 2 + render-wasm/src/shapes/text_vertical/cells.rs | 21 +++- render-wasm/src/shapes/text_vertical/flow.rs | 68 ++++++++++ .../src/shapes/text_vertical/layout.rs | 108 +++++++++++++++- .../src/shapes/text_vertical/orientation.rs | 15 ++- .../src/shapes/text_vertical/shaping.rs | 40 +++++- .../src/shapes/text_vertical/test_support.rs | 5 +- render-wasm/src/wasm/text.rs | 34 ++++- render-wasm/src/wasm/text/helpers.rs | 73 ++++++++--- render-wasm/src/wasm/text_editor.rs | 7 +- 27 files changed, 819 insertions(+), 96 deletions(-) create mode 100644 render-wasm/scripts/build-cjk-test-font.py create mode 100644 render-wasm/src/fonts/cjk-locl-test.ttf create mode 100644 render-wasm/src/shapes/text_policy.rs diff --git a/common/src/app/common/render_wasm/text_content.cljs b/common/src/app/common/render_wasm/text_content.cljs index 2aec0df710..fd643749af 100644 --- a/common/src/app/common/render_wasm/text_content.cljs +++ b/common/src/app/common/render_wasm/text_content.cljs @@ -6,8 +6,9 @@ (ns app.common.render-wasm.text-content "Single source of truth for writing a text shape's content into the WASM design - state. The binary layout ([num-spans][paragraph attrs][span attrs][text][ruby-text]) is - identical for the workspace and the headless exporter, and so is the font-id + state. The binary layout is [num-spans][paragraph attrs][span attrs][text] + [ruby-text][language], identical for the workspace and headless exporter. + Both hosts also share the font-id -> uuid mapping (`cfnt/font-id->uuid`). Only *variant* resolution differs — the workspace has a loaded fonts DB, the exporter does not — so that part is injected via the `opts` map passed to `write-shape-text!`. @@ -24,7 +25,7 @@ [app.common.uuid :as uuid] [cuerdas.core :as str])) -(def ^:const PARAGRAPH-ATTR-U8-SIZE 16) +(def ^:const PARAGRAPH-ATTR-U8-SIZE 20) (def ^:const SPAN-ATTR-U8-SIZE 80) (def ^:const MAX-TEXT-FILLS types.fills.impl/MAX-FILLS) @@ -126,11 +127,12 @@ (mem/write-u8 dview text-transform) (mem/write-u8 dview writing-mode) (mem/write-u8 dview text-orientation) + (mem/write-u8 dview (if (= "spaces-only" (:korean-wrap paragraph)) 1 0)) ;; Alignment padding; must match RawParagraphData in Rust. (mem/write-u8 dview 0) - (mem/write-u8 dview 0) (mem/write-f32 dview line-height) (mem/write-f32 dview letter-spacing) + (mem/write-u32 dview (mem/size (encode-text (d/nilv (:text-language paragraph) "")))) (mem/assert-written offset PARAGRAPH-ATTR-U8-SIZE)))) (defn- write-spans @@ -225,24 +227,26 @@ :or {normalize-font-id cfnt/font-id->uuid normalize-paragraph identity normalize-span (fn [span _paragraph] span)}}] - (let [paragraph (normalize-paragraph paragraph) - spans (map #(normalize-span % paragraph) spans) - num-spans (count spans) - fills-size (* types.fills.impl/FILL-U8-SIZE MAX-TEXT-FILLS) - metadata-size (+ PARAGRAPH-ATTR-U8-SIZE - (* num-spans (+ SPAN-ATTR-U8-SIZE fills-size))) - text-buffer (encode-text text) - text-size (mem/size text-buffer) - ruby-buffer (encode-text (apply str (map span-ruby spans))) - ruby-size (mem/size ruby-buffer) - total-size (+ 4 metadata-size text-size ruby-size) - heapu8 (mem/get-heap-u8) - dview (mem/get-data-view) - offset (mem/alloc total-size)] + (let [paragraph (normalize-paragraph paragraph) + spans (map #(normalize-span % paragraph) spans) + num-spans (count spans) + fills-size (* types.fills.impl/FILL-U8-SIZE MAX-TEXT-FILLS) + metadata-size (+ PARAGRAPH-ATTR-U8-SIZE + (* num-spans (+ SPAN-ATTR-U8-SIZE fills-size))) + text-buffer (encode-text text) + text-size (mem/size text-buffer) + ruby-buffer (encode-text (apply str (map span-ruby spans))) + ruby-size (mem/size ruby-buffer) + language-buffer (encode-text (d/nilv (:text-language paragraph) "")) + total-size (+ 4 metadata-size text-size ruby-size (mem/size language-buffer)) + heapu8 (mem/get-heap-u8) + dview (mem/get-data-view) + offset (mem/alloc total-size)] (-> offset (mem/write-u32 dview num-spans) (write-paragraph dview paragraph) (write-spans dview spans paragraph normalize-font-id) (mem/write-buffer heapu8 text-buffer) - (mem/write-buffer heapu8 ruby-buffer)) + (mem/write-buffer heapu8 ruby-buffer) + (mem/write-buffer heapu8 language-buffer)) (h/call wasm/internal-module "_set_shape_text_content"))) diff --git a/common/src/app/common/types/shape/text.cljc b/common/src/app/common/types/shape/text.cljc index b17ddbd77c..ddc8982ffa 100644 --- a/common/src/app/common/types/shape/text.cljc +++ b/common/src/app/common/types/shape/text.cljc @@ -44,6 +44,12 @@ [:font-style {:optional true} ::sm/text] [:font-weight {:optional true} ::sm/text] [:direction {:optional true} ::sm/text] + ;; Paragraph language selects typography; it never follows the UI locale. + [:text-language {:optional true} + [:re {:gen/elements ["ja" "zh-Hans" "zh-Hant-TW" "ko" "en-GB"]} + #"^(?:[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*|[xXiI](?:-[A-Za-z0-9]{1,8})+)$"]] + ;; Experimental POC mode, deliberately distinct from CSS keep-all. + [:korean-wrap {:optional true} [:enum "syllable" "spaces-only"]] [:writing-mode {:optional true} (schema:jl-enum :writing-mode)] [:text-orientation {:optional true} (schema:jl-enum :text-orientation)] [:text-decoration {:optional true} ::sm/text] diff --git a/common/src/app/common/types/text.cljc b/common/src/app/common/types/text.cljc index 40f70afc16..3173142967 100644 --- a/common/src/app/common/types/text.cljc +++ b/common/src/app/common/types/text.cljc @@ -108,6 +108,7 @@ (d/concat-vec text-align-attrs text-direction-attrs + [:text-language :korean-wrap] cjk/whole-shape-paragraph-attrs)) (def text-node-attrs diff --git a/common/test/common_tests/types/text_test.cljc b/common/test/common_tests/types/text_test.cljc index ab7f15a668..f8f5b3129f 100644 --- a/common/test/common_tests/types/text_test.cljc +++ b/common/test/common_tests/types/text_test.cljc @@ -379,6 +379,19 @@ [attrs] (update-in content-base [:children 0 :children 0] merge attrs)) +(t/deftest paragraph-language-and-experimental-korean-wrap + (doseq [language ["ja" "zh-Hans" "zh-Hant-TW" "ko" "en-GB"]] + (t/is (ctst/valid-content? + (with-paragraph-attrs {:text-language language + :korean-wrap "spaces-only"})))) + (doseq [attrs [{:text-language 42} + {:text-language "zh_Hans"} + {:text-language ""} + {:korean-wrap "keep-all"}]] + (t/is (not (ctst/valid-content? (with-paragraph-attrs attrs))) (pr-str attrs))) + (t/is (contains? (set cttx/paragraph-attrs) :text-language)) + (t/is (not (contains? (set cjk/whole-shape-paragraph-attrs) :text-language)))) + (t/deftest japanese-span-attrs-accept-supported-values (t/is (ctst/valid-content? (with-span-attrs {:text-combine-upright "digits2" diff --git a/frontend/src/app/render_wasm/text_editor.cljs b/frontend/src/app/render_wasm/text_editor.cljs index c0af201b4d..6307e5f184 100644 --- a/frontend/src/app/render_wasm/text_editor.cljs +++ b/frontend/src/app/render_wasm/text_editor.cljs @@ -610,7 +610,10 @@ styling is taken from there; a span WASM never saw falls back to its position and then to the last existing span. - exported vector of paragraphs, each a vector of `{:p 0 :s 0 :t \"text\"}` + `:pp` tracks paragraph attrs independently of span styling, including + when a merge removes all the surviving paragraph's original characters. + + exported vector of paragraphs, each a vector of `{:pp 0 :p 0 :s 0 :t \"text\"}` content existing Penpot content map (root -> paragraph-set -> …)" [content exported] (let [para-set (first (get content :children)) @@ -621,7 +624,8 @@ styling-para (fn [para-idx spans] - (or (get orig-paras (get (first spans) :p)) + (or (get orig-paras (get (first spans) :pp)) + (get orig-paras (get (first spans) :p)) (get orig-paras para-idx) last-orig-para)) diff --git a/frontend/test/frontend_tests/render_wasm/texts_test.cljs b/frontend/test/frontend_tests/render_wasm/texts_test.cljs index ba9f87db8e..fa7fc49224 100644 --- a/frontend/test/frontend_tests/render_wasm/texts_test.cljs +++ b/frontend/test/frontend_tests/render_wasm/texts_test.cljs @@ -314,6 +314,49 @@ (t/is (not-any? #(contains? % :text-orientation) paragraphs)) (t/is (= [nil "center" nil] (mapv :text-align paragraphs))))) +(t/deftest language-and-wrapping-only-change-selected-paragraphs + (let [content (editor-content [{:text "日本"}] [{:text "한국어"}] [{:text "中文"}]) + result (text-editor/apply-paragraph-attrs-to-range + content (selection 1 0 1 0) + {:text-language "ko" :korean-wrap "spaces-only"}) + paragraphs (-> result :children first :children)] + (t/is (= [nil "ko" nil] (mapv :text-language paragraphs))) + (t/is (= [nil "spaces-only" nil] (mapv :korean-wrap paragraphs))) + (t/is (= result + (text-editor/merge-exported-texts-into-content + result [[{:p 0 :s 0 :t "日本"}] + [{:p 1 :s 0 :t "한국어"}] + [{:p 2 :s 0 :t "中文"}]])))) + (let [content (assoc-in (editor-content [{:text "한글"}]) + [:children 0 :children 0 :text-language] "ko") + result (text-editor/merge-exported-texts-into-content + content [[{:p 0 :s 0 :t "한"}] [{:p 0 :s 0 :t "글"}]])] + (t/is (= ["ko" "ko"] (mapv :text-language (-> result :children first :children)))))) + +(t/deftest paragraph-language-binary-header-matches-rust + (let [dview (js/DataView. (js/ArrayBuffer. tc/PARAGRAPH-ATTR-U8-SIZE)) + end (@#'tc/write-paragraph 0 dview + {:text-language "zh-Hant-TW" :korean-wrap "spaces-only" + :line-height "1.5" :letter-spacing "2"})] + (t/is (= 20 end)) + (t/is (= 1 (.getUint8 dview 6))) + (t/is (= 0 (.getUint8 dview 7))) + (t/is (= 1.5 (.getFloat32 dview 8 true))) + (t/is (= 2 (.getFloat32 dview 12 true))) + (t/is (= 10 (.getUint32 dview 16 true))))) + +(t/deftest merging-away-original-spans-keeps-the-surviving-paragraph-profile + (let [content (-> (editor-content [{:text "日"}] [{:text "한"}]) + (assoc-in [:children 0 :children 0 :text-language] "ja") + (assoc-in [:children 0 :children 1 :text-language] "ko") + (assoc-in [:children 0 :children 1 :korean-wrap] "spaces-only")) + result (text-editor/merge-exported-texts-into-content + content [[{:pp 0 :p 1 :s 0 :t "한"}]]) + para (get-in result [:children 0 :children 0])] + (t/is (= "ja" (:text-language para))) + (t/is (not (contains? para :korean-wrap))) + (t/is (= "한" (get-in para [:children 0 :text]))))) + ;; Byte offsets inside one span's attribute block (see `write-spans` and ;; RawTextSpan in render-wasm). (def ^:private span-attr-offset diff --git a/render-wasm/scripts/build-cjk-test-font.py b/render-wasm/scripts/build-cjk-test-font.py new file mode 100644 index 0000000000..b9e8281b9e --- /dev/null +++ b/render-wasm/scripts/build-cjk-test-font.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Build the synthetic CJK language fixture (requires fonttools). + +The one Han character has distinct JAN, ZHS and KOR localized glyphs and +advances. This tests language propagation without a system font dependency. +All outlines are original rectangles; no external font data is included. +""" + +from pathlib import Path + +from fontTools.feaLib.builder import addOpenTypeFeaturesFromString +from fontTools.fontBuilder import FontBuilder +from fontTools.pens.ttGlyphPen import TTGlyphPen + + +def main(): + builder = FontBuilder(1000, isTTF=True) + names = [".notdef", "base", "japanese", "chinese", "korean", "square", "space", "thirdEm"] + builder.setupGlyphOrder(names) + cmap = {ord(char): "square" for char in "文あっ한국어문장한글한「」，。"} + cmap.update({0x4E2D: "base", 0x20: "space", 0x2004: "thirdEm"}) + builder.setupCharacterMap(cmap) + glyphs = {} + for index, name in enumerate(names): + pen = TTGlyphPen(None) + pen.moveTo((50, 0)) + pen.lineTo((350 + index * 20, 0)) + pen.lineTo((350 + index * 20, 700)) + pen.lineTo((50, 700)) + pen.closePath() + glyphs[name] = pen.glyph() + builder.setupGlyf(glyphs) + metrics = {name: (400 + index * 100, 50) for index, name in enumerate(names)} + metrics.update({"square": (1000, 50), "space": (500, 50), "thirdEm": (333, 50)}) + builder.setupHorizontalMetrics(metrics) + builder.setupHorizontalHeader(ascent=800, descent=-200) + builder.setupNameTable({ + "familyName": "Penpot CJK Test", + "styleName": "Regular", + "uniqueFontIdentifier": "PenpotCJKTest", + "fullName": "Penpot CJK Test Regular", + "psName": "PenpotCJKTest-Regular", + }) + builder.setupOS2(sTypoAscender=800, sTypoDescender=-200, + usWinAscent=800, usWinDescent=200) + builder.setupPost() + builder.setupMaxp() + addOpenTypeFeaturesFromString(builder.font, """ + languagesystem DFLT dflt; + languagesystem hani JAN; + languagesystem hani ZHS; + languagesystem hani KOR; + feature locl { + script hani; + language JAN; sub base by japanese; + language ZHS; sub base by chinese; + language KOR; sub base by korean; + } locl; + """) + builder.font["head"].created = 3800000000 + builder.font["head"].modified = 3800000000 + builder.font.recalcTimestamp = False + builder.save(Path(__file__).resolve().parents[1] / "src/fonts/cjk-locl-test.ttf") + + +if __name__ == "__main__": + main() diff --git a/render-wasm/src/fonts/cjk-locl-test.ttf b/render-wasm/src/fonts/cjk-locl-test.ttf new file mode 100644 index 0000000000000000000000000000000000000000..90a910c2fc8fadfa8c865659114176bfea795ece GIT binary patch literal 1480 zcma)6T}V_x6#nMkyX#h(t!x(iAY(~|WNur;B7#2^iIP97LDW*-)~jpoU%M+9g%w0m z1oaz5qhbmz0$kg`Njb-AH|))h#vnud*Us{V-!E?2_No0KU`T2 zDCa1i3>kXYnvI1Q$$v_wteyIkkLRlQ7gm3Ts!kDt568NN z$yzT-rWLUs6^exA9F=E$1Wm!439@OK$t0z5NW=lnWaeh(G^!GyN^GR~M-$~PQl)5_ z7;C7J;hB7M4IH8xYvW*)+skI*Rd!Q<2(QYi(`p6Ac7m#s)rh<>6O zC+VFN;}e%}zZ@O(kN9odUR1UDI?J!&ySZ1CQa1OT^P9ENz5xYrV=XqL4$at$5Ft#| z#r9FX^o~uX&QdR1pMEJ0FUusXOl|s6jyYvzI?^|C%&3(KrAKniEh}?0eKN;9wlYKM zc#e5(Wv-+HIY!*E`Bjgl8w3No&4cEyFciuP1*#yAB5`KoRu-Vx^fj!&YSXu4IjY3T zYKsUdL5t}tSc-`0tF(hhOkcwW44b|kTx}4B#pdas8?8fUo2thQG4=Gzn(DiQlZ{N*Y*hI z6Q{Kpwde*5>Cs+LKB$d#%^bx|Eq;!YJq_)gzCF-4B~Mmb@}yOu(%&u&mmJ-4 bool { || matches!(u32::from(c), 0x3000..=0x30FF | 0x31F0..=0x31FF | 0xFF00..=0xFFEF) } -/// True for characters that extend the preceding grapheme cluster -/// (Grapheme_Cluster_Break=Extend or ZWJ): combining marks, voiced sound -/// marks, variation selectors, emoji modifiers and tags. -pub fn extends_grapheme(c: char) -> bool { - matches!(u32::from(c), - 0x0300..=0x036F - | 0x1AB0..=0x1AFF - | 0x1DC0..=0x1DFF - | 0x200C..=0x200D - | 0x20D0..=0x20FF - | 0x3099..=0x309A - | 0xFE00..=0xFE0F - | 0xFE20..=0xFE2F - | 0xFF9E..=0xFF9F - | 0x1F3FB..=0x1F3FF - | 0xE0020..=0xE007F - | 0xE0100..=0xE01EF - ) -} - fn is_hiragana(c: char) -> bool { matches!(u32::from(c), 0x3041..=0x309F) } diff --git a/render-wasm/src/shapes/kinsoku.rs b/render-wasm/src/shapes/kinsoku.rs index 8a7bcabad0..6543ee114b 100644 --- a/render-wasm/src/shapes/kinsoku.rs +++ b/render-wasm/src/shapes/kinsoku.rs @@ -31,12 +31,10 @@ pub const WESTERN_WORD_SPACE: char = '\u{2004}'; /// Unicode IDEOGRAPHIC SPACE, the one-em space after a sentence-ending /// question or exclamation mark. pub const IDEOGRAPHIC_SPACE: char = '\u{3000}'; -/// Joins emoji into one grapheme cluster. -const ZERO_WIDTH_JOINER: char = '\u{200D}'; -use super::japanese::{ - classify, extends_grapheme, keeps_together_classified, pair_rule, JapaneseClass, -}; +use super::japanese::{classify, keeps_together_classified, pair_rule, JapaneseClass}; +use super::text_policy::TextLayoutPolicy; +use unicode_segmentation::UnicodeSegmentation; pub fn forbidden_at_line_start(c: char) -> bool { classify(c).forbids_line_start() @@ -98,15 +96,26 @@ impl OffsetMap { /// grapheme clusters are not, so nothing lands inside one. `ruby_breaks[span] /// == Some(boundaries)` forbids breaks at every internal scalar boundary of /// that span except those UTF-16 offsets. Returns `None` when nothing changes. -pub fn apply_to_span_texts_with_ruby_breaks( +#[cfg(test)] +fn apply_to_span_texts_with_ruby_breaks( span_texts: &[String], ruby_breaks: &[Option>], ) -> Option<(Vec, OffsetMap)> { + apply_policy_to_span_texts(span_texts, ruby_breaks, TextLayoutPolicy::Japanese) +} + +pub fn apply_policy_to_span_texts( + span_texts: &[String], + ruby_breaks: &[Option>], + policy: TextLayoutPolicy, +) -> Option<(Vec, OffsetMap)> { + let source = span_texts.concat(); + let boundaries: Vec = source.grapheme_indices(true).map(|(at, _)| at).collect(); + let mut source_byte = 0; let mut inserted: Vec = Vec::new(); let mut out: Vec = Vec::with_capacity(span_texts.len()); // Base character of the previous grapheme cluster, with its class. let mut prev: Option<(char, JapaneseClass)> = None; - let mut after_zwj = false; let mut changed = false; // Running position in shifted UTF-16 coordinates. let mut shifted_pos: usize = 0; @@ -115,11 +124,12 @@ pub fn apply_to_span_texts_with_ruby_breaks( let mut shifted_text = String::with_capacity(text.len() + 4); let mut local_utf16 = 0usize; for c in text.chars() { - if after_zwj || extends_grapheme(c) { + let at_boundary = boundaries.binary_search(&source_byte).is_ok(); + source_byte += c.len_utf8(); + if !at_boundary { shifted_text.push(c); shifted_pos += c.len_utf16(); local_utf16 += c.len_utf16(); - after_zwj = c == ZERO_WIDTH_JOINER; continue; } let current = (c, classify(c)); @@ -130,8 +140,12 @@ pub fn apply_to_span_texts_with_ruby_breaks( .is_some_and(|breaks| !breaks.contains(&local_utf16)); let forbid_break = ruby_forbids_break || prev.is_some_and(|p| { - pair_rule(p.1, current.1).suppress_break_with_joiner - || keeps_together_classified(p, current) + if policy == TextLayoutPolicy::Japanese { + pair_rule(p.1, current.1).suppress_break_with_joiner + || keeps_together_classified(p, current) + } else { + !policy.break_allowed(p.0, c) + } }); let mut insert = |ch: char| { shifted_text.push(ch); @@ -139,21 +153,26 @@ pub fn apply_to_span_texts_with_ruby_breaks( shifted_pos += 1; changed = true; }; - if prev.is_some_and(|p| takes_em_space_after(p, current)) { + if policy == TextLayoutPolicy::Japanese + && prev.is_some_and(|p| takes_em_space_after(p, current)) + { insert(IDEOGRAPHIC_SPACE); } - if prev.is_some_and(|p| is_japanese_western_boundary(p, current)) { + if policy == TextLayoutPolicy::Japanese + && prev.is_some_and(|p| is_japanese_western_boundary(p, current)) + { insert(JAPANESE_WESTERN_SPACE); } // After an inserted space, the joiner keeps the break blocked. if forbid_break { insert(WORD_JOINER); - } else if current.1 == JapaneseClass::Inseparable + } else if policy == TextLayoutPolicy::Japanese + && current.1 == JapaneseClass::Inseparable && prev.is_some_and(|p| pair_rule(p.1, current.1).break_allowed) { insert(BREAK_OPPORTUNITY); } - let layout_char = if c == ' ' { + let layout_char = if policy == TextLayoutPolicy::Japanese && c == ' ' { changed = true; WESTERN_WORD_SPACE } else { diff --git a/render-wasm/src/shapes/text.rs b/render-wasm/src/shapes/text.rs index 20f484f4ed..e13d35100d 100644 --- a/render-wasm/src/shapes/text.rs +++ b/render-wasm/src/shapes/text.rs @@ -1864,6 +1864,10 @@ pub struct Paragraph { text_transform: Option, writing_mode: WritingMode, text_orientation: TextOrientation, + language: String, + korean_spaces_only: bool, + /// Paragraph in the content tree last exchanged with the editor host. + source_position: u32, line_height: f32, letter_spacing: f32, children: Vec, @@ -1878,6 +1882,9 @@ impl Default for Paragraph { text_transform: None, writing_mode: WritingMode::default(), text_orientation: TextOrientation::default(), + language: String::new(), + korean_spaces_only: false, + source_position: u32::MAX, line_height: 1.0, letter_spacing: 0.0, children: vec![], @@ -1903,6 +1910,9 @@ impl Paragraph { text_transform, writing_mode: WritingMode::default(), text_orientation: TextOrientation::default(), + language: String::new(), + korean_spaces_only: false, + source_position: u32::MAX, line_height, letter_spacing, children, @@ -1913,6 +1923,37 @@ impl Paragraph { self.writing_mode } + pub fn language(&self) -> &str { + &self.language + } + + pub fn set_language(&mut self, language: String) { + for span in &mut self.children { + span.language.clone_from(&language); + } + self.language = language; + } + + pub fn korean_spaces_only(&self) -> bool { + self.korean_spaces_only + } + + pub fn set_korean_spaces_only(&mut self, value: bool) { + self.korean_spaces_only = value; + } + + pub fn layout_policy(&self) -> super::text_policy::TextLayoutPolicy { + super::text_policy::TextLayoutPolicy::for_language(&self.language, self.korean_spaces_only) + } + + pub fn source_position(&self) -> u32 { + self.source_position + } + + pub fn set_source_position(&mut self, position: u32) { + self.source_position = position; + } + pub fn set_writing_mode(&mut self, writing_mode: WritingMode) { self.writing_mode = writing_mode; } @@ -1933,7 +1974,16 @@ impl Paragraph { &mut self.children } + /// Move spans into this paragraph, inheriting its shaping language. + pub fn extend_children(&mut self, spans: impl IntoIterator) { + self.children.extend(spans.into_iter().map(|mut span| { + span.language.clone_from(&self.language); + span + })); + } + fn set_span_positions(&mut self, index: u32) { + self.source_position = index; for (span_index, span) in self.children.iter_mut().enumerate() { span.set_position(index, span_index as u32); } @@ -2041,7 +2091,7 @@ impl Paragraph { /// Japanese spacing normalized, kinsoku break suppressions inserted), /// plus the map from original to builder-text UTF-16 offsets. Consumers /// of laid-out offsets must translate through the map. Only paragraphs - /// with Japanese text or ruby are transformed. + /// with an explicit typography profile are transformed. pub fn layout_span_texts(&self) -> (Vec, kinsoku::OffsetMap) { layout_span_texts(self) } @@ -2179,6 +2229,8 @@ fn apply_text_transform_with_source_ranges( #[derive(Debug, PartialEq, Clone)] pub struct TextSpan { pub text: String, + /// Effective paragraph language, copied when the paragraph is decoded. + pub language: String, pub font_family: FontFamily, pub font_size: f32, pub line_height: f32, @@ -2211,6 +2263,7 @@ impl Default for TextSpan { fn default() -> Self { Self { text: String::default(), + language: String::new(), font_family: FontFamily::new(Uuid::nil(), 400, FontStyle::Normal), font_size: 14.0, line_height: 1.2, @@ -2366,6 +2419,9 @@ impl TextSpan { fill_layer_from_bottom: Option, ) -> skia::textlayout::TextStyle { let mut style = skia::textlayout::TextStyle::default(); + if !self.language.is_empty() { + style.set_locale(&self.language); + } let paint = if remove_alpha { let mut paint = paint::Paint::default(); paint.set_color(skia::Color::BLACK); diff --git a/render-wasm/src/shapes/text_horizontal/mod.rs b/render-wasm/src/shapes/text_horizontal/mod.rs index d6c702e9ee..a12bda3b45 100644 --- a/render-wasm/src/shapes/text_horizontal/mod.rs +++ b/render-wasm/src/shapes/text_horizontal/mod.rs @@ -27,6 +27,7 @@ use crate::math::Point; use crate::shapes::japanese::{ aki_class, is_japanese_text_char, pair_rule, punctuation_aki_sheds, AkiShed, JapaneseClass, }; +use crate::shapes::text_policy::TextLayoutPolicy; use crate::shapes::{kinsoku, merge_fills}; use crate::utils::{get_fallback_fonts, get_font_collection}; use skia_safe::{ @@ -41,10 +42,11 @@ use skia_safe::{ /// True when a paragraph with layout texts `texts` needs the Japanese passes: /// it has ruby or a Japanese character. pub(crate) fn uses_japanese_layout(paragraph: &Paragraph, texts: &[String]) -> bool { - paragraph.children().iter().any(TextSpan::has_ruby) - || texts - .iter() - .any(|text| text.chars().any(is_japanese_text_char)) + paragraph.layout_policy() == TextLayoutPolicy::Japanese + && (paragraph.children().iter().any(TextSpan::has_ruby) + || texts + .iter() + .any(|text| text.chars().any(is_japanese_text_char))) } pub(crate) fn layout_span_texts(paragraph: &Paragraph) -> (Vec, kinsoku::OffsetMap) { @@ -54,14 +56,22 @@ pub(crate) fn layout_span_texts(paragraph: &Paragraph) -> (Vec, kinsoku: .map(TextSpan::apply_text_transform) .collect(); // Text without Japanese characters or ruby keeps plain Skia layout. - if uses_japanese_layout(paragraph, &texts) { + let policy = paragraph.layout_policy(); + if uses_japanese_layout(paragraph, &texts) + || matches!( + policy, + TextLayoutPolicy::Chinese + | TextLayoutPolicy::KoreanSyllable + | TextLayoutPolicy::KoreanSpacesOnly + ) + { let ruby_breaks: Vec>> = paragraph .children() .iter() .map(|span| span.has_ruby().then(Vec::new)) .collect(); if let Some((shifted, map)) = - kinsoku::apply_to_span_texts_with_ruby_breaks(&texts, &ruby_breaks) + kinsoku::apply_policy_to_span_texts(&texts, &ruby_breaks, policy) { return (shifted, map); } @@ -78,6 +88,9 @@ pub(crate) fn horizontal_aki_sheds( paragraph: &Paragraph, texts: &[String], ) -> Vec> { + if paragraph.layout_policy() != TextLayoutPolicy::Japanese { + return vec![Vec::new(); texts.len()]; + } let spans = paragraph.children(); let half_em = |span: usize| spans.get(span).map_or(0.0, |span| span.font_size * 0.5); // (span, char index in the span text) of every real character. diff --git a/render-wasm/src/shapes/text_horizontal/ruby.rs b/render-wasm/src/shapes/text_horizontal/ruby.rs index 65b4c793f5..90cd49de36 100644 --- a/render-wasm/src/shapes/text_horizontal/ruby.rs +++ b/render-wasm/src/shapes/text_horizontal/ruby.rs @@ -108,6 +108,7 @@ pub(super) fn horizontal_ruby_spacing_with( false, span.font_features, &fallback_mgr, + &span.language, ) .iter() .flat_map(|run| run.advances.iter().copied()) @@ -151,6 +152,7 @@ pub(super) fn horizontal_ruby_spacing_with( false, span.font_features, &fallback_mgr, + &span.language, ); let (_, pieces, _) = horizontal_ruby_pieces(&ruby_glyph_refs(span.ruby_text(), &shaped, ruby_font_size)); @@ -392,6 +394,7 @@ pub(crate) fn paint_horizontal_ruby( false, span.font_features, &fallback_mgr, + &span.language, ); let glyphs = ruby_glyph_refs(span.ruby_text(), &shaped, span.ruby_font_size()); let extents: Vec = rects.iter().map(|rect| rect.rect.width()).collect(); diff --git a/render-wasm/src/shapes/text_horizontal/tests.rs b/render-wasm/src/shapes/text_horizontal/tests.rs index d38b27012d..b984fa36c1 100644 --- a/render-wasm/src/shapes/text_horizontal/tests.rs +++ b/render-wasm/src/shapes/text_horizontal/tests.rs @@ -7,6 +7,116 @@ use super::*; use crate::shapes::{TextAlign, TextDirection, TextTransform}; use crate::Uuid; +#[test] +fn untagged_cjk_keeps_plain_layout() { + init_state(); + for text in ["中文，。", "日本 hello world", "한국 漢字"] { + let paragraph = Paragraph::new( + TextAlign::Left, + TextDirection::LTR, + None, + None, + 1.0, + 0.0, + vec![make_span(text, 0.0)], + ); + let (texts, offsets) = paragraph.layout_span_texts(); + assert_eq!(texts, vec![text.to_string()]); + assert!(offsets.is_empty()); + assert!(horizontal_aki_sheds(¶graph, &texts)[0].is_empty()); + } +} + +#[test] +fn explicit_profiles_use_distinct_rules_across_styles() { + init_state(); + let mut paragraph = make_paragraph( + vec![make_span("中A 文「", 0.0), make_span("字」っ", 0.0)], + 0.0, + ); + paragraph.set_language("zh-Hans".to_string()); + let (texts, map) = paragraph.layout_span_texts(); + assert_eq!(texts.concat(), "中A 文「\u{2060}字\u{2060}」っ"); + for offset in 0..="中A 文「字」っ".encode_utf16().count() { + assert_eq!(map.to_original(map.to_shifted(offset)), offset); + } + let plan = HorizontalParagraphPlan::new(¶graph); + assert!(plan.sheds.iter().all(Vec::is_empty)); + + let mut paragraph = + make_paragraph(vec![make_span("한", 0.0), make_span("국어 문장", 0.0)], 0.0); + paragraph.set_language("ko".to_string()); + assert_eq!(paragraph.layout_span_texts().0.concat(), "한국어 문장"); + paragraph.set_korean_spaces_only(true); + assert_eq!( + paragraph.layout_span_texts().0.concat(), + "한\u{2060}국\u{2060}어 문\u{2060}장" + ); +} + +#[test] +fn korean_joiners_stay_outside_decomposed_syllables_and_emoji() { + init_state(); + let mut paragraph = make_paragraph(vec![make_span("ᄒ", 0.0), make_span("ᅡᆫ국 👩‍💻", 0.0)], 0.0); + paragraph.set_language("ko".to_string()); + paragraph.set_korean_spaces_only(true); + assert_eq!(paragraph.layout_span_texts().0.concat(), "한\u{2060}국 👩‍💻"); +} + +#[test] +fn same_paragraph_wraps_differently_by_profile_in_both_orientations() { + use crate::shapes::text_vertical::test_support::{ + content_of, layout_with_height, provider, EM, + }; + use crate::shapes::GrowType; + use std::collections::HashSet; + + init_state(); + let font_provider = provider(include_bytes!("../../fonts/cjk-locl-test.ttf")); + let mut fonts = skia::textlayout::FontCollection::new(); + fonts.set_asset_font_manager(Some(font_provider.clone().into())); + let mut horizontal_ends = Vec::new(); + let mut vertical_ends = Vec::new(); + for language in ["ja", "zh-Hans", "ko"] { + let mut span = make_span("文文 あっ文文", 0.0); + span.font_size = EM; + let mut paragraph = make_paragraph(vec![span], 0.0); + paragraph.set_language(language.to_string()); + paragraph.set_korean_spaces_only(true); + let plan = HorizontalParagraphPlan::new(¶graph); + let content = content_of(vec![paragraph.clone()], EM * 5.45, GrowType::Fixed); + let span = ¶graph.children()[0]; + let style = span.to_style(&content.bounds(), &HashSet::new(), false, 1.0); + let mut builder = ParagraphBuilder::new(¶graph.paragraph_to_style(), &fonts); + builder.push_style(&style); + add_horizontal_span( + &mut builder, + span, + &plan.texts[0], + &plan.sheds[0], + &[], + &style, + &fonts, + ); + let mut horizontal = builder.build(); + horizontal.layout(EM * 5.45); + let (_, offset_map) = paragraph.layout_span_texts(); + horizontal_ends.push(offset_map.to_original(horizontal.get_line_metrics()[0].end_index)); + let vertical = layout_with_height(&font_provider, &content, EM * 5.45); + vertical_ends.push( + vertical + .cells + .iter() + .filter(|cell| cell.column == 0) + .map(|cell| cell.end) + .max() + .unwrap(), + ); + } + assert_eq!(horizontal_ends, vec![6, 5, 3]); + assert_eq!(vertical_ends, vec![6, 5, 3]); +} + #[test] fn horizontal_ruby_ranges_use_utf16_across_spans() { init_state(); @@ -53,7 +163,7 @@ fn make_span(text: &str, letter_spacing: f32) -> TextSpan { } fn make_paragraph(spans: Vec, letter_spacing: f32) -> Paragraph { - Paragraph::new( + let mut paragraph = Paragraph::new( TextAlign::default(), TextDirection::LTR, None, @@ -61,7 +171,9 @@ fn make_paragraph(spans: Vec, letter_spacing: f32) -> Paragraph { 1.0, letter_spacing, spans, - ) + ); + paragraph.set_language("ja".to_string()); + paragraph } #[test] @@ -769,6 +881,7 @@ fn horizontal_ruby_glyphs_share_one_baseline() { false, FontFeatures::None, &FontMgr::from(font_provider.clone()), + "ja", ); let glyphs: Vec = shaped .iter() @@ -826,6 +939,7 @@ fn horizontal_western_ruby_keeps_its_word_whole() { false, FontFeatures::None, &FontMgr::from(font_provider.clone()), + "ja", ); let glyphs = ruby_glyph_refs("editor", &shaped, EM / 2.0); assert!(glyphs.iter().all(|(_, _, _, western)| *western)); diff --git a/render-wasm/src/shapes/text_policy.rs b/render-wasm/src/shapes/text_policy.rs new file mode 100644 index 0000000000..c561dcd8f5 --- /dev/null +++ b/render-wasm/src/shapes/text_policy.rs @@ -0,0 +1,85 @@ +//! Paragraph typography policies. Geometry and source offsets are shared; +//! Japanese spacing remains in the Japanese layout passes. + +use super::japanese::break_allowed_between; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum TextLayoutPolicy { + #[default] + Plain, + Japanese, + Chinese, + KoreanSyllable, + /// POC only: suppress every non-space boundary, including punctuation. + KoreanSpacesOnly, +} + +impl TextLayoutPolicy { + pub fn for_language(language: &str, korean_spaces_only: bool) -> Self { + let primary = language.split('-').next().unwrap_or_default(); + if primary.eq_ignore_ascii_case("ja") { + Self::Japanese + } else if primary.eq_ignore_ascii_case("zh") { + Self::Chinese + } else if primary.eq_ignore_ascii_case("ko") { + if korean_spaces_only { + Self::KoreanSpacesOnly + } else { + Self::KoreanSyllable + } + } else { + Self::Plain + } + } + + /// Legal boundary between graphemes, before fitting a line or column. + pub fn break_allowed(self, before: char, after: char) -> bool { + match self { + Self::Japanese => break_allowed_between(before, after), + Self::KoreanSpacesOnly => before.is_whitespace() || after.is_whitespace(), + Self::Chinese | Self::KoreanSyllable => { + !"（［｛〈《「『【〔〖〘〚‘“([{".contains(before) + && !"，。、．！？：；）］｝〉》」』】〕〗〙〛’”)]},.!?:;".contains(after) + && !(before == after && matches!(after, '…' | '—')) + && !(before.is_ascii_alphanumeric() && after.is_ascii_alphanumeric()) + } + Self::Plain => true, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn language_selects_one_policy_without_script_detection() { + for (language, expected) in [ + ("", TextLayoutPolicy::Plain), + ("en", TextLayoutPolicy::Plain), + ("ja-JP", TextLayoutPolicy::Japanese), + ("ZH-Hant-TW", TextLayoutPolicy::Chinese), + ("ko", TextLayoutPolicy::KoreanSyllable), + ] { + assert_eq!(TextLayoutPolicy::for_language(language, false), expected); + } + assert_eq!( + TextLayoutPolicy::for_language("ko-KR", true), + TextLayoutPolicy::KoreanSpacesOnly + ); + assert_eq!( + TextLayoutPolicy::for_language("zh", true), + TextLayoutPolicy::Chinese + ); + } + + #[test] + fn chinese_basic_restrictions_do_not_include_small_kana() { + let policy = TextLayoutPolicy::Chinese; + assert!(!policy.break_allowed('「', '中')); + assert!(!policy.break_allowed('文', '」')); + assert!(policy.break_allowed('中', '文')); + assert!(policy.break_allowed('あ', 'っ')); + assert!(!TextLayoutPolicy::Japanese.break_allowed('あ', 'っ')); + } +} diff --git a/render-wasm/src/shapes/text_vertical/annotations.rs b/render-wasm/src/shapes/text_vertical/annotations.rs index 3fd9160d98..4516a2f2c8 100644 --- a/render-wasm/src/shapes/text_vertical/annotations.rs +++ b/render-wasm/src/shapes/text_vertical/annotations.rs @@ -161,6 +161,7 @@ fn shape_ruby(span: &TextSpan, fonts: &Fonts) -> Vec<(ShapedRun, bool)> { segment.upright, span.font_features, fonts.fallback_mgr, + &span.language, ); for mut run in shaped { for cluster in &mut run.clusters { @@ -625,6 +626,7 @@ pub(super) fn layout_emphasis( true, FontFeatures::None, fonts.fallback_mgr.clone(), + &span.language, ); if shaped.first().is_none_or(|run| run.glyphs.is_empty()) { continue; diff --git a/render-wasm/src/shapes/text_vertical/cells.rs b/render-wasm/src/shapes/text_vertical/cells.rs index 0e03b51c07..ebb51246bb 100644 --- a/render-wasm/src/shapes/text_vertical/cells.rs +++ b/render-wasm/src/shapes/text_vertical/cells.rs @@ -3,6 +3,7 @@ // composite. Warichu notes span several spans and build their own cells. use std::ops::Range; +use unicode_segmentation::UnicodeSegmentation; use skia_safe::{textlayout::TypefaceFontProvider, FontMgr, Typeface}; @@ -180,6 +181,7 @@ impl<'a> SpanCells<'a> { upright, self.span.font_features, self.fonts.fallback_mgr, + &self.span.language, ) } @@ -233,6 +235,19 @@ impl<'a> SpanCells<'a> { for mut run in self.shape(&segment.text, self.span.font_size, segment.upright) { let run_index = runs.len(); if segment.upright { + // A font may leave conjoining Jamo as separate glyph + // clusters. Group them by Unicode grapheme boundaries. + let boundaries: Vec = segment + .text + .grapheme_indices(true) + .map(|(at, _)| at) + .collect(); + for cluster in &mut run.clusters { + let at = boundaries + .partition_point(|at| *at <= *cluster as usize) + .saturating_sub(1); + *cluster = boundaries[at].max(run.utf8_range.start) as u32; + } self.push_upright_clusters(&segment, piece_base, &run, run_index, cells); } else { cells.push(self.rotated_cell(&segment, piece_base, &mut run, run_index)); @@ -426,7 +441,11 @@ impl<'a> SpanCells<'a> { run_index: usize, ) -> FlowCell { let letter_spacing = self.span.letter_spacing; - run.normalize_word_spaces(&segment.text, self.span.font_size); + if crate::shapes::text_policy::TextLayoutPolicy::for_language(&self.span.language, false) + == crate::shapes::text_policy::TextLayoutPolicy::Japanese + { + run.normalize_word_spaces(&segment.text, self.span.font_size); + } let extent = run.spread_glyphs(letter_spacing); let run_text = &segment.text[run.utf8_range.clone()]; let script = FlowScript::Rotated { diff --git a/render-wasm/src/shapes/text_vertical/flow.rs b/render-wasm/src/shapes/text_vertical/flow.rs index 4bb61887f6..8fed4fc787 100644 --- a/render-wasm/src/shapes/text_vertical/flow.rs +++ b/render-wasm/src/shapes/text_vertical/flow.rs @@ -9,7 +9,9 @@ use crate::shapes::japanese::{ }; use crate::shapes::kinsoku::{forbidden_at_line_end, forbidden_at_line_start}; use crate::shapes::text_japanese::capped_equal_allocations; +use crate::shapes::text_policy::TextLayoutPolicy; use crate::shapes::{LineAdjustment, TextAlign}; +use unicode_segmentation::UnicodeSegmentation; use super::layout::{CellKind, VerticalCell}; @@ -273,6 +275,72 @@ pub(super) fn plan_columns(items: &[FlowItem], max_height: f32) -> Vec<(usize, f placements } +/// Basic non-Japanese flow. Boundaries use the entire paragraph, so font +/// and style runs cannot split a grapheme or create a word boundary. +pub(super) fn plan_policy_columns( + cells: &[FlowCell], + text: &str, + policy: TextLayoutPolicy, + max_height: f32, +) -> Vec<(usize, f32)> { + let mut boundaries = Vec::new(); + let mut offset = 0; + let mut previous = None; + for grapheme in text.graphemes(true) { + let first = grapheme.chars().next().unwrap(); + let allowed = previous.is_none_or(|before| policy.break_allowed(before, first)); + boundaries.push((offset, allowed)); + previous = Some(first); + offset += grapheme.encode_utf16().count(); + } + boundaries.push((offset, true)); + let mut atomic_start = Vec::with_capacity(cells.len()); + let mut unit_start = Vec::with_capacity(cells.len()); + let mut prefix = vec![0.0]; + for (i, flow) in cells.iter().enumerate() { + let boundary = boundaries + .binary_search_by_key(&flow.cell.start, |(at, _)| *at) + .ok(); + let atomic = + i > 0 && (flow.keep_with_previous || flow.shares_previous_box || boundary.is_none()); + atomic_start.push(if atomic { atomic_start[i - 1] } else { i }); + let joined = atomic || boundary.is_some_and(|at| !boundaries[at].1); + unit_start.push(if i > 0 && joined { + unit_start[i - 1] + } else { + i + }); + prefix.push(prefix[i] + flow.cell.extent); + } + let mut placements = Vec::with_capacity(cells.len()); + let (mut column, mut column_start, mut cursor) = (0, 0, 0.0); + for (i, flow) in cells.iter().enumerate() { + if cursor > 0.0 && cursor + flow.cell.extent > max_height + FIT_TOLERANCE { + let candidate = unit_start[i]; + let break_at = if candidate > column_start + && prefix[i + 1] - prefix[candidate] <= max_height + FIT_TOLERANCE + { + candidate + } else { + atomic_start[i] + }; + // An overlong unit may break, but a grapheme or annotation may not. + if break_at > column_start { + column += 1; + cursor = 0.0; + for k in break_at..i { + placements[k] = (column, cursor); + cursor += cells[k].cell.extent; + } + column_start = break_at; + } + } + placements.push((column, cursor)); + cursor += flow.cell.extent; + } + placements +} + /// True when `max_height` is a real wrap budget (auto-width columns are /// unbounded). pub(super) fn is_bounded(max_height: f32) -> bool { diff --git a/render-wasm/src/shapes/text_vertical/layout.rs b/render-wasm/src/shapes/text_vertical/layout.rs index 21623bad87..365002aaed 100644 --- a/render-wasm/src/shapes/text_vertical/layout.rs +++ b/render-wasm/src/shapes/text_vertical/layout.rs @@ -10,6 +10,7 @@ use crate::globals::get_resources; use crate::math::Rect; use crate::shapes::japanese::JapaneseClass; use crate::shapes::text_japanese::EMPHASIS_FONT_SCALE; +use crate::shapes::text_policy::TextLayoutPolicy; use crate::shapes::{ merge_fills, AppliedTextTransform, FontFeatures, GrowType, LineAdjustment, Paragraph, RubySide, TextAlign, TextContent, TextSpan, VerticalAlign, @@ -25,7 +26,8 @@ use super::cells::{Fonts, SpanCells, WarichuNote}; use super::flow::{ aki_classes, align_offset_along_column, apply_inter_script_spacing, apply_ordered_oikomi, flow_classes, is_bounded, materialize_explicit_pair_spacing, ordered_expansion_offsets, - plan_with_edge_trimming, preferred_pair_spacing, shed_punctuation_aki, FlowCell, FIT_TOLERANCE, + plan_policy_columns, plan_with_edge_trimming, preferred_pair_spacing, shed_punctuation_aki, + FlowCell, FIT_TOLERANCE, }; use super::paint::LayoutDraws; use super::shaping::ShapedRun; @@ -516,6 +518,7 @@ fn aligned_tops( /// A paragraph's flow after spacing and column planning. struct PlannedParagraph { + policy: TextLayoutPolicy, flow: Vec, ruby_units: Vec, classes: Vec>, @@ -554,6 +557,26 @@ impl PlannedParagraph { let mut flow = cells.flow(&warichu_splits, runs); let mut ruby_units = ruby_units.clone(); + let policy = cells.paragraph.layout_policy(); + if policy != TextLayoutPolicy::Japanese { + grow_ruby_bases(&mut flow, &ruby_units); + let text: String = cells + .text_map + .span_transforms + .iter() + .map(|t| t.text.as_str()) + .collect(); + let placements = plan_policy_columns(&flow, &text, policy, max_height); + return Self { + policy, + classes: vec![None; flow.len()], + pair_spacing_em: vec![0.0; flow.len().saturating_sub(1)], + flow, + ruby_units, + placements, + }; + } + apply_inter_script_spacing(&mut flow); let classes = flow_classes(&flow, &ruby_spans); let aki = aki_classes(&flow, &classes, &vpal_spans); @@ -586,6 +609,7 @@ impl PlannedParagraph { Some(split) => warichu_splits.push(split), None => { return Self { + policy, flow, ruby_units, classes, @@ -604,6 +628,12 @@ impl PlannedParagraph { /// Final cells, aligned along their columns, numbered from `column_base`. /// A later span's cell of a warichu piece takes the piece's box. fn place(&self, align: TextAlign, max_height: f32, column_base: usize) -> Vec { + // The POC defines no non-Japanese justification expansion table yet. + let align = if self.policy != TextLayoutPolicy::Japanese && align == TextAlign::Justify { + TextAlign::Left + } else { + align + }; let tops = aligned_tops( &self.flow, &self.classes, @@ -777,6 +807,82 @@ mod tests { use super::*; use crate::shapes::TextOrientation; + #[test] + fn explicit_profiles_change_wrapping_without_japanese_spacing() { + let columns = |language: &str, spaces_only: bool, texts: &[&str]| { + let mut paragraph = vertical_paragraph( + texts.iter().map(|text| make_span(text)).collect(), + TextAlign::Left, + 1.0, + ); + paragraph.set_language(language.to_string()); + paragraph.set_korean_spaces_only(spaces_only); + let content = content_of(vec![paragraph], EM * 2.1, GrowType::Fixed); + let layout = layout_with_height( + &provider(include_bytes!("../../fonts/cjk-locl-test.ttf")), + &content, + EM * 2.1, + ); + ( + layout + .cells + .iter() + .map(|cell| (cell.start, cell.column)) + .collect::>(), + layout, + ) + }; + // Small kana stay attached in Japanese only; Chinese uses its own + // basic punctuation policy even when the fixture contains kana. + let (ja, _) = columns("ja", false, &["文あっ文"]); + let (zh, _) = columns("zh-Hans", false, &["文あっ文"]); + assert_ne!(ja, zh); + let (syllables, _) = columns("ko", false, &["한", "국어 문장"]); + let (words, layout) = columns("ko", true, &["한", "국어 문장"]); + // A three-syllable word cannot fit here: safe emergency wrapping. + assert!(words.iter().any(|(_, column)| *column > 0)); + assert_eq!(syllables.len(), words.len()); + assert!(layout + .cells + .iter() + .all(|cell| cell.minimum_oikomi_extent <= cell.extent)); + } + + #[test] + fn korean_word_boundaries_cross_styles_and_graphemes_stay_together() { + let mut paragraph = vertical_paragraph( + vec![make_span("한글 "), make_span("한"), make_span("국어")], + TextAlign::Left, + 1.0, + ); + paragraph.set_language("ko".to_string()); + paragraph.set_korean_spaces_only(true); + let content = content_of(vec![paragraph], EM * 3.6, GrowType::Fixed); + let layout = layout_with_height( + &provider(include_bytes!("../../fonts/cjk-locl-test.ttf")), + &content, + EM * 3.6, + ); + let word: Vec<_> = layout.cells.iter().filter(|cell| cell.start >= 3).collect(); + assert_eq!(word.len(), 3); + assert!(word.iter().all(|cell| cell.column == word[0].column)); + assert!(word[0].column > 0); + + let mut paragraph = + vertical_paragraph(vec![make_span("ᄒ"), make_span("ᅡᆫ국")], TextAlign::Left, 1.0); + paragraph.set_language("ko".to_string()); + let content = content_of(vec![paragraph], EM, GrowType::Fixed); + let layout = layout_with_height( + &provider(include_bytes!("../../fonts/cjk-locl-test.ttf")), + &content, + EM, + ); + let syllable: Vec<_> = layout.cells.iter().filter(|cell| cell.start < 3).collect(); + assert!(syllable + .iter() + .all(|cell| cell.column == syllable[0].column)); + } + fn with_headless_fonts(test: impl FnOnce() -> T) -> T { let mut resources = crate::render::RenderResources::try_new_headless().expect("headless resources"); diff --git a/render-wasm/src/shapes/text_vertical/orientation.rs b/render-wasm/src/shapes/text_vertical/orientation.rs index b11731738c..2e20eb98fd 100644 --- a/render-wasm/src/shapes/text_vertical/orientation.rs +++ b/render-wasm/src/shapes/text_vertical/orientation.rs @@ -1,4 +1,5 @@ use crate::shapes::TextOrientation; +use unicode_segmentation::UnicodeSegmentation; /// Emoji ranges recognized by the frontend font-loader. Emoji stay upright /// in vertical flow; Latin text rotates under `text-orientation: mixed`. @@ -18,6 +19,11 @@ pub(crate) fn is_upright_char(c: char) -> bool { is_emoji_char(c) || matches!(u32::from(c), 0x2E80..=0x2FDF // CJK radicals, Kangxi radicals + | 0x1100..=0x11FF // conjoining Hangul Jamo + | 0x3100..=0x318F // Bopomofo and compatibility Jamo + | 0x31A0..=0x31BF // Bopomofo extended + | 0xA960..=0xA97F // Hangul Jamo extended A + | 0xD7B0..=0xD7FF // Hangul Jamo extended B | 0x3000..=0x303F // CJK symbols and punctuation | 0x3040..=0x30FF // hiragana, katakana | 0x31C0..=0x31EF // CJK strokes @@ -78,7 +84,8 @@ pub(super) struct Segment { pub(super) fn segment_by_orientation(text: &str, orientation: TextOrientation) -> Vec { let mut segments: Vec = Vec::new(); let mut utf16_offset = 0; - for c in text.chars() { + for grapheme in text.graphemes(true) { + let c = grapheme.chars().next().unwrap(); let upright = orientation == TextOrientation::Upright || is_upright_char(c); let emoji = is_emoji_char(c); // CJK and emoji are both upright, but split them so the segment @@ -90,15 +97,15 @@ pub(super) fn segment_by_orientation(text: &str, orientation: TextOrientation) - if last.upright == upright && last.text.chars().next_back().is_some_and(is_emoji_char) == emoji => { - last.text.push(c) + last.text.push_str(grapheme) } _ => segments.push(Segment { - text: c.to_string(), + text: grapheme.to_string(), utf16_start: utf16_offset, upright, }), } - utf16_offset += c.len_utf16(); + utf16_offset += grapheme.encode_utf16().count(); } segments } diff --git a/render-wasm/src/shapes/text_vertical/shaping.rs b/render-wasm/src/shapes/text_vertical/shaping.rs index 82a957383a..cdc7e1a5d5 100644 --- a/render-wasm/src/shapes/text_vertical/shaping.rs +++ b/render-wasm/src/shapes/text_vertical/shaping.rs @@ -329,11 +329,12 @@ pub(super) fn shape_segment( upright: bool, font_features: FontFeatures, fallback: FontMgr, + language: &str, ) -> Vec { let mut font_iter = skia::Shaper::new_font_mgr_run_iterator(text, font, Some(fallback.clone())); let mut bidi_iter = skia::shapers::primitive::trivial_bidi_run_iterator(0, text.len()); let mut script_iter = skia::Shaper::new_hb_icu_script_run_iterator(text); - let mut lang_iter = skia::Shaper::new_trivial_language_run_iterator("ja", text.len()); + let mut lang_iter = skia::Shaper::new_trivial_language_run_iterator(language, text.len()); let mut features = if upright { vec![feature(b"vert"), feature(b"vrt2")] @@ -421,6 +422,7 @@ fn typeface_chunks(text: &str, typefaces: &[Typeface]) -> Vec<(Range, usi /// Shape a segment with explicit per-character font fallback. /// `TypefaceFontProvider` resolves named families, but some Skia builds lack /// its character fallback hook, so glyph coverage is resolved here. +#[allow(clippy::too_many_arguments)] pub(crate) fn shape_segment_with_fallbacks( text: &str, font_size: f32, @@ -429,6 +431,7 @@ pub(crate) fn shape_segment_with_fallbacks( upright: bool, font_features: FontFeatures, fallback_mgr: &FontMgr, + language: &str, ) -> Vec { shape_with_typefaces( text, @@ -437,6 +440,7 @@ pub(crate) fn shape_segment_with_fallbacks( upright, font_features, fallback_mgr, + language, ) } @@ -459,6 +463,7 @@ pub(crate) fn shape_with_typefaces( upright: bool, font_features: FontFeatures, fallback_mgr: &FontMgr, + language: &str, ) -> Vec { if typefaces.is_empty() { return Vec::new(); @@ -473,6 +478,7 @@ pub(crate) fn shape_with_typefaces( upright, font_features, fallback_mgr.clone(), + language, ); for run in &mut shaped { run.utf8_range = @@ -544,6 +550,38 @@ mod tests { use super::super::test_support::*; use super::*; + #[test] + fn selected_language_reaches_horizontal_and_vertical_localized_glyphs() { + use crate::shapes::{GrowType, TextAlign}; + use skia_safe::textlayout::{FontCollection, ParagraphBuilder}; + use std::collections::HashSet; + + let font_provider = provider(include_bytes!("../../fonts/cjk-locl-test.ttf")); + let mut fonts = FontCollection::new(); + fonts.set_asset_font_manager(Some(font_provider.clone().into())); + for (language, glyph, width) in [("ja", 2, 12.0), ("zh-Hans", 3, 14.0), ("ko", 4, 16.0)] { + let mut paragraph = vertical_paragraph(vec![make_span("中")], TextAlign::Left, 1.0); + paragraph.set_language(language.to_string()); + let content = content_of(vec![paragraph.clone()], 100.0, GrowType::Fixed); + let layout = layout_with(&font_provider, &content); + assert_eq!(layout.runs[0].glyphs, vec![glyph], "vertical {language}"); + + let span = ¶graph.children()[0]; + let style = span.to_style(&content.bounds(), &HashSet::new(), false, 1.0); + assert_eq!(style.locale(), language); + let mut builder = ParagraphBuilder::new(¶graph.paragraph_to_style(), &fonts); + builder.push_style(&style); + builder.add_text(&span.text); + let mut horizontal = builder.build(); + horizontal.layout(100.0); + assert!( + (horizontal.longest_line() - width).abs() < 0.01, + "horizontal {language}: {}", + horizontal.longest_line() + ); + } + } + #[test] fn span_families_try_emoji_after_the_span_font() { let families = span_font_families(&make_span("😆"), &["fallback".to_string()]); diff --git a/render-wasm/src/shapes/text_vertical/test_support.rs b/render-wasm/src/shapes/text_vertical/test_support.rs index f6872c9d66..978b68538c 100644 --- a/render-wasm/src/shapes/text_vertical/test_support.rs +++ b/render-wasm/src/shapes/text_vertical/test_support.rs @@ -70,11 +70,12 @@ pub(super) fn vertical_paragraph( spans, ); paragraph.set_writing_mode(WritingMode::VerticalRl); + paragraph.set_language("ja".to_string()); paragraph } /// A 200px wide box `height` tall holding `paragraphs`. -pub(super) fn content_of( +pub(crate) fn content_of( paragraphs: Vec, height: f32, grow_type: GrowType, @@ -167,7 +168,7 @@ pub(super) fn layout_with_fallback( ) } -pub(super) fn layout_with_height( +pub(crate) fn layout_with_height( provider: &TypefaceFontProvider, content: &TextContent, max_height: f32, diff --git a/render-wasm/src/wasm/text.rs b/render-wasm/src/wasm/text.rs index ec252b87dd..75e7d4311c 100644 --- a/render-wasm/src/wasm/text.rs +++ b/render-wasm/src/wasm/text.rs @@ -108,10 +108,12 @@ pub struct RawParagraphData { text_transform: RawTextTransform, writing_mode: u8, text_orientation: u8, + korean_wrap: u8, // Padding for the CLJS writer's 4-byte-aligned layout; always zero. - _padding: [u8; 2], + _padding: [u8; 1], line_height: f32, letter_spacing: f32, + language_length: u32, } impl From<[u8; RAW_PARAGRAPH_DATA_SIZE]> for RawParagraphData { @@ -297,6 +299,12 @@ impl From for shapes::Paragraph { paragraph.set_writing_mode(RawWritingMode::from(value.attrs.writing_mode).into()); paragraph .set_text_orientation(RawTextOrientation::from(value.attrs.text_orientation).into()); + let language = ruby_offset + .checked_add(value.attrs.language_length as usize) + .and_then(|end| value.text_buffer.get(ruby_offset..end)) + .unwrap_or(&[]); + paragraph.set_language(String::from_utf8_lossy(language).into_owned()); + paragraph.set_korean_spaces_only(value.attrs.korean_wrap == 1); paragraph } } @@ -507,13 +515,13 @@ mod tests { AnnotationClearance, FontFeatures, RubyAlign, RubyOverhang, RubySide, RubySize, }; - /// The CLJS writer (texts.cljs) writes PARAGRAPH-ATTR-U8-SIZE (16) + /// The shared CLJS writer writes PARAGRAPH-ATTR-U8-SIZE (20) /// attr bytes after the u32 span count, and SPAN-ATTR-U8-SIZE (80) /// attr bytes before the fills block. These sizes must match the struct /// layouts. #[test] fn raw_struct_sizes_match_cljs_writer() { - const PARAGRAPH_ATTR_U8_SIZE: usize = 16; + const PARAGRAPH_ATTR_U8_SIZE: usize = 20; const SPAN_ATTR_U8_SIZE: usize = 80; assert_eq!(RAW_PARAGRAPH_DATA_SIZE, 4 + PARAGRAPH_ATTR_U8_SIZE); assert_eq!( @@ -522,6 +530,26 @@ mod tests { ); } + #[test] + fn language_decodes_after_base_and_ruby_text() { + let mut attrs = [0u8; RAW_PARAGRAPH_DATA_SIZE]; + attrs[10] = 1; // Korean spaces-only POC; byte 6 after span count. + attrs[20..24].copy_from_slice(&2u32.to_le_bytes()); + let mut span = [0u8; RAW_SPAN_DATA_SIZE]; + span[68..72].copy_from_slice(&3u32.to_le_bytes()); + span[72..76].copy_from_slice(&3u32.to_le_bytes()); + let paragraph = shapes::Paragraph::from(RawParagraph { + attrs: RawParagraphData::from(attrs), + spans: vec![RawTextSpan::from(span)], + text_buffer: "韓한ko".as_bytes().to_vec(), + }); + assert_eq!(paragraph.language(), "ko"); + assert!(paragraph.korean_spaces_only()); + assert_eq!(paragraph.children()[0].text, "韓"); + assert_eq!(paragraph.children()[0].ruby, "한"); + assert_eq!(paragraph.children()[0].language, "ko"); + } + #[test] fn raw_text_combine_upright_counts_deserialize() { // Byte 5 of the span attr block; the counted digits variants map diff --git a/render-wasm/src/wasm/text/helpers.rs b/render-wasm/src/wasm/text/helpers.rs index a0ae969667..5a442257f4 100644 --- a/render-wasm/src/wasm/text/helpers.rs +++ b/render-wasm/src/wasm/text/helpers.rs @@ -477,11 +477,8 @@ pub fn delete_selection_range(text_content: &mut TextContent, selection: &TextSe delete_range_in_paragraph(&mut paragraphs[end_paragraph], 0, end.offset); - let end_para_children: Vec<_> = - paragraphs[end_paragraph].children_mut().drain(..).collect(); - paragraphs[start.paragraph] - .children_mut() - .extend(end_para_children); + let end_para_children = std::mem::take(paragraphs[end_paragraph].children_mut()); + paragraphs[start.paragraph].extend_children(end_para_children); paragraphs.drain((start.paragraph + 1)..=end_paragraph); @@ -593,13 +590,8 @@ pub fn delete_char_before( let paragraphs = text_content.paragraphs_mut(); let prev_para_len = paragraph_char_count(¶graphs[prev_para_idx]); - let current_children: Vec<_> = paragraphs[cursor.paragraph] - .children_mut() - .drain(..) - .collect(); - paragraphs[prev_para_idx] - .children_mut() - .extend(current_children); + let current_children = std::mem::take(paragraphs[cursor.paragraph].children_mut()); + paragraphs[prev_para_idx].extend_children(current_children); paragraphs.remove(cursor.paragraph); @@ -770,10 +762,8 @@ pub fn delete_char_after(text_content: &mut TextContent, cursor: &TextPositionWi delete_range_in_paragraph(para, cursor.offset, end); } else if cursor.paragraph < paragraphs.len() - 1 { let next_para_idx = cursor.paragraph + 1; - let next_children: Vec<_> = paragraphs[next_para_idx].children_mut().drain(..).collect(); - paragraphs[cursor.paragraph] - .children_mut() - .extend(next_children); + let next_children = std::mem::take(paragraphs[next_para_idx].children_mut()); + paragraphs[cursor.paragraph].extend_children(next_children); paragraphs.remove(next_para_idx); } @@ -825,6 +815,9 @@ pub fn split_paragraph_at_cursor( let text_transform = para.text_transform(); let writing_mode = para.writing_mode(); let text_orientation = para.text_orientation(); + let language = para.language().to_owned(); + let korean_spaces_only = para.korean_spaces_only(); + let source_position = para.source_position(); let line_height = para.line_height(); let letter_spacing = para.letter_spacing(); @@ -851,6 +844,9 @@ pub fn split_paragraph_at_cursor( ); new_para.set_writing_mode(writing_mode); new_para.set_text_orientation(text_orientation); + new_para.set_language(language); + new_para.set_korean_spaces_only(korean_spaces_only); + new_para.set_source_position(source_position); paragraphs.insert(cursor.paragraph + 1, new_para); @@ -864,6 +860,51 @@ mod tests { use crate::shapes::{FontFamily, FontStyle, GrowType, TextAlign, TextSpan}; use crate::uuid::Uuid; + #[test] + fn paragraph_split_keeps_language_and_korean_wrap() { + let mut para = paragraph(&["한국어"]); + para.set_language("ko".to_string()); + para.set_korean_spaces_only(true); + let mut content = content(vec![para]); + let cursor = TextPositionWithAffinity { + paragraph: 0, + offset: 1, + ..Default::default() + }; + assert!(split_paragraph_at_cursor(&mut content, &cursor)); + assert_eq!(content.paragraphs().len(), 2); + for para in content.paragraphs() { + assert_eq!(para.language(), "ko"); + assert!(para.korean_spaces_only()); + assert_eq!(para.source_position(), 0); + assert!(para.children().iter().all(|span| span.language == "ko")); + } + } + + #[test] + fn paragraph_merge_inherits_the_surviving_paragraph_language() { + let mut first = paragraph(&["中"]); + first.set_language("ja".to_string()); + let mut second = paragraph(&["中"]); + second.set_language("ko".to_string()); + let mut content = content(vec![first, second]); + delete_char_after( + &mut content, + &TextPositionWithAffinity { + paragraph: 0, + offset: 1, + ..Default::default() + }, + ); + assert_eq!(content.paragraphs().len(), 1); + let paragraph = &content.paragraphs()[0]; + assert_eq!(paragraph.language(), "ja"); + assert!(paragraph + .children() + .iter() + .all(|span| span.language == "ja")); + } + fn span(text: &str) -> TextSpan { TextSpan::new( text.to_string(), diff --git a/render-wasm/src/wasm/text_editor.rs b/render-wasm/src/wasm/text_editor.rs index a16df85d13..bd875ffdcb 100644 --- a/render-wasm/src/wasm/text_editor.rs +++ b/render-wasm/src/wasm/text_editor.rs @@ -984,8 +984,11 @@ pub extern "C" fn text_editor_export_content() -> *mut u8 { .replace('\r', "\\r") .replace('\t', "\\t"); span_parts.push(format!( - "{{\"p\":{},\"s\":{},\"t\":\"{}\"}}", - span.paragraph_position, span.span_position, escaped_text + "{{\"pp\":{},\"p\":{},\"s\":{},\"t\":\"{}\"}}", + para.source_position(), + span.paragraph_position, + span.span_position, + escaped_text )); } json_parts.push(format!("[{}]", span_parts.join(",")));