diff --git a/common/src/app/common/render_wasm/text_content.cljs b/common/src/app/common/render_wasm/text_content.cljs index 2aec0df710..fd643749af 100644 --- a/common/src/app/common/render_wasm/text_content.cljs +++ b/common/src/app/common/render_wasm/text_content.cljs @@ -6,8 +6,9 @@ (ns app.common.render-wasm.text-content "Single source of truth for writing a text shape's content into the WASM design - state. The binary layout ([num-spans][paragraph attrs][span attrs][text][ruby-text]) is - identical for the workspace and the headless exporter, and so is the font-id + state. The binary layout is [num-spans][paragraph attrs][span attrs][text] + [ruby-text][language], identical for the workspace and headless exporter. + Both hosts also share the font-id -> uuid mapping (`cfnt/font-id->uuid`). Only *variant* resolution differs — the workspace has a loaded fonts DB, the exporter does not — so that part is injected via the `opts` map passed to `write-shape-text!`. @@ -24,7 +25,7 @@ [app.common.uuid :as uuid] [cuerdas.core :as str])) -(def ^:const PARAGRAPH-ATTR-U8-SIZE 16) +(def ^:const PARAGRAPH-ATTR-U8-SIZE 20) (def ^:const SPAN-ATTR-U8-SIZE 80) (def ^:const MAX-TEXT-FILLS types.fills.impl/MAX-FILLS) @@ -126,11 +127,12 @@ (mem/write-u8 dview text-transform) (mem/write-u8 dview writing-mode) (mem/write-u8 dview text-orientation) + (mem/write-u8 dview (if (= "spaces-only" (:korean-wrap paragraph)) 1 0)) ;; Alignment padding; must match RawParagraphData in Rust. (mem/write-u8 dview 0) - (mem/write-u8 dview 0) (mem/write-f32 dview line-height) (mem/write-f32 dview letter-spacing) + (mem/write-u32 dview (mem/size (encode-text (d/nilv (:text-language paragraph) "")))) (mem/assert-written offset PARAGRAPH-ATTR-U8-SIZE)))) (defn- write-spans @@ -225,24 +227,26 @@ :or {normalize-font-id cfnt/font-id->uuid normalize-paragraph identity normalize-span (fn [span _paragraph] span)}}] - (let [paragraph (normalize-paragraph paragraph) - spans (map #(normalize-span % paragraph) spans) - num-spans (count spans) - fills-size (* types.fills.impl/FILL-U8-SIZE MAX-TEXT-FILLS) - metadata-size (+ PARAGRAPH-ATTR-U8-SIZE - (* num-spans (+ SPAN-ATTR-U8-SIZE fills-size))) - text-buffer (encode-text text) - text-size (mem/size text-buffer) - ruby-buffer (encode-text (apply str (map span-ruby spans))) - ruby-size (mem/size ruby-buffer) - total-size (+ 4 metadata-size text-size ruby-size) - heapu8 (mem/get-heap-u8) - dview (mem/get-data-view) - offset (mem/alloc total-size)] + (let [paragraph (normalize-paragraph paragraph) + spans (map #(normalize-span % paragraph) spans) + num-spans (count spans) + fills-size (* types.fills.impl/FILL-U8-SIZE MAX-TEXT-FILLS) + metadata-size (+ PARAGRAPH-ATTR-U8-SIZE + (* num-spans (+ SPAN-ATTR-U8-SIZE fills-size))) + text-buffer (encode-text text) + text-size (mem/size text-buffer) + ruby-buffer (encode-text (apply str (map span-ruby spans))) + ruby-size (mem/size ruby-buffer) + language-buffer (encode-text (d/nilv (:text-language paragraph) "")) + total-size (+ 4 metadata-size text-size ruby-size (mem/size language-buffer)) + heapu8 (mem/get-heap-u8) + dview (mem/get-data-view) + offset (mem/alloc total-size)] (-> offset (mem/write-u32 dview num-spans) (write-paragraph dview paragraph) (write-spans dview spans paragraph normalize-font-id) (mem/write-buffer heapu8 text-buffer) - (mem/write-buffer heapu8 ruby-buffer)) + (mem/write-buffer heapu8 ruby-buffer) + (mem/write-buffer heapu8 language-buffer)) (h/call wasm/internal-module "_set_shape_text_content"))) diff --git a/common/src/app/common/types/shape/text.cljc b/common/src/app/common/types/shape/text.cljc index b17ddbd77c..ddc8982ffa 100644 --- a/common/src/app/common/types/shape/text.cljc +++ b/common/src/app/common/types/shape/text.cljc @@ -44,6 +44,12 @@ [:font-style {:optional true} ::sm/text] [:font-weight {:optional true} ::sm/text] [:direction {:optional true} ::sm/text] + ;; Paragraph language selects typography; it never follows the UI locale. + [:text-language {:optional true} + [:re {:gen/elements ["ja" "zh-Hans" "zh-Hant-TW" "ko" "en-GB"]} + #"^(?:[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*|[xXiI](?:-[A-Za-z0-9]{1,8})+)$"]] + ;; Experimental POC mode, deliberately distinct from CSS keep-all. + [:korean-wrap {:optional true} [:enum "syllable" "spaces-only"]] [:writing-mode {:optional true} (schema:jl-enum :writing-mode)] [:text-orientation {:optional true} (schema:jl-enum :text-orientation)] [:text-decoration {:optional true} ::sm/text] diff --git a/common/src/app/common/types/text.cljc b/common/src/app/common/types/text.cljc index 40f70afc16..3173142967 100644 --- a/common/src/app/common/types/text.cljc +++ b/common/src/app/common/types/text.cljc @@ -108,6 +108,7 @@ (d/concat-vec text-align-attrs text-direction-attrs + [:text-language :korean-wrap] cjk/whole-shape-paragraph-attrs)) (def text-node-attrs diff --git a/common/test/common_tests/types/text_test.cljc b/common/test/common_tests/types/text_test.cljc index ab7f15a668..f8f5b3129f 100644 --- a/common/test/common_tests/types/text_test.cljc +++ b/common/test/common_tests/types/text_test.cljc @@ -379,6 +379,19 @@ [attrs] (update-in content-base [:children 0 :children 0] merge attrs)) +(t/deftest paragraph-language-and-experimental-korean-wrap + (doseq [language ["ja" "zh-Hans" "zh-Hant-TW" "ko" "en-GB"]] + (t/is (ctst/valid-content? + (with-paragraph-attrs {:text-language language + :korean-wrap "spaces-only"})))) + (doseq [attrs [{:text-language 42} + {:text-language "zh_Hans"} + {:text-language ""} + {:korean-wrap "keep-all"}]] + (t/is (not (ctst/valid-content? (with-paragraph-attrs attrs))) (pr-str attrs))) + (t/is (contains? (set cttx/paragraph-attrs) :text-language)) + (t/is (not (contains? (set cjk/whole-shape-paragraph-attrs) :text-language)))) + (t/deftest japanese-span-attrs-accept-supported-values (t/is (ctst/valid-content? (with-span-attrs {:text-combine-upright "digits2" diff --git a/frontend/src/app/render_wasm/text_editor.cljs b/frontend/src/app/render_wasm/text_editor.cljs index c0af201b4d..6307e5f184 100644 --- a/frontend/src/app/render_wasm/text_editor.cljs +++ b/frontend/src/app/render_wasm/text_editor.cljs @@ -610,7 +610,10 @@ styling is taken from there; a span WASM never saw falls back to its position and then to the last existing span. - exported vector of paragraphs, each a vector of `{:p 0 :s 0 :t \"text\"}` + `:pp` tracks paragraph attrs independently of span styling, including + when a merge removes all the surviving paragraph's original characters. + + exported vector of paragraphs, each a vector of `{:pp 0 :p 0 :s 0 :t \"text\"}` content existing Penpot content map (root -> paragraph-set -> …)" [content exported] (let [para-set (first (get content :children)) @@ -621,7 +624,8 @@ styling-para (fn [para-idx spans] - (or (get orig-paras (get (first spans) :p)) + (or (get orig-paras (get (first spans) :pp)) + (get orig-paras (get (first spans) :p)) (get orig-paras para-idx) last-orig-para)) diff --git a/frontend/test/frontend_tests/render_wasm/texts_test.cljs b/frontend/test/frontend_tests/render_wasm/texts_test.cljs index ba9f87db8e..fa7fc49224 100644 --- a/frontend/test/frontend_tests/render_wasm/texts_test.cljs +++ b/frontend/test/frontend_tests/render_wasm/texts_test.cljs @@ -314,6 +314,49 @@ (t/is (not-any? #(contains? % :text-orientation) paragraphs)) (t/is (= [nil "center" nil] (mapv :text-align paragraphs))))) +(t/deftest language-and-wrapping-only-change-selected-paragraphs + (let [content (editor-content [{:text "日本"}] [{:text "한국어"}] [{:text "中文"}]) + result (text-editor/apply-paragraph-attrs-to-range + content (selection 1 0 1 0) + {:text-language "ko" :korean-wrap "spaces-only"}) + paragraphs (-> result :children first :children)] + (t/is (= [nil "ko" nil] (mapv :text-language paragraphs))) + (t/is (= [nil "spaces-only" nil] (mapv :korean-wrap paragraphs))) + (t/is (= result + (text-editor/merge-exported-texts-into-content + result [[{:p 0 :s 0 :t "日本"}] + [{:p 1 :s 0 :t "한국어"}] + [{:p 2 :s 0 :t "中文"}]])))) + (let [content (assoc-in (editor-content [{:text "한글"}]) + [:children 0 :children 0 :text-language] "ko") + result (text-editor/merge-exported-texts-into-content + content [[{:p 0 :s 0 :t "한"}] [{:p 0 :s 0 :t "글"}]])] + (t/is (= ["ko" "ko"] (mapv :text-language (-> result :children first :children)))))) + +(t/deftest paragraph-language-binary-header-matches-rust + (let [dview (js/DataView. (js/ArrayBuffer. tc/PARAGRAPH-ATTR-U8-SIZE)) + end (@#'tc/write-paragraph 0 dview + {:text-language "zh-Hant-TW" :korean-wrap "spaces-only" + :line-height "1.5" :letter-spacing "2"})] + (t/is (= 20 end)) + (t/is (= 1 (.getUint8 dview 6))) + (t/is (= 0 (.getUint8 dview 7))) + (t/is (= 1.5 (.getFloat32 dview 8 true))) + (t/is (= 2 (.getFloat32 dview 12 true))) + (t/is (= 10 (.getUint32 dview 16 true))))) + +(t/deftest merging-away-original-spans-keeps-the-surviving-paragraph-profile + (let [content (-> (editor-content [{:text "日"}] [{:text "한"}]) + (assoc-in [:children 0 :children 0 :text-language] "ja") + (assoc-in [:children 0 :children 1 :text-language] "ko") + (assoc-in [:children 0 :children 1 :korean-wrap] "spaces-only")) + result (text-editor/merge-exported-texts-into-content + content [[{:pp 0 :p 1 :s 0 :t "한"}]]) + para (get-in result [:children 0 :children 0])] + (t/is (= "ja" (:text-language para))) + (t/is (not (contains? para :korean-wrap))) + (t/is (= "한" (get-in para [:children 0 :text]))))) + ;; Byte offsets inside one span's attribute block (see `write-spans` and ;; RawTextSpan in render-wasm). (def ^:private span-attr-offset diff --git a/render-wasm/scripts/build-cjk-test-font.py b/render-wasm/scripts/build-cjk-test-font.py new file mode 100644 index 0000000000..b9e8281b9e --- /dev/null +++ b/render-wasm/scripts/build-cjk-test-font.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Build the synthetic CJK language fixture (requires fonttools). + +The one Han character has distinct JAN, ZHS and KOR localized glyphs and +advances. This tests language propagation without a system font dependency. +All outlines are original rectangles; no external font data is included. +""" + +from pathlib import Path + +from fontTools.feaLib.builder import addOpenTypeFeaturesFromString +from fontTools.fontBuilder import FontBuilder +from fontTools.pens.ttGlyphPen import TTGlyphPen + + +def main(): + builder = FontBuilder(1000, isTTF=True) + names = [".notdef", "base", "japanese", "chinese", "korean", "square", "space", "thirdEm"] + builder.setupGlyphOrder(names) + cmap = {ord(char): "square" for char in "文あっ한국어문장한글한「」,。"} + cmap.update({0x4E2D: "base", 0x20: "space", 0x2004: "thirdEm"}) + builder.setupCharacterMap(cmap) + glyphs = {} + for index, name in enumerate(names): + pen = TTGlyphPen(None) + pen.moveTo((50, 0)) + pen.lineTo((350 + index * 20, 0)) + pen.lineTo((350 + index * 20, 700)) + pen.lineTo((50, 700)) + pen.closePath() + glyphs[name] = pen.glyph() + builder.setupGlyf(glyphs) + metrics = {name: (400 + index * 100, 50) for index, name in enumerate(names)} + metrics.update({"square": (1000, 50), "space": (500, 50), "thirdEm": (333, 50)}) + builder.setupHorizontalMetrics(metrics) + builder.setupHorizontalHeader(ascent=800, descent=-200) + builder.setupNameTable({ + "familyName": "Penpot CJK Test", + "styleName": "Regular", + "uniqueFontIdentifier": "PenpotCJKTest", + "fullName": "Penpot CJK Test Regular", + "psName": "PenpotCJKTest-Regular", + }) + builder.setupOS2(sTypoAscender=800, sTypoDescender=-200, + usWinAscent=800, usWinDescent=200) + builder.setupPost() + builder.setupMaxp() + addOpenTypeFeaturesFromString(builder.font, """ + languagesystem DFLT dflt; + languagesystem hani JAN; + languagesystem hani ZHS; + languagesystem hani KOR; + feature locl { + script hani; + language JAN; sub base by japanese; + language ZHS; sub base by chinese; + language KOR; sub base by korean; + } locl; + """) + builder.font["head"].created = 3800000000 + builder.font["head"].modified = 3800000000 + builder.font.recalcTimestamp = False + builder.save(Path(__file__).resolve().parents[1] / "src/fonts/cjk-locl-test.ttf") + + +if __name__ == "__main__": + main() diff --git a/render-wasm/src/fonts/cjk-locl-test.ttf b/render-wasm/src/fonts/cjk-locl-test.ttf new file mode 100644 index 0000000000..90a910c2fc Binary files /dev/null and b/render-wasm/src/fonts/cjk-locl-test.ttf differ diff --git a/render-wasm/src/render/text.rs b/render-wasm/src/render/text.rs index 55bcbdf37b..f549759ca4 100644 --- a/render-wasm/src/render/text.rs +++ b/render-wasm/src/render/text.rs @@ -1578,10 +1578,11 @@ mod tests { #[test] fn decorated_ranges_follow_inserted_kinsoku_characters() { // A word joiner goes before 。, so the next span starts one unit later. - let para = paragraph(vec![ + let mut para = paragraph(vec![ span("あ。", None, None), span("い", Some(TextDecoration::UNDERLINE), None), ]); + para.set_language("ja".to_string()); assert_eq!( decorated_span_ranges(¶, &HorizontalOffsets::new(¶)), diff --git a/render-wasm/src/shapes.rs b/render-wasm/src/shapes.rs index 069fce58c0..6b3d5bb3df 100644 --- a/render-wasm/src/shapes.rs +++ b/render-wasm/src/shapes.rs @@ -33,6 +33,7 @@ mod text; mod text_horizontal; mod text_japanese; pub mod text_paths; +pub mod text_policy; pub mod text_vertical; mod transform; diff --git a/render-wasm/src/shapes/japanese.rs b/render-wasm/src/shapes/japanese.rs index 315a692d35..1901cbf7c0 100644 --- a/render-wasm/src/shapes/japanese.rs +++ b/render-wasm/src/shapes/japanese.rs @@ -509,26 +509,6 @@ pub fn is_japanese_text_char(c: char) -> bool { || matches!(u32::from(c), 0x3000..=0x30FF | 0x31F0..=0x31FF | 0xFF00..=0xFFEF) } -/// True for characters that extend the preceding grapheme cluster -/// (Grapheme_Cluster_Break=Extend or ZWJ): combining marks, voiced sound -/// marks, variation selectors, emoji modifiers and tags. -pub fn extends_grapheme(c: char) -> bool { - matches!(u32::from(c), - 0x0300..=0x036F - | 0x1AB0..=0x1AFF - | 0x1DC0..=0x1DFF - | 0x200C..=0x200D - | 0x20D0..=0x20FF - | 0x3099..=0x309A - | 0xFE00..=0xFE0F - | 0xFE20..=0xFE2F - | 0xFF9E..=0xFF9F - | 0x1F3FB..=0x1F3FF - | 0xE0020..=0xE007F - | 0xE0100..=0xE01EF - ) -} - fn is_hiragana(c: char) -> bool { matches!(u32::from(c), 0x3041..=0x309F) } diff --git a/render-wasm/src/shapes/kinsoku.rs b/render-wasm/src/shapes/kinsoku.rs index 8a7bcabad0..6543ee114b 100644 --- a/render-wasm/src/shapes/kinsoku.rs +++ b/render-wasm/src/shapes/kinsoku.rs @@ -31,12 +31,10 @@ pub const WESTERN_WORD_SPACE: char = '\u{2004}'; /// Unicode IDEOGRAPHIC SPACE, the one-em space after a sentence-ending /// question or exclamation mark. pub const IDEOGRAPHIC_SPACE: char = '\u{3000}'; -/// Joins emoji into one grapheme cluster. -const ZERO_WIDTH_JOINER: char = '\u{200D}'; -use super::japanese::{ - classify, extends_grapheme, keeps_together_classified, pair_rule, JapaneseClass, -}; +use super::japanese::{classify, keeps_together_classified, pair_rule, JapaneseClass}; +use super::text_policy::TextLayoutPolicy; +use unicode_segmentation::UnicodeSegmentation; pub fn forbidden_at_line_start(c: char) -> bool { classify(c).forbids_line_start() @@ -98,15 +96,26 @@ impl OffsetMap { /// grapheme clusters are not, so nothing lands inside one. `ruby_breaks[span] /// == Some(boundaries)` forbids breaks at every internal scalar boundary of /// that span except those UTF-16 offsets. Returns `None` when nothing changes. -pub fn apply_to_span_texts_with_ruby_breaks( +#[cfg(test)] +fn apply_to_span_texts_with_ruby_breaks( span_texts: &[String], ruby_breaks: &[Option>], ) -> Option<(Vec, OffsetMap)> { + apply_policy_to_span_texts(span_texts, ruby_breaks, TextLayoutPolicy::Japanese) +} + +pub fn apply_policy_to_span_texts( + span_texts: &[String], + ruby_breaks: &[Option>], + policy: TextLayoutPolicy, +) -> Option<(Vec, OffsetMap)> { + let source = span_texts.concat(); + let boundaries: Vec = source.grapheme_indices(true).map(|(at, _)| at).collect(); + let mut source_byte = 0; let mut inserted: Vec = Vec::new(); let mut out: Vec = Vec::with_capacity(span_texts.len()); // Base character of the previous grapheme cluster, with its class. let mut prev: Option<(char, JapaneseClass)> = None; - let mut after_zwj = false; let mut changed = false; // Running position in shifted UTF-16 coordinates. let mut shifted_pos: usize = 0; @@ -115,11 +124,12 @@ pub fn apply_to_span_texts_with_ruby_breaks( let mut shifted_text = String::with_capacity(text.len() + 4); let mut local_utf16 = 0usize; for c in text.chars() { - if after_zwj || extends_grapheme(c) { + let at_boundary = boundaries.binary_search(&source_byte).is_ok(); + source_byte += c.len_utf8(); + if !at_boundary { shifted_text.push(c); shifted_pos += c.len_utf16(); local_utf16 += c.len_utf16(); - after_zwj = c == ZERO_WIDTH_JOINER; continue; } let current = (c, classify(c)); @@ -130,8 +140,12 @@ pub fn apply_to_span_texts_with_ruby_breaks( .is_some_and(|breaks| !breaks.contains(&local_utf16)); let forbid_break = ruby_forbids_break || prev.is_some_and(|p| { - pair_rule(p.1, current.1).suppress_break_with_joiner - || keeps_together_classified(p, current) + if policy == TextLayoutPolicy::Japanese { + pair_rule(p.1, current.1).suppress_break_with_joiner + || keeps_together_classified(p, current) + } else { + !policy.break_allowed(p.0, c) + } }); let mut insert = |ch: char| { shifted_text.push(ch); @@ -139,21 +153,26 @@ pub fn apply_to_span_texts_with_ruby_breaks( shifted_pos += 1; changed = true; }; - if prev.is_some_and(|p| takes_em_space_after(p, current)) { + if policy == TextLayoutPolicy::Japanese + && prev.is_some_and(|p| takes_em_space_after(p, current)) + { insert(IDEOGRAPHIC_SPACE); } - if prev.is_some_and(|p| is_japanese_western_boundary(p, current)) { + if policy == TextLayoutPolicy::Japanese + && prev.is_some_and(|p| is_japanese_western_boundary(p, current)) + { insert(JAPANESE_WESTERN_SPACE); } // After an inserted space, the joiner keeps the break blocked. if forbid_break { insert(WORD_JOINER); - } else if current.1 == JapaneseClass::Inseparable + } else if policy == TextLayoutPolicy::Japanese + && current.1 == JapaneseClass::Inseparable && prev.is_some_and(|p| pair_rule(p.1, current.1).break_allowed) { insert(BREAK_OPPORTUNITY); } - let layout_char = if c == ' ' { + let layout_char = if policy == TextLayoutPolicy::Japanese && c == ' ' { changed = true; WESTERN_WORD_SPACE } else { diff --git a/render-wasm/src/shapes/text.rs b/render-wasm/src/shapes/text.rs index 20f484f4ed..e13d35100d 100644 --- a/render-wasm/src/shapes/text.rs +++ b/render-wasm/src/shapes/text.rs @@ -1864,6 +1864,10 @@ pub struct Paragraph { text_transform: Option, writing_mode: WritingMode, text_orientation: TextOrientation, + language: String, + korean_spaces_only: bool, + /// Paragraph in the content tree last exchanged with the editor host. + source_position: u32, line_height: f32, letter_spacing: f32, children: Vec, @@ -1878,6 +1882,9 @@ impl Default for Paragraph { text_transform: None, writing_mode: WritingMode::default(), text_orientation: TextOrientation::default(), + language: String::new(), + korean_spaces_only: false, + source_position: u32::MAX, line_height: 1.0, letter_spacing: 0.0, children: vec![], @@ -1903,6 +1910,9 @@ impl Paragraph { text_transform, writing_mode: WritingMode::default(), text_orientation: TextOrientation::default(), + language: String::new(), + korean_spaces_only: false, + source_position: u32::MAX, line_height, letter_spacing, children, @@ -1913,6 +1923,37 @@ impl Paragraph { self.writing_mode } + pub fn language(&self) -> &str { + &self.language + } + + pub fn set_language(&mut self, language: String) { + for span in &mut self.children { + span.language.clone_from(&language); + } + self.language = language; + } + + pub fn korean_spaces_only(&self) -> bool { + self.korean_spaces_only + } + + pub fn set_korean_spaces_only(&mut self, value: bool) { + self.korean_spaces_only = value; + } + + pub fn layout_policy(&self) -> super::text_policy::TextLayoutPolicy { + super::text_policy::TextLayoutPolicy::for_language(&self.language, self.korean_spaces_only) + } + + pub fn source_position(&self) -> u32 { + self.source_position + } + + pub fn set_source_position(&mut self, position: u32) { + self.source_position = position; + } + pub fn set_writing_mode(&mut self, writing_mode: WritingMode) { self.writing_mode = writing_mode; } @@ -1933,7 +1974,16 @@ impl Paragraph { &mut self.children } + /// Move spans into this paragraph, inheriting its shaping language. + pub fn extend_children(&mut self, spans: impl IntoIterator) { + self.children.extend(spans.into_iter().map(|mut span| { + span.language.clone_from(&self.language); + span + })); + } + fn set_span_positions(&mut self, index: u32) { + self.source_position = index; for (span_index, span) in self.children.iter_mut().enumerate() { span.set_position(index, span_index as u32); } @@ -2041,7 +2091,7 @@ impl Paragraph { /// Japanese spacing normalized, kinsoku break suppressions inserted), /// plus the map from original to builder-text UTF-16 offsets. Consumers /// of laid-out offsets must translate through the map. Only paragraphs - /// with Japanese text or ruby are transformed. + /// with an explicit typography profile are transformed. pub fn layout_span_texts(&self) -> (Vec, kinsoku::OffsetMap) { layout_span_texts(self) } @@ -2179,6 +2229,8 @@ fn apply_text_transform_with_source_ranges( #[derive(Debug, PartialEq, Clone)] pub struct TextSpan { pub text: String, + /// Effective paragraph language, copied when the paragraph is decoded. + pub language: String, pub font_family: FontFamily, pub font_size: f32, pub line_height: f32, @@ -2211,6 +2263,7 @@ impl Default for TextSpan { fn default() -> Self { Self { text: String::default(), + language: String::new(), font_family: FontFamily::new(Uuid::nil(), 400, FontStyle::Normal), font_size: 14.0, line_height: 1.2, @@ -2366,6 +2419,9 @@ impl TextSpan { fill_layer_from_bottom: Option, ) -> skia::textlayout::TextStyle { let mut style = skia::textlayout::TextStyle::default(); + if !self.language.is_empty() { + style.set_locale(&self.language); + } let paint = if remove_alpha { let mut paint = paint::Paint::default(); paint.set_color(skia::Color::BLACK); diff --git a/render-wasm/src/shapes/text_horizontal/mod.rs b/render-wasm/src/shapes/text_horizontal/mod.rs index d6c702e9ee..a12bda3b45 100644 --- a/render-wasm/src/shapes/text_horizontal/mod.rs +++ b/render-wasm/src/shapes/text_horizontal/mod.rs @@ -27,6 +27,7 @@ use crate::math::Point; use crate::shapes::japanese::{ aki_class, is_japanese_text_char, pair_rule, punctuation_aki_sheds, AkiShed, JapaneseClass, }; +use crate::shapes::text_policy::TextLayoutPolicy; use crate::shapes::{kinsoku, merge_fills}; use crate::utils::{get_fallback_fonts, get_font_collection}; use skia_safe::{ @@ -41,10 +42,11 @@ use skia_safe::{ /// True when a paragraph with layout texts `texts` needs the Japanese passes: /// it has ruby or a Japanese character. pub(crate) fn uses_japanese_layout(paragraph: &Paragraph, texts: &[String]) -> bool { - paragraph.children().iter().any(TextSpan::has_ruby) - || texts - .iter() - .any(|text| text.chars().any(is_japanese_text_char)) + paragraph.layout_policy() == TextLayoutPolicy::Japanese + && (paragraph.children().iter().any(TextSpan::has_ruby) + || texts + .iter() + .any(|text| text.chars().any(is_japanese_text_char))) } pub(crate) fn layout_span_texts(paragraph: &Paragraph) -> (Vec, kinsoku::OffsetMap) { @@ -54,14 +56,22 @@ pub(crate) fn layout_span_texts(paragraph: &Paragraph) -> (Vec, kinsoku: .map(TextSpan::apply_text_transform) .collect(); // Text without Japanese characters or ruby keeps plain Skia layout. - if uses_japanese_layout(paragraph, &texts) { + let policy = paragraph.layout_policy(); + if uses_japanese_layout(paragraph, &texts) + || matches!( + policy, + TextLayoutPolicy::Chinese + | TextLayoutPolicy::KoreanSyllable + | TextLayoutPolicy::KoreanSpacesOnly + ) + { let ruby_breaks: Vec>> = paragraph .children() .iter() .map(|span| span.has_ruby().then(Vec::new)) .collect(); if let Some((shifted, map)) = - kinsoku::apply_to_span_texts_with_ruby_breaks(&texts, &ruby_breaks) + kinsoku::apply_policy_to_span_texts(&texts, &ruby_breaks, policy) { return (shifted, map); } @@ -78,6 +88,9 @@ pub(crate) fn horizontal_aki_sheds( paragraph: &Paragraph, texts: &[String], ) -> Vec> { + if paragraph.layout_policy() != TextLayoutPolicy::Japanese { + return vec![Vec::new(); texts.len()]; + } let spans = paragraph.children(); let half_em = |span: usize| spans.get(span).map_or(0.0, |span| span.font_size * 0.5); // (span, char index in the span text) of every real character. diff --git a/render-wasm/src/shapes/text_horizontal/ruby.rs b/render-wasm/src/shapes/text_horizontal/ruby.rs index 65b4c793f5..90cd49de36 100644 --- a/render-wasm/src/shapes/text_horizontal/ruby.rs +++ b/render-wasm/src/shapes/text_horizontal/ruby.rs @@ -108,6 +108,7 @@ pub(super) fn horizontal_ruby_spacing_with( false, span.font_features, &fallback_mgr, + &span.language, ) .iter() .flat_map(|run| run.advances.iter().copied()) @@ -151,6 +152,7 @@ pub(super) fn horizontal_ruby_spacing_with( false, span.font_features, &fallback_mgr, + &span.language, ); let (_, pieces, _) = horizontal_ruby_pieces(&ruby_glyph_refs(span.ruby_text(), &shaped, ruby_font_size)); @@ -392,6 +394,7 @@ pub(crate) fn paint_horizontal_ruby( false, span.font_features, &fallback_mgr, + &span.language, ); let glyphs = ruby_glyph_refs(span.ruby_text(), &shaped, span.ruby_font_size()); let extents: Vec = rects.iter().map(|rect| rect.rect.width()).collect(); diff --git a/render-wasm/src/shapes/text_horizontal/tests.rs b/render-wasm/src/shapes/text_horizontal/tests.rs index d38b27012d..b984fa36c1 100644 --- a/render-wasm/src/shapes/text_horizontal/tests.rs +++ b/render-wasm/src/shapes/text_horizontal/tests.rs @@ -7,6 +7,116 @@ use super::*; use crate::shapes::{TextAlign, TextDirection, TextTransform}; use crate::Uuid; +#[test] +fn untagged_cjk_keeps_plain_layout() { + init_state(); + for text in ["中文,。", "日本 hello world", "한국 漢字"] { + let paragraph = Paragraph::new( + TextAlign::Left, + TextDirection::LTR, + None, + None, + 1.0, + 0.0, + vec![make_span(text, 0.0)], + ); + let (texts, offsets) = paragraph.layout_span_texts(); + assert_eq!(texts, vec![text.to_string()]); + assert!(offsets.is_empty()); + assert!(horizontal_aki_sheds(¶graph, &texts)[0].is_empty()); + } +} + +#[test] +fn explicit_profiles_use_distinct_rules_across_styles() { + init_state(); + let mut paragraph = make_paragraph( + vec![make_span("中A 文「", 0.0), make_span("字」っ", 0.0)], + 0.0, + ); + paragraph.set_language("zh-Hans".to_string()); + let (texts, map) = paragraph.layout_span_texts(); + assert_eq!(texts.concat(), "中A 文「\u{2060}字\u{2060}」っ"); + for offset in 0..="中A 文「字」っ".encode_utf16().count() { + assert_eq!(map.to_original(map.to_shifted(offset)), offset); + } + let plan = HorizontalParagraphPlan::new(¶graph); + assert!(plan.sheds.iter().all(Vec::is_empty)); + + let mut paragraph = + make_paragraph(vec![make_span("한", 0.0), make_span("국어 문장", 0.0)], 0.0); + paragraph.set_language("ko".to_string()); + assert_eq!(paragraph.layout_span_texts().0.concat(), "한국어 문장"); + paragraph.set_korean_spaces_only(true); + assert_eq!( + paragraph.layout_span_texts().0.concat(), + "한\u{2060}국\u{2060}어 문\u{2060}장" + ); +} + +#[test] +fn korean_joiners_stay_outside_decomposed_syllables_and_emoji() { + init_state(); + let mut paragraph = make_paragraph(vec![make_span("ᄒ", 0.0), make_span("ᅡᆫ국 👩‍💻", 0.0)], 0.0); + paragraph.set_language("ko".to_string()); + paragraph.set_korean_spaces_only(true); + assert_eq!(paragraph.layout_span_texts().0.concat(), "한\u{2060}국 👩‍💻"); +} + +#[test] +fn same_paragraph_wraps_differently_by_profile_in_both_orientations() { + use crate::shapes::text_vertical::test_support::{ + content_of, layout_with_height, provider, EM, + }; + use crate::shapes::GrowType; + use std::collections::HashSet; + + init_state(); + let font_provider = provider(include_bytes!("../../fonts/cjk-locl-test.ttf")); + let mut fonts = skia::textlayout::FontCollection::new(); + fonts.set_asset_font_manager(Some(font_provider.clone().into())); + let mut horizontal_ends = Vec::new(); + let mut vertical_ends = Vec::new(); + for language in ["ja", "zh-Hans", "ko"] { + let mut span = make_span("文文 あっ文文", 0.0); + span.font_size = EM; + let mut paragraph = make_paragraph(vec![span], 0.0); + paragraph.set_language(language.to_string()); + paragraph.set_korean_spaces_only(true); + let plan = HorizontalParagraphPlan::new(¶graph); + let content = content_of(vec![paragraph.clone()], EM * 5.45, GrowType::Fixed); + let span = ¶graph.children()[0]; + let style = span.to_style(&content.bounds(), &HashSet::new(), false, 1.0); + let mut builder = ParagraphBuilder::new(¶graph.paragraph_to_style(), &fonts); + builder.push_style(&style); + add_horizontal_span( + &mut builder, + span, + &plan.texts[0], + &plan.sheds[0], + &[], + &style, + &fonts, + ); + let mut horizontal = builder.build(); + horizontal.layout(EM * 5.45); + let (_, offset_map) = paragraph.layout_span_texts(); + horizontal_ends.push(offset_map.to_original(horizontal.get_line_metrics()[0].end_index)); + let vertical = layout_with_height(&font_provider, &content, EM * 5.45); + vertical_ends.push( + vertical + .cells + .iter() + .filter(|cell| cell.column == 0) + .map(|cell| cell.end) + .max() + .unwrap(), + ); + } + assert_eq!(horizontal_ends, vec![6, 5, 3]); + assert_eq!(vertical_ends, vec![6, 5, 3]); +} + #[test] fn horizontal_ruby_ranges_use_utf16_across_spans() { init_state(); @@ -53,7 +163,7 @@ fn make_span(text: &str, letter_spacing: f32) -> TextSpan { } fn make_paragraph(spans: Vec, letter_spacing: f32) -> Paragraph { - Paragraph::new( + let mut paragraph = Paragraph::new( TextAlign::default(), TextDirection::LTR, None, @@ -61,7 +171,9 @@ fn make_paragraph(spans: Vec, letter_spacing: f32) -> Paragraph { 1.0, letter_spacing, spans, - ) + ); + paragraph.set_language("ja".to_string()); + paragraph } #[test] @@ -769,6 +881,7 @@ fn horizontal_ruby_glyphs_share_one_baseline() { false, FontFeatures::None, &FontMgr::from(font_provider.clone()), + "ja", ); let glyphs: Vec = shaped .iter() @@ -826,6 +939,7 @@ fn horizontal_western_ruby_keeps_its_word_whole() { false, FontFeatures::None, &FontMgr::from(font_provider.clone()), + "ja", ); let glyphs = ruby_glyph_refs("editor", &shaped, EM / 2.0); assert!(glyphs.iter().all(|(_, _, _, western)| *western)); diff --git a/render-wasm/src/shapes/text_policy.rs b/render-wasm/src/shapes/text_policy.rs new file mode 100644 index 0000000000..c561dcd8f5 --- /dev/null +++ b/render-wasm/src/shapes/text_policy.rs @@ -0,0 +1,85 @@ +//! Paragraph typography policies. Geometry and source offsets are shared; +//! Japanese spacing remains in the Japanese layout passes. + +use super::japanese::break_allowed_between; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum TextLayoutPolicy { + #[default] + Plain, + Japanese, + Chinese, + KoreanSyllable, + /// POC only: suppress every non-space boundary, including punctuation. + KoreanSpacesOnly, +} + +impl TextLayoutPolicy { + pub fn for_language(language: &str, korean_spaces_only: bool) -> Self { + let primary = language.split('-').next().unwrap_or_default(); + if primary.eq_ignore_ascii_case("ja") { + Self::Japanese + } else if primary.eq_ignore_ascii_case("zh") { + Self::Chinese + } else if primary.eq_ignore_ascii_case("ko") { + if korean_spaces_only { + Self::KoreanSpacesOnly + } else { + Self::KoreanSyllable + } + } else { + Self::Plain + } + } + + /// Legal boundary between graphemes, before fitting a line or column. + pub fn break_allowed(self, before: char, after: char) -> bool { + match self { + Self::Japanese => break_allowed_between(before, after), + Self::KoreanSpacesOnly => before.is_whitespace() || after.is_whitespace(), + Self::Chinese | Self::KoreanSyllable => { + !"([{〈《「『【〔〖〘〚‘“([{".contains(before) + && !",。、.!?:;)]}〉》」』】〕〗〙〛’”)]},.!?:;".contains(after) + && !(before == after && matches!(after, '…' | '—')) + && !(before.is_ascii_alphanumeric() && after.is_ascii_alphanumeric()) + } + Self::Plain => true, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn language_selects_one_policy_without_script_detection() { + for (language, expected) in [ + ("", TextLayoutPolicy::Plain), + ("en", TextLayoutPolicy::Plain), + ("ja-JP", TextLayoutPolicy::Japanese), + ("ZH-Hant-TW", TextLayoutPolicy::Chinese), + ("ko", TextLayoutPolicy::KoreanSyllable), + ] { + assert_eq!(TextLayoutPolicy::for_language(language, false), expected); + } + assert_eq!( + TextLayoutPolicy::for_language("ko-KR", true), + TextLayoutPolicy::KoreanSpacesOnly + ); + assert_eq!( + TextLayoutPolicy::for_language("zh", true), + TextLayoutPolicy::Chinese + ); + } + + #[test] + fn chinese_basic_restrictions_do_not_include_small_kana() { + let policy = TextLayoutPolicy::Chinese; + assert!(!policy.break_allowed('「', '中')); + assert!(!policy.break_allowed('文', '」')); + assert!(policy.break_allowed('中', '文')); + assert!(policy.break_allowed('あ', 'っ')); + assert!(!TextLayoutPolicy::Japanese.break_allowed('あ', 'っ')); + } +} diff --git a/render-wasm/src/shapes/text_vertical/annotations.rs b/render-wasm/src/shapes/text_vertical/annotations.rs index 3fd9160d98..4516a2f2c8 100644 --- a/render-wasm/src/shapes/text_vertical/annotations.rs +++ b/render-wasm/src/shapes/text_vertical/annotations.rs @@ -161,6 +161,7 @@ fn shape_ruby(span: &TextSpan, fonts: &Fonts) -> Vec<(ShapedRun, bool)> { segment.upright, span.font_features, fonts.fallback_mgr, + &span.language, ); for mut run in shaped { for cluster in &mut run.clusters { @@ -625,6 +626,7 @@ pub(super) fn layout_emphasis( true, FontFeatures::None, fonts.fallback_mgr.clone(), + &span.language, ); if shaped.first().is_none_or(|run| run.glyphs.is_empty()) { continue; diff --git a/render-wasm/src/shapes/text_vertical/cells.rs b/render-wasm/src/shapes/text_vertical/cells.rs index 0e03b51c07..ebb51246bb 100644 --- a/render-wasm/src/shapes/text_vertical/cells.rs +++ b/render-wasm/src/shapes/text_vertical/cells.rs @@ -3,6 +3,7 @@ // composite. Warichu notes span several spans and build their own cells. use std::ops::Range; +use unicode_segmentation::UnicodeSegmentation; use skia_safe::{textlayout::TypefaceFontProvider, FontMgr, Typeface}; @@ -180,6 +181,7 @@ impl<'a> SpanCells<'a> { upright, self.span.font_features, self.fonts.fallback_mgr, + &self.span.language, ) } @@ -233,6 +235,19 @@ impl<'a> SpanCells<'a> { for mut run in self.shape(&segment.text, self.span.font_size, segment.upright) { let run_index = runs.len(); if segment.upright { + // A font may leave conjoining Jamo as separate glyph + // clusters. Group them by Unicode grapheme boundaries. + let boundaries: Vec = segment + .text + .grapheme_indices(true) + .map(|(at, _)| at) + .collect(); + for cluster in &mut run.clusters { + let at = boundaries + .partition_point(|at| *at <= *cluster as usize) + .saturating_sub(1); + *cluster = boundaries[at].max(run.utf8_range.start) as u32; + } self.push_upright_clusters(&segment, piece_base, &run, run_index, cells); } else { cells.push(self.rotated_cell(&segment, piece_base, &mut run, run_index)); @@ -426,7 +441,11 @@ impl<'a> SpanCells<'a> { run_index: usize, ) -> FlowCell { let letter_spacing = self.span.letter_spacing; - run.normalize_word_spaces(&segment.text, self.span.font_size); + if crate::shapes::text_policy::TextLayoutPolicy::for_language(&self.span.language, false) + == crate::shapes::text_policy::TextLayoutPolicy::Japanese + { + run.normalize_word_spaces(&segment.text, self.span.font_size); + } let extent = run.spread_glyphs(letter_spacing); let run_text = &segment.text[run.utf8_range.clone()]; let script = FlowScript::Rotated { diff --git a/render-wasm/src/shapes/text_vertical/flow.rs b/render-wasm/src/shapes/text_vertical/flow.rs index 4bb61887f6..8fed4fc787 100644 --- a/render-wasm/src/shapes/text_vertical/flow.rs +++ b/render-wasm/src/shapes/text_vertical/flow.rs @@ -9,7 +9,9 @@ use crate::shapes::japanese::{ }; use crate::shapes::kinsoku::{forbidden_at_line_end, forbidden_at_line_start}; use crate::shapes::text_japanese::capped_equal_allocations; +use crate::shapes::text_policy::TextLayoutPolicy; use crate::shapes::{LineAdjustment, TextAlign}; +use unicode_segmentation::UnicodeSegmentation; use super::layout::{CellKind, VerticalCell}; @@ -273,6 +275,72 @@ pub(super) fn plan_columns(items: &[FlowItem], max_height: f32) -> Vec<(usize, f placements } +/// Basic non-Japanese flow. Boundaries use the entire paragraph, so font +/// and style runs cannot split a grapheme or create a word boundary. +pub(super) fn plan_policy_columns( + cells: &[FlowCell], + text: &str, + policy: TextLayoutPolicy, + max_height: f32, +) -> Vec<(usize, f32)> { + let mut boundaries = Vec::new(); + let mut offset = 0; + let mut previous = None; + for grapheme in text.graphemes(true) { + let first = grapheme.chars().next().unwrap(); + let allowed = previous.is_none_or(|before| policy.break_allowed(before, first)); + boundaries.push((offset, allowed)); + previous = Some(first); + offset += grapheme.encode_utf16().count(); + } + boundaries.push((offset, true)); + let mut atomic_start = Vec::with_capacity(cells.len()); + let mut unit_start = Vec::with_capacity(cells.len()); + let mut prefix = vec![0.0]; + for (i, flow) in cells.iter().enumerate() { + let boundary = boundaries + .binary_search_by_key(&flow.cell.start, |(at, _)| *at) + .ok(); + let atomic = + i > 0 && (flow.keep_with_previous || flow.shares_previous_box || boundary.is_none()); + atomic_start.push(if atomic { atomic_start[i - 1] } else { i }); + let joined = atomic || boundary.is_some_and(|at| !boundaries[at].1); + unit_start.push(if i > 0 && joined { + unit_start[i - 1] + } else { + i + }); + prefix.push(prefix[i] + flow.cell.extent); + } + let mut placements = Vec::with_capacity(cells.len()); + let (mut column, mut column_start, mut cursor) = (0, 0, 0.0); + for (i, flow) in cells.iter().enumerate() { + if cursor > 0.0 && cursor + flow.cell.extent > max_height + FIT_TOLERANCE { + let candidate = unit_start[i]; + let break_at = if candidate > column_start + && prefix[i + 1] - prefix[candidate] <= max_height + FIT_TOLERANCE + { + candidate + } else { + atomic_start[i] + }; + // An overlong unit may break, but a grapheme or annotation may not. + if break_at > column_start { + column += 1; + cursor = 0.0; + for k in break_at..i { + placements[k] = (column, cursor); + cursor += cells[k].cell.extent; + } + column_start = break_at; + } + } + placements.push((column, cursor)); + cursor += flow.cell.extent; + } + placements +} + /// True when `max_height` is a real wrap budget (auto-width columns are /// unbounded). pub(super) fn is_bounded(max_height: f32) -> bool { diff --git a/render-wasm/src/shapes/text_vertical/layout.rs b/render-wasm/src/shapes/text_vertical/layout.rs index 21623bad87..365002aaed 100644 --- a/render-wasm/src/shapes/text_vertical/layout.rs +++ b/render-wasm/src/shapes/text_vertical/layout.rs @@ -10,6 +10,7 @@ use crate::globals::get_resources; use crate::math::Rect; use crate::shapes::japanese::JapaneseClass; use crate::shapes::text_japanese::EMPHASIS_FONT_SCALE; +use crate::shapes::text_policy::TextLayoutPolicy; use crate::shapes::{ merge_fills, AppliedTextTransform, FontFeatures, GrowType, LineAdjustment, Paragraph, RubySide, TextAlign, TextContent, TextSpan, VerticalAlign, @@ -25,7 +26,8 @@ use super::cells::{Fonts, SpanCells, WarichuNote}; use super::flow::{ aki_classes, align_offset_along_column, apply_inter_script_spacing, apply_ordered_oikomi, flow_classes, is_bounded, materialize_explicit_pair_spacing, ordered_expansion_offsets, - plan_with_edge_trimming, preferred_pair_spacing, shed_punctuation_aki, FlowCell, FIT_TOLERANCE, + plan_policy_columns, plan_with_edge_trimming, preferred_pair_spacing, shed_punctuation_aki, + FlowCell, FIT_TOLERANCE, }; use super::paint::LayoutDraws; use super::shaping::ShapedRun; @@ -516,6 +518,7 @@ fn aligned_tops( /// A paragraph's flow after spacing and column planning. struct PlannedParagraph { + policy: TextLayoutPolicy, flow: Vec, ruby_units: Vec, classes: Vec>, @@ -554,6 +557,26 @@ impl PlannedParagraph { let mut flow = cells.flow(&warichu_splits, runs); let mut ruby_units = ruby_units.clone(); + let policy = cells.paragraph.layout_policy(); + if policy != TextLayoutPolicy::Japanese { + grow_ruby_bases(&mut flow, &ruby_units); + let text: String = cells + .text_map + .span_transforms + .iter() + .map(|t| t.text.as_str()) + .collect(); + let placements = plan_policy_columns(&flow, &text, policy, max_height); + return Self { + policy, + classes: vec![None; flow.len()], + pair_spacing_em: vec![0.0; flow.len().saturating_sub(1)], + flow, + ruby_units, + placements, + }; + } + apply_inter_script_spacing(&mut flow); let classes = flow_classes(&flow, &ruby_spans); let aki = aki_classes(&flow, &classes, &vpal_spans); @@ -586,6 +609,7 @@ impl PlannedParagraph { Some(split) => warichu_splits.push(split), None => { return Self { + policy, flow, ruby_units, classes, @@ -604,6 +628,12 @@ impl PlannedParagraph { /// Final cells, aligned along their columns, numbered from `column_base`. /// A later span's cell of a warichu piece takes the piece's box. fn place(&self, align: TextAlign, max_height: f32, column_base: usize) -> Vec { + // The POC defines no non-Japanese justification expansion table yet. + let align = if self.policy != TextLayoutPolicy::Japanese && align == TextAlign::Justify { + TextAlign::Left + } else { + align + }; let tops = aligned_tops( &self.flow, &self.classes, @@ -777,6 +807,82 @@ mod tests { use super::*; use crate::shapes::TextOrientation; + #[test] + fn explicit_profiles_change_wrapping_without_japanese_spacing() { + let columns = |language: &str, spaces_only: bool, texts: &[&str]| { + let mut paragraph = vertical_paragraph( + texts.iter().map(|text| make_span(text)).collect(), + TextAlign::Left, + 1.0, + ); + paragraph.set_language(language.to_string()); + paragraph.set_korean_spaces_only(spaces_only); + let content = content_of(vec![paragraph], EM * 2.1, GrowType::Fixed); + let layout = layout_with_height( + &provider(include_bytes!("../../fonts/cjk-locl-test.ttf")), + &content, + EM * 2.1, + ); + ( + layout + .cells + .iter() + .map(|cell| (cell.start, cell.column)) + .collect::>(), + layout, + ) + }; + // Small kana stay attached in Japanese only; Chinese uses its own + // basic punctuation policy even when the fixture contains kana. + let (ja, _) = columns("ja", false, &["文あっ文"]); + let (zh, _) = columns("zh-Hans", false, &["文あっ文"]); + assert_ne!(ja, zh); + let (syllables, _) = columns("ko", false, &["한", "국어 문장"]); + let (words, layout) = columns("ko", true, &["한", "국어 문장"]); + // A three-syllable word cannot fit here: safe emergency wrapping. + assert!(words.iter().any(|(_, column)| *column > 0)); + assert_eq!(syllables.len(), words.len()); + assert!(layout + .cells + .iter() + .all(|cell| cell.minimum_oikomi_extent <= cell.extent)); + } + + #[test] + fn korean_word_boundaries_cross_styles_and_graphemes_stay_together() { + let mut paragraph = vertical_paragraph( + vec![make_span("한글 "), make_span("한"), make_span("국어")], + TextAlign::Left, + 1.0, + ); + paragraph.set_language("ko".to_string()); + paragraph.set_korean_spaces_only(true); + let content = content_of(vec![paragraph], EM * 3.6, GrowType::Fixed); + let layout = layout_with_height( + &provider(include_bytes!("../../fonts/cjk-locl-test.ttf")), + &content, + EM * 3.6, + ); + let word: Vec<_> = layout.cells.iter().filter(|cell| cell.start >= 3).collect(); + assert_eq!(word.len(), 3); + assert!(word.iter().all(|cell| cell.column == word[0].column)); + assert!(word[0].column > 0); + + let mut paragraph = + vertical_paragraph(vec![make_span("ᄒ"), make_span("ᅡᆫ국")], TextAlign::Left, 1.0); + paragraph.set_language("ko".to_string()); + let content = content_of(vec![paragraph], EM, GrowType::Fixed); + let layout = layout_with_height( + &provider(include_bytes!("../../fonts/cjk-locl-test.ttf")), + &content, + EM, + ); + let syllable: Vec<_> = layout.cells.iter().filter(|cell| cell.start < 3).collect(); + assert!(syllable + .iter() + .all(|cell| cell.column == syllable[0].column)); + } + fn with_headless_fonts(test: impl FnOnce() -> T) -> T { let mut resources = crate::render::RenderResources::try_new_headless().expect("headless resources"); diff --git a/render-wasm/src/shapes/text_vertical/orientation.rs b/render-wasm/src/shapes/text_vertical/orientation.rs index b11731738c..2e20eb98fd 100644 --- a/render-wasm/src/shapes/text_vertical/orientation.rs +++ b/render-wasm/src/shapes/text_vertical/orientation.rs @@ -1,4 +1,5 @@ use crate::shapes::TextOrientation; +use unicode_segmentation::UnicodeSegmentation; /// Emoji ranges recognized by the frontend font-loader. Emoji stay upright /// in vertical flow; Latin text rotates under `text-orientation: mixed`. @@ -18,6 +19,11 @@ pub(crate) fn is_upright_char(c: char) -> bool { is_emoji_char(c) || matches!(u32::from(c), 0x2E80..=0x2FDF // CJK radicals, Kangxi radicals + | 0x1100..=0x11FF // conjoining Hangul Jamo + | 0x3100..=0x318F // Bopomofo and compatibility Jamo + | 0x31A0..=0x31BF // Bopomofo extended + | 0xA960..=0xA97F // Hangul Jamo extended A + | 0xD7B0..=0xD7FF // Hangul Jamo extended B | 0x3000..=0x303F // CJK symbols and punctuation | 0x3040..=0x30FF // hiragana, katakana | 0x31C0..=0x31EF // CJK strokes @@ -78,7 +84,8 @@ pub(super) struct Segment { pub(super) fn segment_by_orientation(text: &str, orientation: TextOrientation) -> Vec { let mut segments: Vec = Vec::new(); let mut utf16_offset = 0; - for c in text.chars() { + for grapheme in text.graphemes(true) { + let c = grapheme.chars().next().unwrap(); let upright = orientation == TextOrientation::Upright || is_upright_char(c); let emoji = is_emoji_char(c); // CJK and emoji are both upright, but split them so the segment @@ -90,15 +97,15 @@ pub(super) fn segment_by_orientation(text: &str, orientation: TextOrientation) - if last.upright == upright && last.text.chars().next_back().is_some_and(is_emoji_char) == emoji => { - last.text.push(c) + last.text.push_str(grapheme) } _ => segments.push(Segment { - text: c.to_string(), + text: grapheme.to_string(), utf16_start: utf16_offset, upright, }), } - utf16_offset += c.len_utf16(); + utf16_offset += grapheme.encode_utf16().count(); } segments } diff --git a/render-wasm/src/shapes/text_vertical/shaping.rs b/render-wasm/src/shapes/text_vertical/shaping.rs index 82a957383a..cdc7e1a5d5 100644 --- a/render-wasm/src/shapes/text_vertical/shaping.rs +++ b/render-wasm/src/shapes/text_vertical/shaping.rs @@ -329,11 +329,12 @@ pub(super) fn shape_segment( upright: bool, font_features: FontFeatures, fallback: FontMgr, + language: &str, ) -> Vec { let mut font_iter = skia::Shaper::new_font_mgr_run_iterator(text, font, Some(fallback.clone())); let mut bidi_iter = skia::shapers::primitive::trivial_bidi_run_iterator(0, text.len()); let mut script_iter = skia::Shaper::new_hb_icu_script_run_iterator(text); - let mut lang_iter = skia::Shaper::new_trivial_language_run_iterator("ja", text.len()); + let mut lang_iter = skia::Shaper::new_trivial_language_run_iterator(language, text.len()); let mut features = if upright { vec![feature(b"vert"), feature(b"vrt2")] @@ -421,6 +422,7 @@ fn typeface_chunks(text: &str, typefaces: &[Typeface]) -> Vec<(Range, usi /// Shape a segment with explicit per-character font fallback. /// `TypefaceFontProvider` resolves named families, but some Skia builds lack /// its character fallback hook, so glyph coverage is resolved here. +#[allow(clippy::too_many_arguments)] pub(crate) fn shape_segment_with_fallbacks( text: &str, font_size: f32, @@ -429,6 +431,7 @@ pub(crate) fn shape_segment_with_fallbacks( upright: bool, font_features: FontFeatures, fallback_mgr: &FontMgr, + language: &str, ) -> Vec { shape_with_typefaces( text, @@ -437,6 +440,7 @@ pub(crate) fn shape_segment_with_fallbacks( upright, font_features, fallback_mgr, + language, ) } @@ -459,6 +463,7 @@ pub(crate) fn shape_with_typefaces( upright: bool, font_features: FontFeatures, fallback_mgr: &FontMgr, + language: &str, ) -> Vec { if typefaces.is_empty() { return Vec::new(); @@ -473,6 +478,7 @@ pub(crate) fn shape_with_typefaces( upright, font_features, fallback_mgr.clone(), + language, ); for run in &mut shaped { run.utf8_range = @@ -544,6 +550,38 @@ mod tests { use super::super::test_support::*; use super::*; + #[test] + fn selected_language_reaches_horizontal_and_vertical_localized_glyphs() { + use crate::shapes::{GrowType, TextAlign}; + use skia_safe::textlayout::{FontCollection, ParagraphBuilder}; + use std::collections::HashSet; + + let font_provider = provider(include_bytes!("../../fonts/cjk-locl-test.ttf")); + let mut fonts = FontCollection::new(); + fonts.set_asset_font_manager(Some(font_provider.clone().into())); + for (language, glyph, width) in [("ja", 2, 12.0), ("zh-Hans", 3, 14.0), ("ko", 4, 16.0)] { + let mut paragraph = vertical_paragraph(vec![make_span("中")], TextAlign::Left, 1.0); + paragraph.set_language(language.to_string()); + let content = content_of(vec![paragraph.clone()], 100.0, GrowType::Fixed); + let layout = layout_with(&font_provider, &content); + assert_eq!(layout.runs[0].glyphs, vec![glyph], "vertical {language}"); + + let span = ¶graph.children()[0]; + let style = span.to_style(&content.bounds(), &HashSet::new(), false, 1.0); + assert_eq!(style.locale(), language); + let mut builder = ParagraphBuilder::new(¶graph.paragraph_to_style(), &fonts); + builder.push_style(&style); + builder.add_text(&span.text); + let mut horizontal = builder.build(); + horizontal.layout(100.0); + assert!( + (horizontal.longest_line() - width).abs() < 0.01, + "horizontal {language}: {}", + horizontal.longest_line() + ); + } + } + #[test] fn span_families_try_emoji_after_the_span_font() { let families = span_font_families(&make_span("😆"), &["fallback".to_string()]); diff --git a/render-wasm/src/shapes/text_vertical/test_support.rs b/render-wasm/src/shapes/text_vertical/test_support.rs index f6872c9d66..978b68538c 100644 --- a/render-wasm/src/shapes/text_vertical/test_support.rs +++ b/render-wasm/src/shapes/text_vertical/test_support.rs @@ -70,11 +70,12 @@ pub(super) fn vertical_paragraph( spans, ); paragraph.set_writing_mode(WritingMode::VerticalRl); + paragraph.set_language("ja".to_string()); paragraph } /// A 200px wide box `height` tall holding `paragraphs`. -pub(super) fn content_of( +pub(crate) fn content_of( paragraphs: Vec, height: f32, grow_type: GrowType, @@ -167,7 +168,7 @@ pub(super) fn layout_with_fallback( ) } -pub(super) fn layout_with_height( +pub(crate) fn layout_with_height( provider: &TypefaceFontProvider, content: &TextContent, max_height: f32, diff --git a/render-wasm/src/wasm/text.rs b/render-wasm/src/wasm/text.rs index ec252b87dd..75e7d4311c 100644 --- a/render-wasm/src/wasm/text.rs +++ b/render-wasm/src/wasm/text.rs @@ -108,10 +108,12 @@ pub struct RawParagraphData { text_transform: RawTextTransform, writing_mode: u8, text_orientation: u8, + korean_wrap: u8, // Padding for the CLJS writer's 4-byte-aligned layout; always zero. - _padding: [u8; 2], + _padding: [u8; 1], line_height: f32, letter_spacing: f32, + language_length: u32, } impl From<[u8; RAW_PARAGRAPH_DATA_SIZE]> for RawParagraphData { @@ -297,6 +299,12 @@ impl From for shapes::Paragraph { paragraph.set_writing_mode(RawWritingMode::from(value.attrs.writing_mode).into()); paragraph .set_text_orientation(RawTextOrientation::from(value.attrs.text_orientation).into()); + let language = ruby_offset + .checked_add(value.attrs.language_length as usize) + .and_then(|end| value.text_buffer.get(ruby_offset..end)) + .unwrap_or(&[]); + paragraph.set_language(String::from_utf8_lossy(language).into_owned()); + paragraph.set_korean_spaces_only(value.attrs.korean_wrap == 1); paragraph } } @@ -507,13 +515,13 @@ mod tests { AnnotationClearance, FontFeatures, RubyAlign, RubyOverhang, RubySide, RubySize, }; - /// The CLJS writer (texts.cljs) writes PARAGRAPH-ATTR-U8-SIZE (16) + /// The shared CLJS writer writes PARAGRAPH-ATTR-U8-SIZE (20) /// attr bytes after the u32 span count, and SPAN-ATTR-U8-SIZE (80) /// attr bytes before the fills block. These sizes must match the struct /// layouts. #[test] fn raw_struct_sizes_match_cljs_writer() { - const PARAGRAPH_ATTR_U8_SIZE: usize = 16; + const PARAGRAPH_ATTR_U8_SIZE: usize = 20; const SPAN_ATTR_U8_SIZE: usize = 80; assert_eq!(RAW_PARAGRAPH_DATA_SIZE, 4 + PARAGRAPH_ATTR_U8_SIZE); assert_eq!( @@ -522,6 +530,26 @@ mod tests { ); } + #[test] + fn language_decodes_after_base_and_ruby_text() { + let mut attrs = [0u8; RAW_PARAGRAPH_DATA_SIZE]; + attrs[10] = 1; // Korean spaces-only POC; byte 6 after span count. + attrs[20..24].copy_from_slice(&2u32.to_le_bytes()); + let mut span = [0u8; RAW_SPAN_DATA_SIZE]; + span[68..72].copy_from_slice(&3u32.to_le_bytes()); + span[72..76].copy_from_slice(&3u32.to_le_bytes()); + let paragraph = shapes::Paragraph::from(RawParagraph { + attrs: RawParagraphData::from(attrs), + spans: vec![RawTextSpan::from(span)], + text_buffer: "韓한ko".as_bytes().to_vec(), + }); + assert_eq!(paragraph.language(), "ko"); + assert!(paragraph.korean_spaces_only()); + assert_eq!(paragraph.children()[0].text, "韓"); + assert_eq!(paragraph.children()[0].ruby, "한"); + assert_eq!(paragraph.children()[0].language, "ko"); + } + #[test] fn raw_text_combine_upright_counts_deserialize() { // Byte 5 of the span attr block; the counted digits variants map diff --git a/render-wasm/src/wasm/text/helpers.rs b/render-wasm/src/wasm/text/helpers.rs index a0ae969667..5a442257f4 100644 --- a/render-wasm/src/wasm/text/helpers.rs +++ b/render-wasm/src/wasm/text/helpers.rs @@ -477,11 +477,8 @@ pub fn delete_selection_range(text_content: &mut TextContent, selection: &TextSe delete_range_in_paragraph(&mut paragraphs[end_paragraph], 0, end.offset); - let end_para_children: Vec<_> = - paragraphs[end_paragraph].children_mut().drain(..).collect(); - paragraphs[start.paragraph] - .children_mut() - .extend(end_para_children); + let end_para_children = std::mem::take(paragraphs[end_paragraph].children_mut()); + paragraphs[start.paragraph].extend_children(end_para_children); paragraphs.drain((start.paragraph + 1)..=end_paragraph); @@ -593,13 +590,8 @@ pub fn delete_char_before( let paragraphs = text_content.paragraphs_mut(); let prev_para_len = paragraph_char_count(¶graphs[prev_para_idx]); - let current_children: Vec<_> = paragraphs[cursor.paragraph] - .children_mut() - .drain(..) - .collect(); - paragraphs[prev_para_idx] - .children_mut() - .extend(current_children); + let current_children = std::mem::take(paragraphs[cursor.paragraph].children_mut()); + paragraphs[prev_para_idx].extend_children(current_children); paragraphs.remove(cursor.paragraph); @@ -770,10 +762,8 @@ pub fn delete_char_after(text_content: &mut TextContent, cursor: &TextPositionWi delete_range_in_paragraph(para, cursor.offset, end); } else if cursor.paragraph < paragraphs.len() - 1 { let next_para_idx = cursor.paragraph + 1; - let next_children: Vec<_> = paragraphs[next_para_idx].children_mut().drain(..).collect(); - paragraphs[cursor.paragraph] - .children_mut() - .extend(next_children); + let next_children = std::mem::take(paragraphs[next_para_idx].children_mut()); + paragraphs[cursor.paragraph].extend_children(next_children); paragraphs.remove(next_para_idx); } @@ -825,6 +815,9 @@ pub fn split_paragraph_at_cursor( let text_transform = para.text_transform(); let writing_mode = para.writing_mode(); let text_orientation = para.text_orientation(); + let language = para.language().to_owned(); + let korean_spaces_only = para.korean_spaces_only(); + let source_position = para.source_position(); let line_height = para.line_height(); let letter_spacing = para.letter_spacing(); @@ -851,6 +844,9 @@ pub fn split_paragraph_at_cursor( ); new_para.set_writing_mode(writing_mode); new_para.set_text_orientation(text_orientation); + new_para.set_language(language); + new_para.set_korean_spaces_only(korean_spaces_only); + new_para.set_source_position(source_position); paragraphs.insert(cursor.paragraph + 1, new_para); @@ -864,6 +860,51 @@ mod tests { use crate::shapes::{FontFamily, FontStyle, GrowType, TextAlign, TextSpan}; use crate::uuid::Uuid; + #[test] + fn paragraph_split_keeps_language_and_korean_wrap() { + let mut para = paragraph(&["한국어"]); + para.set_language("ko".to_string()); + para.set_korean_spaces_only(true); + let mut content = content(vec![para]); + let cursor = TextPositionWithAffinity { + paragraph: 0, + offset: 1, + ..Default::default() + }; + assert!(split_paragraph_at_cursor(&mut content, &cursor)); + assert_eq!(content.paragraphs().len(), 2); + for para in content.paragraphs() { + assert_eq!(para.language(), "ko"); + assert!(para.korean_spaces_only()); + assert_eq!(para.source_position(), 0); + assert!(para.children().iter().all(|span| span.language == "ko")); + } + } + + #[test] + fn paragraph_merge_inherits_the_surviving_paragraph_language() { + let mut first = paragraph(&["中"]); + first.set_language("ja".to_string()); + let mut second = paragraph(&["中"]); + second.set_language("ko".to_string()); + let mut content = content(vec![first, second]); + delete_char_after( + &mut content, + &TextPositionWithAffinity { + paragraph: 0, + offset: 1, + ..Default::default() + }, + ); + assert_eq!(content.paragraphs().len(), 1); + let paragraph = &content.paragraphs()[0]; + assert_eq!(paragraph.language(), "ja"); + assert!(paragraph + .children() + .iter() + .all(|span| span.language == "ja")); + } + fn span(text: &str) -> TextSpan { TextSpan::new( text.to_string(), diff --git a/render-wasm/src/wasm/text_editor.rs b/render-wasm/src/wasm/text_editor.rs index a16df85d13..bd875ffdcb 100644 --- a/render-wasm/src/wasm/text_editor.rs +++ b/render-wasm/src/wasm/text_editor.rs @@ -984,8 +984,11 @@ pub extern "C" fn text_editor_export_content() -> *mut u8 { .replace('\r', "\\r") .replace('\t', "\\t"); span_parts.push(format!( - "{{\"p\":{},\"s\":{},\"t\":\"{}\"}}", - span.paragraph_position, span.span_position, escaped_text + "{{\"pp\":{},\"p\":{},\"s\":{},\"t\":\"{}\"}}", + para.source_position(), + span.paragraph_position, + span.span_position, + escaped_text )); } json_parts.push(format!("[{}]", span_parts.join(",")));