github/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2026-10-29 05:50:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

✨ Add a CJK text layout proof of concept

Browse Source 
Choose paragraph layout rules from an optional language tag. Pass language
to horizontal and vertical shaping, add basic Chinese punctuation rules,
and add experimental Korean spaces-only wrapping.

Keep text boundaries grapheme-safe and preserve paragraph profiles across
editor splits and merges. Add reproducible font fixtures for localized
glyphs and regression tests for the shared WASM protocol and layout paths.

Validation: renderer, common JVM/JS, frontend and exporter tests pass.
Both WASM targets build. Full CI still reports baseline lint and format
errors and one backend multipart-upload test failure.

AI-assisted-by: GPT-6
This commit is contained in:
alonso.torres 2026-10-19 18:03:54 +02:00
parent e04d02d722
commit 0dd12b09e9
27 changed files with 819 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
common/src/app/common/render_wasm/text_content.cljs
View File

@ -6,8 +6,9 @@
(ns app.common.render-wasm.text-content
"Single source of truth for writing a text shape's content into the WASM design
state. The binary layout ([num-spans][paragraph attrs][span attrs][text][ruby-text]) is
identical for the workspace and the headless exporter, and so is the font-id
state. The binary layout is [num-spans][paragraph attrs][span attrs][text]
[ruby-text][language], identical for the workspace and headless exporter.
Both hosts also share the font-id
-> uuid mapping (`cfnt/font-id->uuid`). Only *variant* resolution differs —
the workspace has a loaded fonts DB, the exporter does not — so that part is
injected via the `opts` map passed to `write-shape-text!`.
@ -24,7 +25,7 @@
[app.common.uuid :as uuid]
[cuerdas.core :as str]))
(def ^:const PARAGRAPH-ATTR-U8-SIZE 16)
(def ^:const PARAGRAPH-ATTR-U8-SIZE 20)
(def ^:const SPAN-ATTR-U8-SIZE 80)
(def ^:const MAX-TEXT-FILLS types.fills.impl/MAX-FILLS)
@ -126,11 +127,12 @@
(mem/write-u8 dview text-transform)
(mem/write-u8 dview writing-mode)
(mem/write-u8 dview text-orientation)
(mem/write-u8 dview (if (= "spaces-only" (:korean-wrap paragraph)) 1 0))
;; Alignment padding; must match RawParagraphData in Rust.
(mem/write-u8 dview 0)
(mem/write-u8 dview 0)
(mem/write-f32 dview line-height)
(mem/write-f32 dview letter-spacing)
(mem/write-u32 dview (mem/size (encode-text (d/nilv (:text-language paragraph) ""))))
(mem/assert-written offset PARAGRAPH-ATTR-U8-SIZE))))
(defn- write-spans
@ -225,24 +227,26 @@
:or {normalize-font-id cfnt/font-id->uuid
normalize-paragraph identity
normalize-span (fn [span _paragraph] span)}}]
(let [paragraph (normalize-paragraph paragraph)
spans (map #(normalize-span % paragraph) spans)
num-spans (count spans)
fills-size (* types.fills.impl/FILL-U8-SIZE MAX-TEXT-FILLS)
metadata-size (+ PARAGRAPH-ATTR-U8-SIZE
(* num-spans (+ SPAN-ATTR-U8-SIZE fills-size)))
text-buffer (encode-text text)
text-size (mem/size text-buffer)
ruby-buffer (encode-text (apply str (map span-ruby spans)))
ruby-size (mem/size ruby-buffer)
total-size (+ 4 metadata-size text-size ruby-size)
heapu8 (mem/get-heap-u8)
dview (mem/get-data-view)
offset (mem/alloc total-size)]
(let [paragraph (normalize-paragraph paragraph)
spans (map #(normalize-span % paragraph) spans)
num-spans (count spans)
fills-size (* types.fills.impl/FILL-U8-SIZE MAX-TEXT-FILLS)
metadata-size (+ PARAGRAPH-ATTR-U8-SIZE
(* num-spans (+ SPAN-ATTR-U8-SIZE fills-size)))
text-buffer (encode-text text)
text-size (mem/size text-buffer)
ruby-buffer (encode-text (apply str (map span-ruby spans)))
ruby-size (mem/size ruby-buffer)
language-buffer (encode-text (d/nilv (:text-language paragraph) ""))
total-size (+ 4 metadata-size text-size ruby-size (mem/size language-buffer))
heapu8 (mem/get-heap-u8)
dview (mem/get-data-view)
offset (mem/alloc total-size)]
(-> offset
(mem/write-u32 dview num-spans)
(write-paragraph dview paragraph)
(write-spans dview spans paragraph normalize-font-id)
(mem/write-buffer heapu8 text-buffer)
(mem/write-buffer heapu8 ruby-buffer))
(mem/write-buffer heapu8 ruby-buffer)
(mem/write-buffer heapu8 language-buffer))
(h/call wasm/internal-module "_set_shape_text_content")))

6
common/src/app/common/types/shape/text.cljc
View File

@ -44,6 +44,12 @@
[:font-style {:optional true} ::sm/text]
[:font-weight {:optional true} ::sm/text]
[:direction {:optional true} ::sm/text]
;; Paragraph language selects typography; it never follows the UI locale.
[:text-language {:optional true}
[:re {:gen/elements ["ja" "zh-Hans" "zh-Hant-TW" "ko" "en-GB"]}
#"^(?:[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*|[xXiI](?:-[A-Za-z0-9]{1,8})+)$"]]
;; Experimental POC mode, deliberately distinct from CSS keep-all.
[:korean-wrap {:optional true} [:enum "syllable" "spaces-only"]]
[:writing-mode {:optional true} (schema:jl-enum :writing-mode)]
[:text-orientation {:optional true} (schema:jl-enum :text-orientation)]
[:text-decoration {:optional true} ::sm/text]

1
common/src/app/common/types/text.cljc
View File

@ -108,6 +108,7 @@
(d/concat-vec
text-align-attrs
text-direction-attrs
[:text-language :korean-wrap]
cjk/whole-shape-paragraph-attrs))
(def text-node-attrs

13
common/test/common_tests/types/text_test.cljc
View File

@ -379,6 +379,19 @@
[attrs]
(update-in content-base [:children 0 :children 0] merge attrs))
(t/deftest paragraph-language-and-experimental-korean-wrap
(doseq [language ["ja" "zh-Hans" "zh-Hant-TW" "ko" "en-GB"]]
(t/is (ctst/valid-content?
(with-paragraph-attrs {:text-language language
:korean-wrap "spaces-only"}))))
(doseq [attrs [{:text-language 42}
{:text-language "zh_Hans"}
{:text-language ""}
{:korean-wrap "keep-all"}]]
(t/is (not (ctst/valid-content? (with-paragraph-attrs attrs))) (pr-str attrs)))
(t/is (contains? (set cttx/paragraph-attrs) :text-language))
(t/is (not (contains? (set cjk/whole-shape-paragraph-attrs) :text-language))))
(t/deftest japanese-span-attrs-accept-supported-values
(t/is (ctst/valid-content?
(with-span-attrs {:text-combine-upright "digits2"

8
frontend/src/app/render_wasm/text_editor.cljs
View File

@ -610,7 +610,10 @@
styling is taken from there; a span WASM never saw falls back to its position
and then to the last existing span.
exported vector of paragraphs, each a vector of `{:p 0 :s 0 :t \"text\"}`
`:pp` tracks paragraph attrs independently of span styling, including
when a merge removes all the surviving paragraph's original characters.
exported vector of paragraphs, each a vector of `{:pp 0 :p 0 :s 0 :t \"text\"}`
content existing Penpot content map (root -> paragraph-set -> …)"
[content exported]
(let [para-set (first (get content :children))
@ -621,7 +624,8 @@
styling-para
(fn [para-idx spans]
(or (get orig-paras (get (first spans) :p))
(or (get orig-paras (get (first spans) :pp))
(get orig-paras (get (first spans) :p))
(get orig-paras para-idx)
last-orig-para))

43
frontend/test/frontend_tests/render_wasm/texts_test.cljs
View File

@ -314,6 +314,49 @@
(t/is (not-any? #(contains? % :text-orientation) paragraphs))
(t/is (= [nil "center" nil] (mapv :text-align paragraphs)))))
(t/deftest language-and-wrapping-only-change-selected-paragraphs
(let [content (editor-content [{:text "日本"}] [{:text "한국어"}] [{:text "中文"}])
result (text-editor/apply-paragraph-attrs-to-range
content (selection 1 0 1 0)
{:text-language "ko" :korean-wrap "spaces-only"})
paragraphs (-> result :children first :children)]
(t/is (= [nil "ko" nil] (mapv :text-language paragraphs)))
(t/is (= [nil "spaces-only" nil] (mapv :korean-wrap paragraphs)))
(t/is (= result
(text-editor/merge-exported-texts-into-content
result [[{:p 0 :s 0 :t "日本"}]
[{:p 1 :s 0 :t "한국어"}]
[{:p 2 :s 0 :t "中文"}]]))))
(let [content (assoc-in (editor-content [{:text "한글"}])
[:children 0 :children 0 :text-language] "ko")
result (text-editor/merge-exported-texts-into-content
content [[{:p 0 :s 0 :t "한"}] [{:p 0 :s 0 :t "글"}]])]
(t/is (= ["ko" "ko"] (mapv :text-language (-> result :children first :children))))))
(t/deftest paragraph-language-binary-header-matches-rust
(let [dview (js/DataView. (js/ArrayBuffer. tc/PARAGRAPH-ATTR-U8-SIZE))
end (@#'tc/write-paragraph 0 dview
{:text-language "zh-Hant-TW" :korean-wrap "spaces-only"
:line-height "1.5" :letter-spacing "2"})]
(t/is (= 20 end))
(t/is (= 1 (.getUint8 dview 6)))
(t/is (= 0 (.getUint8 dview 7)))
(t/is (= 1.5 (.getFloat32 dview 8 true)))
(t/is (= 2 (.getFloat32 dview 12 true)))
(t/is (= 10 (.getUint32 dview 16 true)))))
(t/deftest merging-away-original-spans-keeps-the-surviving-paragraph-profile
(let [content (-> (editor-content [{:text "日"}] [{:text "한"}])
(assoc-in [:children 0 :children 0 :text-language] "ja")
(assoc-in [:children 0 :children 1 :text-language] "ko")
(assoc-in [:children 0 :children 1 :korean-wrap] "spaces-only"))
result (text-editor/merge-exported-texts-into-content
content [[{:pp 0 :p 1 :s 0 :t "한"}]])
para (get-in result [:children 0 :children 0])]
(t/is (= "ja" (:text-language para)))
(t/is (not (contains? para :korean-wrap)))
(t/is (= "한" (get-in para [:children 0 :text])))))
;; Byte offsets inside one span's attribute block (see `write-spans` and
;; RawTextSpan in render-wasm).
(def ^:private span-attr-offset

67
render-wasm/scripts/build-cjk-test-font.py Normal file
View File

@ -0,0 +1,67 @@
#!/usr/bin/env python3
"""Build the synthetic CJK language fixture (requires fonttools).
The one Han character has distinct JAN, ZHS and KOR localized glyphs and
advances. This tests language propagation without a system font dependency.
All outlines are original rectangles; no external font data is included.
"""
from pathlib import Path
from fontTools.feaLib.builder import addOpenTypeFeaturesFromString
from fontTools.fontBuilder import FontBuilder
from fontTools.pens.ttGlyphPen import TTGlyphPen
def main():
builder = FontBuilder(1000, isTTF=True)
names = [".notdef", "base", "japanese", "chinese", "korean", "square", "space", "thirdEm"]
builder.setupGlyphOrder(names)
cmap = {ord(char): "square" for char in "文あっ한국어문장한글한「」，。"}
cmap.update({0x4E2D: "base", 0x20: "space", 0x2004: "thirdEm"})
builder.setupCharacterMap(cmap)
glyphs = {}
for index, name in enumerate(names):
pen = TTGlyphPen(None)
pen.moveTo((50, 0))
pen.lineTo((350 + index * 20, 0))
pen.lineTo((350 + index * 20, 700))
pen.lineTo((50, 700))
pen.closePath()
glyphs[name] = pen.glyph()
builder.setupGlyf(glyphs)
metrics = {name: (400 + index * 100, 50) for index, name in enumerate(names)}
metrics.update({"square": (1000, 50), "space": (500, 50), "thirdEm": (333, 50)})
builder.setupHorizontalMetrics(metrics)
builder.setupHorizontalHeader(ascent=800, descent=-200)
builder.setupNameTable({
"familyName": "Penpot CJK Test",
"styleName": "Regular",
"uniqueFontIdentifier": "PenpotCJKTest",
"fullName": "Penpot CJK Test Regular",
"psName": "PenpotCJKTest-Regular",
})
builder.setupOS2(sTypoAscender=800, sTypoDescender=-200,
usWinAscent=800, usWinDescent=200)
builder.setupPost()
builder.setupMaxp()
addOpenTypeFeaturesFromString(builder.font, """
languagesystem DFLT dflt;
languagesystem hani JAN;
languagesystem hani ZHS;
languagesystem hani KOR;
feature locl {
script hani;
language JAN; sub base by japanese;
language ZHS; sub base by chinese;
language KOR; sub base by korean;
} locl;
""")
builder.font["head"].created = 3800000000
builder.font["head"].modified = 3800000000
builder.font.recalcTimestamp = False
builder.save(Path(__file__).resolve().parents[1] / "src/fonts/cjk-locl-test.ttf")
if __name__ == "__main__":
main()

BIN
render-wasm/src/fonts/cjk-locl-test.ttf Normal file
View File

Binary file not shown.

3
render-wasm/src/render/text.rs
View File

@ -1578,10 +1578,11 @@ mod tests {
#[test]
fn decorated_ranges_follow_inserted_kinsoku_characters() {
// A word joiner goes before 。, so the next span starts one unit later.
let para = paragraph(vec![
let mut para = paragraph(vec![
span("あ。", None, None),
span("い", Some(TextDecoration::UNDERLINE), None),
]);
para.set_language("ja".to_string());
assert_eq!(
decorated_span_ranges(&para, &HorizontalOffsets::new(&para)),

1
render-wasm/src/shapes.rs
View File

@ -33,6 +33,7 @@ mod text;
mod text_horizontal;
mod text_japanese;
pub mod text_paths;
pub mod text_policy;
pub mod text_vertical;
mod transform;

20
render-wasm/src/shapes/japanese.rs
View File

@ -509,26 +509,6 @@ pub fn is_japanese_text_char(c: char) -> bool {
|| matches!(u32::from(c), 0x3000..=0x30FF | 0x31F0..=0x31FF | 0xFF00..=0xFFEF)
}
/// True for characters that extend the preceding grapheme cluster
/// (Grapheme_Cluster_Break=Extend or ZWJ): combining marks, voiced sound
/// marks, variation selectors, emoji modifiers and tags.
pub fn extends_grapheme(c: char) -> bool {
matches!(u32::from(c),
0x0300..=0x036F
| 0x1AB0..=0x1AFF
| 0x1DC0..=0x1DFF
| 0x200C..=0x200D
| 0x20D0..=0x20FF
| 0x3099..=0x309A
| 0xFE00..=0xFE0F
| 0xFE20..=0xFE2F
| 0xFF9E..=0xFF9F
| 0x1F3FB..=0x1F3FF
| 0xE0020..=0xE007F
| 0xE0100..=0xE01EF
)
}
fn is_hiragana(c: char) -> bool {
matches!(u32::from(c), 0x3041..=0x309F)
}

49
render-wasm/src/shapes/kinsoku.rs
View File

@ -31,12 +31,10 @@ pub const WESTERN_WORD_SPACE: char = '\u{2004}';
/// Unicode IDEOGRAPHIC SPACE, the one-em space after a sentence-ending
/// question or exclamation mark.
pub const IDEOGRAPHIC_SPACE: char = '\u{3000}';
/// Joins emoji into one grapheme cluster.
const ZERO_WIDTH_JOINER: char = '\u{200D}';
use super::japanese::{
classify, extends_grapheme, keeps_together_classified, pair_rule, JapaneseClass,
};
use super::japanese::{classify, keeps_together_classified, pair_rule, JapaneseClass};
use super::text_policy::TextLayoutPolicy;
use unicode_segmentation::UnicodeSegmentation;
pub fn forbidden_at_line_start(c: char) -> bool {
classify(c).forbids_line_start()
@ -98,15 +96,26 @@ impl OffsetMap {
/// grapheme clusters are not, so nothing lands inside one. `ruby_breaks[span]
/// == Some(boundaries)` forbids breaks at every internal scalar boundary of
/// that span except those UTF-16 offsets. Returns `None` when nothing changes.
pub fn apply_to_span_texts_with_ruby_breaks(
#[cfg(test)]
fn apply_to_span_texts_with_ruby_breaks(
span_texts: &[String],
ruby_breaks: &[Option<Vec<usize>>],
) -> Option<(Vec<String>, OffsetMap)> {
apply_policy_to_span_texts(span_texts, ruby_breaks, TextLayoutPolicy::Japanese)
}
pub fn apply_policy_to_span_texts(
span_texts: &[String],
ruby_breaks: &[Option<Vec<usize>>],
policy: TextLayoutPolicy,
) -> Option<(Vec<String>, OffsetMap)> {
let source = span_texts.concat();
let boundaries: Vec<usize> = source.grapheme_indices(true).map(|(at, _)| at).collect();
let mut source_byte = 0;
let mut inserted: Vec<usize> = Vec::new();
let mut out: Vec<String> = Vec::with_capacity(span_texts.len());
// Base character of the previous grapheme cluster, with its class.
let mut prev: Option<(char, JapaneseClass)> = None;
let mut after_zwj = false;
let mut changed = false;
// Running position in shifted UTF-16 coordinates.
let mut shifted_pos: usize = 0;
@ -115,11 +124,12 @@ pub fn apply_to_span_texts_with_ruby_breaks(
let mut shifted_text = String::with_capacity(text.len() + 4);
let mut local_utf16 = 0usize;
for c in text.chars() {
if after_zwj || extends_grapheme(c) {
let at_boundary = boundaries.binary_search(&source_byte).is_ok();
source_byte += c.len_utf8();
if !at_boundary {
shifted_text.push(c);
shifted_pos += c.len_utf16();
local_utf16 += c.len_utf16();
after_zwj = c == ZERO_WIDTH_JOINER;
continue;
}
let current = (c, classify(c));
@ -130,8 +140,12 @@ pub fn apply_to_span_texts_with_ruby_breaks(
.is_some_and(|breaks| !breaks.contains(&local_utf16));
let forbid_break = ruby_forbids_break
|| prev.is_some_and(|p| {
pair_rule(p.1, current.1).suppress_break_with_joiner
|| keeps_together_classified(p, current)
if policy == TextLayoutPolicy::Japanese {
pair_rule(p.1, current.1).suppress_break_with_joiner
|| keeps_together_classified(p, current)
} else {
!policy.break_allowed(p.0, c)
}
});
let mut insert = |ch: char| {
shifted_text.push(ch);
@ -139,21 +153,26 @@ pub fn apply_to_span_texts_with_ruby_breaks(
shifted_pos += 1;
changed = true;
};
if prev.is_some_and(|p| takes_em_space_after(p, current)) {
if policy == TextLayoutPolicy::Japanese
&& prev.is_some_and(|p| takes_em_space_after(p, current))
{
insert(IDEOGRAPHIC_SPACE);
}
if prev.is_some_and(|p| is_japanese_western_boundary(p, current)) {
if policy == TextLayoutPolicy::Japanese
&& prev.is_some_and(|p| is_japanese_western_boundary(p, current))
{
insert(JAPANESE_WESTERN_SPACE);
}
// After an inserted space, the joiner keeps the break blocked.
if forbid_break {
insert(WORD_JOINER);
} else if current.1 == JapaneseClass::Inseparable
} else if policy == TextLayoutPolicy::Japanese
&& current.1 == JapaneseClass::Inseparable
&& prev.is_some_and(|p| pair_rule(p.1, current.1).break_allowed)
{
insert(BREAK_OPPORTUNITY);
}
let layout_char = if c == ' ' {
let layout_char = if policy == TextLayoutPolicy::Japanese && c == ' ' {
changed = true;
WESTERN_WORD_SPACE
} else {

58
render-wasm/src/shapes/text.rs
View File

@ -1864,6 +1864,10 @@ pub struct Paragraph {
text_transform: Option<TextTransform>,
writing_mode: WritingMode,
text_orientation: TextOrientation,
language: String,
korean_spaces_only: bool,
/// Paragraph in the content tree last exchanged with the editor host.
source_position: u32,
line_height: f32,
letter_spacing: f32,
children: Vec<TextSpan>,
@ -1878,6 +1882,9 @@ impl Default for Paragraph {
text_transform: None,
writing_mode: WritingMode::default(),
text_orientation: TextOrientation::default(),
language: String::new(),
korean_spaces_only: false,
source_position: u32::MAX,
line_height: 1.0,
letter_spacing: 0.0,
children: vec![],
@ -1903,6 +1910,9 @@ impl Paragraph {
text_transform,
writing_mode: WritingMode::default(),
text_orientation: TextOrientation::default(),
language: String::new(),
korean_spaces_only: false,
source_position: u32::MAX,
line_height,
letter_spacing,
children,
@ -1913,6 +1923,37 @@ impl Paragraph {
self.writing_mode
}
pub fn language(&self) -> &str {
&self.language
}
pub fn set_language(&mut self, language: String) {
for span in &mut self.children {
span.language.clone_from(&language);
}
self.language = language;
}
pub fn korean_spaces_only(&self) -> bool {
self.korean_spaces_only
}
pub fn set_korean_spaces_only(&mut self, value: bool) {
self.korean_spaces_only = value;
}
pub fn layout_policy(&self) -> super::text_policy::TextLayoutPolicy {
super::text_policy::TextLayoutPolicy::for_language(&self.language, self.korean_spaces_only)
}
pub fn source_position(&self) -> u32 {
self.source_position
}
pub fn set_source_position(&mut self, position: u32) {
self.source_position = position;
}
pub fn set_writing_mode(&mut self, writing_mode: WritingMode) {
self.writing_mode = writing_mode;
}
@ -1933,7 +1974,16 @@ impl Paragraph {
&mut self.children
}
/// Move spans into this paragraph, inheriting its shaping language.
pub fn extend_children(&mut self, spans: impl IntoIterator<Item = TextSpan>) {
self.children.extend(spans.into_iter().map(|mut span| {
span.language.clone_from(&self.language);
span
}));
}
fn set_span_positions(&mut self, index: u32) {
self.source_position = index;
for (span_index, span) in self.children.iter_mut().enumerate() {
span.set_position(index, span_index as u32);
}
@ -2041,7 +2091,7 @@ impl Paragraph {
/// Japanese spacing normalized, kinsoku break suppressions inserted),
/// plus the map from original to builder-text UTF-16 offsets. Consumers
/// of laid-out offsets must translate through the map. Only paragraphs
/// with Japanese text or ruby are transformed.
/// with an explicit typography profile are transformed.
pub fn layout_span_texts(&self) -> (Vec<String>, kinsoku::OffsetMap) {
layout_span_texts(self)
}
@ -2179,6 +2229,8 @@ fn apply_text_transform_with_source_ranges(
#[derive(Debug, PartialEq, Clone)]
pub struct TextSpan {
pub text: String,
/// Effective paragraph language, copied when the paragraph is decoded.
pub language: String,
pub font_family: FontFamily,
pub font_size: f32,
pub line_height: f32,
@ -2211,6 +2263,7 @@ impl Default for TextSpan {
fn default() -> Self {
Self {
text: String::default(),
language: String::new(),
font_family: FontFamily::new(Uuid::nil(), 400, FontStyle::Normal),
font_size: 14.0,
line_height: 1.2,
@ -2366,6 +2419,9 @@ impl TextSpan {
fill_layer_from_bottom: Option<usize>,
) -> skia::textlayout::TextStyle {
let mut style = skia::textlayout::TextStyle::default();
if !self.language.is_empty() {
style.set_locale(&self.language);
}
let paint = if remove_alpha {
let mut paint = paint::Paint::default();
paint.set_color(skia::Color::BLACK);

25
render-wasm/src/shapes/text_horizontal/mod.rs
View File

@ -27,6 +27,7 @@ use crate::math::Point;
use crate::shapes::japanese::{
aki_class, is_japanese_text_char, pair_rule, punctuation_aki_sheds, AkiShed, JapaneseClass,
};
use crate::shapes::text_policy::TextLayoutPolicy;
use crate::shapes::{kinsoku, merge_fills};
use crate::utils::{get_fallback_fonts, get_font_collection};
use skia_safe::{
@ -41,10 +42,11 @@ use skia_safe::{
/// True when a paragraph with layout texts `texts` needs the Japanese passes:
/// it has ruby or a Japanese character.
pub(crate) fn uses_japanese_layout(paragraph: &Paragraph, texts: &[String]) -> bool {
paragraph.children().iter().any(TextSpan::has_ruby)
|| texts
.iter()
.any(|text| text.chars().any(is_japanese_text_char))
paragraph.layout_policy() == TextLayoutPolicy::Japanese
&& (paragraph.children().iter().any(TextSpan::has_ruby)
|| texts
.iter()
.any(|text| text.chars().any(is_japanese_text_char)))
}
pub(crate) fn layout_span_texts(paragraph: &Paragraph) -> (Vec<String>, kinsoku::OffsetMap) {
@ -54,14 +56,22 @@ pub(crate) fn layout_span_texts(paragraph: &Paragraph) -> (Vec<String>, kinsoku:
.map(TextSpan::apply_text_transform)
.collect();
// Text without Japanese characters or ruby keeps plain Skia layout.
if uses_japanese_layout(paragraph, &texts) {
let policy = paragraph.layout_policy();
if uses_japanese_layout(paragraph, &texts)
|| matches!(
policy,
TextLayoutPolicy::Chinese
| TextLayoutPolicy::KoreanSyllable
| TextLayoutPolicy::KoreanSpacesOnly
)
{
let ruby_breaks: Vec<Option<Vec<usize>>> = paragraph
.children()
.iter()
.map(|span| span.has_ruby().then(Vec::new))
.collect();
if let Some((shifted, map)) =
kinsoku::apply_to_span_texts_with_ruby_breaks(&texts, &ruby_breaks)
kinsoku::apply_policy_to_span_texts(&texts, &ruby_breaks, policy)
{
return (shifted, map);
}
@ -78,6 +88,9 @@ pub(crate) fn horizontal_aki_sheds(
paragraph: &Paragraph,
texts: &[String],
) -> Vec<Vec<(usize, f32)>> {
if paragraph.layout_policy() != TextLayoutPolicy::Japanese {
return vec![Vec::new(); texts.len()];
}
let spans = paragraph.children();
let half_em = |span: usize| spans.get(span).map_or(0.0, |span| span.font_size * 0.5);
// (span, char index in the span text) of every real character.

3
render-wasm/src/shapes/text_horizontal/ruby.rs
View File

@ -108,6 +108,7 @@ pub(super) fn horizontal_ruby_spacing_with(
false,
span.font_features,
&fallback_mgr,
&span.language,
)
.iter()
.flat_map(|run| run.advances.iter().copied())
@ -151,6 +152,7 @@ pub(super) fn horizontal_ruby_spacing_with(
false,
span.font_features,
&fallback_mgr,
&span.language,
);
let (_, pieces, _) =
horizontal_ruby_pieces(&ruby_glyph_refs(span.ruby_text(), &shaped, ruby_font_size));
@ -392,6 +394,7 @@ pub(crate) fn paint_horizontal_ruby(
false,
span.font_features,
&fallback_mgr,
&span.language,
);
let glyphs = ruby_glyph_refs(span.ruby_text(), &shaped, span.ruby_font_size());
let extents: Vec<f32> = rects.iter().map(|rect| rect.rect.width()).collect();

118
render-wasm/src/shapes/text_horizontal/tests.rs
View File

@ -7,6 +7,116 @@ use super::*;
use crate::shapes::{TextAlign, TextDirection, TextTransform};
use crate::Uuid;
#[test]
fn untagged_cjk_keeps_plain_layout() {
init_state();
for text in ["中文，。", "日本 hello world", "한국 漢字"] {
let paragraph = Paragraph::new(
TextAlign::Left,
TextDirection::LTR,
None,
None,
1.0,
0.0,
vec![make_span(text, 0.0)],
);
let (texts, offsets) = paragraph.layout_span_texts();
assert_eq!(texts, vec![text.to_string()]);
assert!(offsets.is_empty());
assert!(horizontal_aki_sheds(&paragraph, &texts)[0].is_empty());
}
}
#[test]
fn explicit_profiles_use_distinct_rules_across_styles() {
init_state();
let mut paragraph = make_paragraph(
vec![make_span("中A 文「", 0.0), make_span("字」っ", 0.0)],
0.0,
);
paragraph.set_language("zh-Hans".to_string());
let (texts, map) = paragraph.layout_span_texts();
assert_eq!(texts.concat(), "中A 文「\u{2060}字\u{2060}」っ");
for offset in 0..="中A 文「字」っ".encode_utf16().count() {
assert_eq!(map.to_original(map.to_shifted(offset)), offset);
}
let plan = HorizontalParagraphPlan::new(&paragraph);
assert!(plan.sheds.iter().all(Vec::is_empty));
let mut paragraph =
make_paragraph(vec![make_span("한", 0.0), make_span("국어 문장", 0.0)], 0.0);
paragraph.set_language("ko".to_string());
assert_eq!(paragraph.layout_span_texts().0.concat(), "한국어 문장");
paragraph.set_korean_spaces_only(true);
assert_eq!(
paragraph.layout_span_texts().0.concat(),
"한\u{2060}국\u{2060}어 문\u{2060}장"
);
}
#[test]
fn korean_joiners_stay_outside_decomposed_syllables_and_emoji() {
init_state();
let mut paragraph = make_paragraph(vec![make_span("ᄒ", 0.0), make_span("ᅡᆫ국 👩‍💻", 0.0)], 0.0);
paragraph.set_language("ko".to_string());
paragraph.set_korean_spaces_only(true);
assert_eq!(paragraph.layout_span_texts().0.concat(), "한\u{2060}국 👩‍💻");
}
#[test]
fn same_paragraph_wraps_differently_by_profile_in_both_orientations() {
use crate::shapes::text_vertical::test_support::{
content_of, layout_with_height, provider, EM,
};
use crate::shapes::GrowType;
use std::collections::HashSet;
init_state();
let font_provider = provider(include_bytes!("../../fonts/cjk-locl-test.ttf"));
let mut fonts = skia::textlayout::FontCollection::new();
fonts.set_asset_font_manager(Some(font_provider.clone().into()));
let mut horizontal_ends = Vec::new();
let mut vertical_ends = Vec::new();
for language in ["ja", "zh-Hans", "ko"] {
let mut span = make_span("文文 あっ文文", 0.0);
span.font_size = EM;
let mut paragraph = make_paragraph(vec![span], 0.0);
paragraph.set_language(language.to_string());
paragraph.set_korean_spaces_only(true);
let plan = HorizontalParagraphPlan::new(&paragraph);
let content = content_of(vec![paragraph.clone()], EM * 5.45, GrowType::Fixed);
let span = &paragraph.children()[0];
let style = span.to_style(&content.bounds(), &HashSet::new(), false, 1.0);
let mut builder = ParagraphBuilder::new(&paragraph.paragraph_to_style(), &fonts);
builder.push_style(&style);
add_horizontal_span(
&mut builder,
span,
&plan.texts[0],
&plan.sheds[0],
&[],
&style,
&fonts,
);
let mut horizontal = builder.build();
horizontal.layout(EM * 5.45);
let (_, offset_map) = paragraph.layout_span_texts();
horizontal_ends.push(offset_map.to_original(horizontal.get_line_metrics()[0].end_index));
let vertical = layout_with_height(&font_provider, &content, EM * 5.45);
vertical_ends.push(
vertical
.cells
.iter()
.filter(|cell| cell.column == 0)
.map(|cell| cell.end)
.max()
.unwrap(),
);
}
assert_eq!(horizontal_ends, vec![6, 5, 3]);
assert_eq!(vertical_ends, vec![6, 5, 3]);
}
#[test]
fn horizontal_ruby_ranges_use_utf16_across_spans() {
init_state();
@ -53,7 +163,7 @@ fn make_span(text: &str, letter_spacing: f32) -> TextSpan {
}
fn make_paragraph(spans: Vec<TextSpan>, letter_spacing: f32) -> Paragraph {
Paragraph::new(
let mut paragraph = Paragraph::new(
TextAlign::default(),
TextDirection::LTR,
None,
@ -61,7 +171,9 @@ fn make_paragraph(spans: Vec<TextSpan>, letter_spacing: f32) -> Paragraph {
1.0,
letter_spacing,
spans,
)
);
paragraph.set_language("ja".to_string());
paragraph
}
#[test]
@ -769,6 +881,7 @@ fn horizontal_ruby_glyphs_share_one_baseline() {
false,
FontFeatures::None,
&FontMgr::from(font_provider.clone()),
"ja",
);
let glyphs: Vec<RubyGlyphRef> = shaped
.iter()
@ -826,6 +939,7 @@ fn horizontal_western_ruby_keeps_its_word_whole() {
false,
FontFeatures::None,
&FontMgr::from(font_provider.clone()),
"ja",
);
let glyphs = ruby_glyph_refs("editor", &shaped, EM / 2.0);
assert!(glyphs.iter().all(|(_, _, _, western)| *western));

85
render-wasm/src/shapes/text_policy.rs Normal file
View File

@ -0,0 +1,85 @@
//! Paragraph typography policies. Geometry and source offsets are shared;
//! Japanese spacing remains in the Japanese layout passes.
use super::japanese::break_allowed_between;
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub enum TextLayoutPolicy {
#[default]
Plain,
Japanese,
Chinese,
KoreanSyllable,
/// POC only: suppress every non-space boundary, including punctuation.
KoreanSpacesOnly,
}
impl TextLayoutPolicy {
pub fn for_language(language: &str, korean_spaces_only: bool) -> Self {
let primary = language.split('-').next().unwrap_or_default();
if primary.eq_ignore_ascii_case("ja") {
Self::Japanese
} else if primary.eq_ignore_ascii_case("zh") {
Self::Chinese
} else if primary.eq_ignore_ascii_case("ko") {
if korean_spaces_only {
Self::KoreanSpacesOnly
} else {
Self::KoreanSyllable
}
} else {
Self::Plain
}
}
/// Legal boundary between graphemes, before fitting a line or column.
pub fn break_allowed(self, before: char, after: char) -> bool {
match self {
Self::Japanese => break_allowed_between(before, after),
Self::KoreanSpacesOnly => before.is_whitespace() || after.is_whitespace(),
Self::Chinese | Self::KoreanSyllable => {
!"（［｛〈《「『【〔〖〘〚‘“([{".contains(before)
&& !"，。、．！？：；）］｝〉》」』】〕〗〙〛’”)]},.!?:;".contains(after)
&& !(before == after && matches!(after, '…' | '—'))
&& !(before.is_ascii_alphanumeric() && after.is_ascii_alphanumeric())
}
Self::Plain => true,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn language_selects_one_policy_without_script_detection() {
for (language, expected) in [
("", TextLayoutPolicy::Plain),
("en", TextLayoutPolicy::Plain),
("ja-JP", TextLayoutPolicy::Japanese),
("ZH-Hant-TW", TextLayoutPolicy::Chinese),
("ko", TextLayoutPolicy::KoreanSyllable),
] {
assert_eq!(TextLayoutPolicy::for_language(language, false), expected);
}
assert_eq!(
TextLayoutPolicy::for_language("ko-KR", true),
TextLayoutPolicy::KoreanSpacesOnly
);
assert_eq!(
TextLayoutPolicy::for_language("zh", true),
TextLayoutPolicy::Chinese
);
}
#[test]
fn chinese_basic_restrictions_do_not_include_small_kana() {
let policy = TextLayoutPolicy::Chinese;
assert!(!policy.break_allowed('「', '中'));
assert!(!policy.break_allowed('文', '」'));
assert!(policy.break_allowed('中', '文'));
assert!(policy.break_allowed('あ', 'っ'));
assert!(!TextLayoutPolicy::Japanese.break_allowed('あ', 'っ'));
}
}

2
render-wasm/src/shapes/text_vertical/annotations.rs
View File

@ -161,6 +161,7 @@ fn shape_ruby(span: &TextSpan, fonts: &Fonts) -> Vec<(ShapedRun, bool)> {
segment.upright,
span.font_features,
fonts.fallback_mgr,
&span.language,
);
for mut run in shaped {
for cluster in &mut run.clusters {
@ -625,6 +626,7 @@ pub(super) fn layout_emphasis(
true,
FontFeatures::None,
fonts.fallback_mgr.clone(),
&span.language,
);
if shaped.first().is_none_or(|run| run.glyphs.is_empty()) {
continue;

21
render-wasm/src/shapes/text_vertical/cells.rs
View File

@ -3,6 +3,7 @@
// composite. Warichu notes span several spans and build their own cells.
use std::ops::Range;
use unicode_segmentation::UnicodeSegmentation;
use skia_safe::{textlayout::TypefaceFontProvider, FontMgr, Typeface};
@ -180,6 +181,7 @@ impl<'a> SpanCells<'a> {
upright,
self.span.font_features,
self.fonts.fallback_mgr,
&self.span.language,
)
}
@ -233,6 +235,19 @@ impl<'a> SpanCells<'a> {
for mut run in self.shape(&segment.text, self.span.font_size, segment.upright) {
let run_index = runs.len();
if segment.upright {
// A font may leave conjoining Jamo as separate glyph
// clusters. Group them by Unicode grapheme boundaries.
let boundaries: Vec<usize> = segment
.text
.grapheme_indices(true)
.map(|(at, _)| at)
.collect();
for cluster in &mut run.clusters {
let at = boundaries
.partition_point(|at| *at <= *cluster as usize)
.saturating_sub(1);
*cluster = boundaries[at].max(run.utf8_range.start) as u32;
}
self.push_upright_clusters(&segment, piece_base, &run, run_index, cells);
} else {
cells.push(self.rotated_cell(&segment, piece_base, &mut run, run_index));
@ -426,7 +441,11 @@ impl<'a> SpanCells<'a> {
run_index: usize,
) -> FlowCell {
let letter_spacing = self.span.letter_spacing;
run.normalize_word_spaces(&segment.text, self.span.font_size);
if crate::shapes::text_policy::TextLayoutPolicy::for_language(&self.span.language, false)
== crate::shapes::text_policy::TextLayoutPolicy::Japanese
{
run.normalize_word_spaces(&segment.text, self.span.font_size);
}
let extent = run.spread_glyphs(letter_spacing);
let run_text = &segment.text[run.utf8_range.clone()];
let script = FlowScript::Rotated {

68
render-wasm/src/shapes/text_vertical/flow.rs
View File

@ -9,7 +9,9 @@ use crate::shapes::japanese::{
};
use crate::shapes::kinsoku::{forbidden_at_line_end, forbidden_at_line_start};
use crate::shapes::text_japanese::capped_equal_allocations;
use crate::shapes::text_policy::TextLayoutPolicy;
use crate::shapes::{LineAdjustment, TextAlign};
use unicode_segmentation::UnicodeSegmentation;
use super::layout::{CellKind, VerticalCell};
@ -273,6 +275,72 @@ pub(super) fn plan_columns(items: &[FlowItem], max_height: f32) -> Vec<(usize, f
placements
}
/// Basic non-Japanese flow. Boundaries use the entire paragraph, so font
/// and style runs cannot split a grapheme or create a word boundary.
pub(super) fn plan_policy_columns(
cells: &[FlowCell],
text: &str,
policy: TextLayoutPolicy,
max_height: f32,
) -> Vec<(usize, f32)> {
let mut boundaries = Vec::new();
let mut offset = 0;
let mut previous = None;
for grapheme in text.graphemes(true) {
let first = grapheme.chars().next().unwrap();
let allowed = previous.is_none_or(|before| policy.break_allowed(before, first));
boundaries.push((offset, allowed));
previous = Some(first);
offset += grapheme.encode_utf16().count();
}
boundaries.push((offset, true));
let mut atomic_start = Vec::with_capacity(cells.len());
let mut unit_start = Vec::with_capacity(cells.len());
let mut prefix = vec![0.0];
for (i, flow) in cells.iter().enumerate() {
let boundary = boundaries
.binary_search_by_key(&flow.cell.start, |(at, _)| *at)
.ok();
let atomic =
i > 0 && (flow.keep_with_previous || flow.shares_previous_box || boundary.is_none());
atomic_start.push(if atomic { atomic_start[i - 1] } else { i });
let joined = atomic || boundary.is_some_and(|at| !boundaries[at].1);
unit_start.push(if i > 0 && joined {
unit_start[i - 1]
} else {
i
});
prefix.push(prefix[i] + flow.cell.extent);
}
let mut placements = Vec::with_capacity(cells.len());
let (mut column, mut column_start, mut cursor) = (0, 0, 0.0);
for (i, flow) in cells.iter().enumerate() {
if cursor > 0.0 && cursor + flow.cell.extent > max_height + FIT_TOLERANCE {
let candidate = unit_start[i];
let break_at = if candidate > column_start
&& prefix[i + 1] - prefix[candidate] <= max_height + FIT_TOLERANCE
{
candidate
} else {
atomic_start[i]
};
// An overlong unit may break, but a grapheme or annotation may not.
if break_at > column_start {
column += 1;
cursor = 0.0;
for k in break_at..i {
placements[k] = (column, cursor);
cursor += cells[k].cell.extent;
}
column_start = break_at;
}
}
placements.push((column, cursor));
cursor += flow.cell.extent;
}
placements
}
/// True when `max_height` is a real wrap budget (auto-width columns are
/// unbounded).
pub(super) fn is_bounded(max_height: f32) -> bool {

108
render-wasm/src/shapes/text_vertical/layout.rs
View File

@ -10,6 +10,7 @@ use crate::globals::get_resources;
use crate::math::Rect;
use crate::shapes::japanese::JapaneseClass;
use crate::shapes::text_japanese::EMPHASIS_FONT_SCALE;
use crate::shapes::text_policy::TextLayoutPolicy;
use crate::shapes::{
merge_fills, AppliedTextTransform, FontFeatures, GrowType, LineAdjustment, Paragraph, RubySide,
TextAlign, TextContent, TextSpan, VerticalAlign,
@ -25,7 +26,8 @@ use super::cells::{Fonts, SpanCells, WarichuNote};
use super::flow::{
aki_classes, align_offset_along_column, apply_inter_script_spacing, apply_ordered_oikomi,
flow_classes, is_bounded, materialize_explicit_pair_spacing, ordered_expansion_offsets,
plan_with_edge_trimming, preferred_pair_spacing, shed_punctuation_aki, FlowCell, FIT_TOLERANCE,
plan_policy_columns, plan_with_edge_trimming, preferred_pair_spacing, shed_punctuation_aki,
FlowCell, FIT_TOLERANCE,
};
use super::paint::LayoutDraws;
use super::shaping::ShapedRun;
@ -516,6 +518,7 @@ fn aligned_tops(
/// A paragraph's flow after spacing and column planning.
struct PlannedParagraph {
policy: TextLayoutPolicy,
flow: Vec<FlowCell>,
ruby_units: Vec<RubyBaseUnit>,
classes: Vec<Option<JapaneseClass>>,
@ -554,6 +557,26 @@ impl PlannedParagraph {
let mut flow = cells.flow(&warichu_splits, runs);
let mut ruby_units = ruby_units.clone();
let policy = cells.paragraph.layout_policy();
if policy != TextLayoutPolicy::Japanese {
grow_ruby_bases(&mut flow, &ruby_units);
let text: String = cells
.text_map
.span_transforms
.iter()
.map(|t| t.text.as_str())
.collect();
let placements = plan_policy_columns(&flow, &text, policy, max_height);
return Self {
policy,
classes: vec![None; flow.len()],
pair_spacing_em: vec![0.0; flow.len().saturating_sub(1)],
flow,
ruby_units,
placements,
};
}
apply_inter_script_spacing(&mut flow);
let classes = flow_classes(&flow, &ruby_spans);
let aki = aki_classes(&flow, &classes, &vpal_spans);
@ -586,6 +609,7 @@ impl PlannedParagraph {
Some(split) => warichu_splits.push(split),
None => {
return Self {
policy,
flow,
ruby_units,
classes,
@ -604,6 +628,12 @@ impl PlannedParagraph {
/// Final cells, aligned along their columns, numbered from `column_base`.
/// A later span's cell of a warichu piece takes the piece's box.
fn place(&self, align: TextAlign, max_height: f32, column_base: usize) -> Vec<VerticalCell> {
// The POC defines no non-Japanese justification expansion table yet.
let align = if self.policy != TextLayoutPolicy::Japanese && align == TextAlign::Justify {
TextAlign::Left
} else {
align
};
let tops = aligned_tops(
&self.flow,
&self.classes,
@ -777,6 +807,82 @@ mod tests {
use super::*;
use crate::shapes::TextOrientation;
#[test]
fn explicit_profiles_change_wrapping_without_japanese_spacing() {
let columns = |language: &str, spaces_only: bool, texts: &[&str]| {
let mut paragraph = vertical_paragraph(
texts.iter().map(|text| make_span(text)).collect(),
TextAlign::Left,
1.0,
);
paragraph.set_language(language.to_string());
paragraph.set_korean_spaces_only(spaces_only);
let content = content_of(vec![paragraph], EM * 2.1, GrowType::Fixed);
let layout = layout_with_height(
&provider(include_bytes!("../../fonts/cjk-locl-test.ttf")),
&content,
EM * 2.1,
);
(
layout
.cells
.iter()
.map(|cell| (cell.start, cell.column))
.collect::<Vec<_>>(),
layout,
)
};
// Small kana stay attached in Japanese only; Chinese uses its own
// basic punctuation policy even when the fixture contains kana.
let (ja, _) = columns("ja", false, &["文あっ文"]);
let (zh, _) = columns("zh-Hans", false, &["文あっ文"]);
assert_ne!(ja, zh);
let (syllables, _) = columns("ko", false, &["한", "국어 문장"]);
let (words, layout) = columns("ko", true, &["한", "국어 문장"]);
// A three-syllable word cannot fit here: safe emergency wrapping.
assert!(words.iter().any(|(_, column)| *column > 0));
assert_eq!(syllables.len(), words.len());
assert!(layout
.cells
.iter()
.all(|cell| cell.minimum_oikomi_extent <= cell.extent));
}
#[test]
fn korean_word_boundaries_cross_styles_and_graphemes_stay_together() {
let mut paragraph = vertical_paragraph(
vec![make_span("한글 "), make_span("한"), make_span("국어")],
TextAlign::Left,
1.0,
);
paragraph.set_language("ko".to_string());
paragraph.set_korean_spaces_only(true);
let content = content_of(vec![paragraph], EM * 3.6, GrowType::Fixed);
let layout = layout_with_height(
&provider(include_bytes!("../../fonts/cjk-locl-test.ttf")),
&content,
EM * 3.6,
);
let word: Vec<_> = layout.cells.iter().filter(|cell| cell.start >= 3).collect();
assert_eq!(word.len(), 3);
assert!(word.iter().all(|cell| cell.column == word[0].column));
assert!(word[0].column > 0);
let mut paragraph =
vertical_paragraph(vec![make_span("ᄒ"), make_span("ᅡᆫ국")], TextAlign::Left, 1.0);
paragraph.set_language("ko".to_string());
let content = content_of(vec![paragraph], EM, GrowType::Fixed);
let layout = layout_with_height(
&provider(include_bytes!("../../fonts/cjk-locl-test.ttf")),
&content,
EM,
);
let syllable: Vec<_> = layout.cells.iter().filter(|cell| cell.start < 3).collect();
assert!(syllable
.iter()
.all(|cell| cell.column == syllable[0].column));
}
fn with_headless_fonts<T>(test: impl FnOnce() -> T) -> T {
let mut resources =
crate::render::RenderResources::try_new_headless().expect("headless resources");

15
render-wasm/src/shapes/text_vertical/orientation.rs
View File

@ -1,4 +1,5 @@
use crate::shapes::TextOrientation;
use unicode_segmentation::UnicodeSegmentation;
/// Emoji ranges recognized by the frontend font-loader. Emoji stay upright
/// in vertical flow; Latin text rotates under `text-orientation: mixed`.
@ -18,6 +19,11 @@ pub(crate) fn is_upright_char(c: char) -> bool {
is_emoji_char(c)
|| matches!(u32::from(c),
0x2E80..=0x2FDF // CJK radicals, Kangxi radicals
| 0x1100..=0x11FF // conjoining Hangul Jamo
| 0x3100..=0x318F // Bopomofo and compatibility Jamo
| 0x31A0..=0x31BF // Bopomofo extended
| 0xA960..=0xA97F // Hangul Jamo extended A
| 0xD7B0..=0xD7FF // Hangul Jamo extended B
| 0x3000..=0x303F // CJK symbols and punctuation
| 0x3040..=0x30FF // hiragana, katakana
| 0x31C0..=0x31EF // CJK strokes
@ -78,7 +84,8 @@ pub(super) struct Segment {
pub(super) fn segment_by_orientation(text: &str, orientation: TextOrientation) -> Vec<Segment> {
let mut segments: Vec<Segment> = Vec::new();
let mut utf16_offset = 0;
for c in text.chars() {
for grapheme in text.graphemes(true) {
let c = grapheme.chars().next().unwrap();
let upright = orientation == TextOrientation::Upright || is_upright_char(c);
let emoji = is_emoji_char(c);
// CJK and emoji are both upright, but split them so the segment
@ -90,15 +97,15 @@ pub(super) fn segment_by_orientation(text: &str, orientation: TextOrientation) -
if last.upright == upright
&& last.text.chars().next_back().is_some_and(is_emoji_char) == emoji =>
{
last.text.push(c)
last.text.push_str(grapheme)
}
_ => segments.push(Segment {
text: c.to_string(),
text: grapheme.to_string(),
utf16_start: utf16_offset,
upright,
}),
}
utf16_offset += c.len_utf16();
utf16_offset += grapheme.encode_utf16().count();
}
segments
}

40
render-wasm/src/shapes/text_vertical/shaping.rs
View File

@ -329,11 +329,12 @@ pub(super) fn shape_segment(
upright: bool,
font_features: FontFeatures,
fallback: FontMgr,
language: &str,
) -> Vec<ShapedRun> {
let mut font_iter = skia::Shaper::new_font_mgr_run_iterator(text, font, Some(fallback.clone()));
let mut bidi_iter = skia::shapers::primitive::trivial_bidi_run_iterator(0, text.len());
let mut script_iter = skia::Shaper::new_hb_icu_script_run_iterator(text);
let mut lang_iter = skia::Shaper::new_trivial_language_run_iterator("ja", text.len());
let mut lang_iter = skia::Shaper::new_trivial_language_run_iterator(language, text.len());
let mut features = if upright {
vec![feature(b"vert"), feature(b"vrt2")]
@ -421,6 +422,7 @@ fn typeface_chunks(text: &str, typefaces: &[Typeface]) -> Vec<(Range<usize>, usi
/// Shape a segment with explicit per-character font fallback.
/// `TypefaceFontProvider` resolves named families, but some Skia builds lack
/// its character fallback hook, so glyph coverage is resolved here.
#[allow(clippy::too_many_arguments)]
pub(crate) fn shape_segment_with_fallbacks(
text: &str,
font_size: f32,
@ -429,6 +431,7 @@ pub(crate) fn shape_segment_with_fallbacks(
upright: bool,
font_features: FontFeatures,
fallback_mgr: &FontMgr,
language: &str,
) -> Vec<ShapedRun> {
shape_with_typefaces(
text,
@ -437,6 +440,7 @@ pub(crate) fn shape_segment_with_fallbacks(
upright,
font_features,
fallback_mgr,
language,
)
}
@ -459,6 +463,7 @@ pub(crate) fn shape_with_typefaces(
upright: bool,
font_features: FontFeatures,
fallback_mgr: &FontMgr,
language: &str,
) -> Vec<ShapedRun> {
if typefaces.is_empty() {
return Vec::new();
@ -473,6 +478,7 @@ pub(crate) fn shape_with_typefaces(
upright,
font_features,
fallback_mgr.clone(),
language,
);
for run in &mut shaped {
run.utf8_range =
@ -544,6 +550,38 @@ mod tests {
use super::super::test_support::*;
use super::*;
#[test]
fn selected_language_reaches_horizontal_and_vertical_localized_glyphs() {
use crate::shapes::{GrowType, TextAlign};
use skia_safe::textlayout::{FontCollection, ParagraphBuilder};
use std::collections::HashSet;
let font_provider = provider(include_bytes!("../../fonts/cjk-locl-test.ttf"));
let mut fonts = FontCollection::new();
fonts.set_asset_font_manager(Some(font_provider.clone().into()));
for (language, glyph, width) in [("ja", 2, 12.0), ("zh-Hans", 3, 14.0), ("ko", 4, 16.0)] {
let mut paragraph = vertical_paragraph(vec![make_span("中")], TextAlign::Left, 1.0);
paragraph.set_language(language.to_string());
let content = content_of(vec![paragraph.clone()], 100.0, GrowType::Fixed);
let layout = layout_with(&font_provider, &content);
assert_eq!(layout.runs[0].glyphs, vec![glyph], "vertical {language}");
let span = &paragraph.children()[0];
let style = span.to_style(&content.bounds(), &HashSet::new(), false, 1.0);
assert_eq!(style.locale(), language);
let mut builder = ParagraphBuilder::new(&paragraph.paragraph_to_style(), &fonts);
builder.push_style(&style);
builder.add_text(&span.text);
let mut horizontal = builder.build();
horizontal.layout(100.0);
assert!(
(horizontal.longest_line() - width).abs() < 0.01,
"horizontal {language}: {}",
horizontal.longest_line()
);
}
}
#[test]
fn span_families_try_emoji_after_the_span_font() {
let families = span_font_families(&make_span("😆"), &["fallback".to_string()]);

5
render-wasm/src/shapes/text_vertical/test_support.rs
View File

@ -70,11 +70,12 @@ pub(super) fn vertical_paragraph(
spans,
);
paragraph.set_writing_mode(WritingMode::VerticalRl);
paragraph.set_language("ja".to_string());
paragraph
}
/// A 200px wide box `height` tall holding `paragraphs`.
pub(super) fn content_of(
pub(crate) fn content_of(
paragraphs: Vec<Paragraph>,
height: f32,
grow_type: GrowType,
@ -167,7 +168,7 @@ pub(super) fn layout_with_fallback(
)
}
pub(super) fn layout_with_height(
pub(crate) fn layout_with_height(
provider: &TypefaceFontProvider,
content: &TextContent,
max_height: f32,

34
render-wasm/src/wasm/text.rs
View File

@ -108,10 +108,12 @@ pub struct RawParagraphData {
text_transform: RawTextTransform,
writing_mode: u8,
text_orientation: u8,
korean_wrap: u8,
// Padding for the CLJS writer's 4-byte-aligned layout; always zero.
_padding: [u8; 2],
_padding: [u8; 1],
line_height: f32,
letter_spacing: f32,
language_length: u32,
}
impl From<[u8; RAW_PARAGRAPH_DATA_SIZE]> for RawParagraphData {
@ -297,6 +299,12 @@ impl From<RawParagraph> for shapes::Paragraph {
paragraph.set_writing_mode(RawWritingMode::from(value.attrs.writing_mode).into());
paragraph
.set_text_orientation(RawTextOrientation::from(value.attrs.text_orientation).into());
let language = ruby_offset
.checked_add(value.attrs.language_length as usize)
.and_then(|end| value.text_buffer.get(ruby_offset..end))
.unwrap_or(&[]);
paragraph.set_language(String::from_utf8_lossy(language).into_owned());
paragraph.set_korean_spaces_only(value.attrs.korean_wrap == 1);
paragraph
}
}
@ -507,13 +515,13 @@ mod tests {
AnnotationClearance, FontFeatures, RubyAlign, RubyOverhang, RubySide, RubySize,
};
/// The CLJS writer (texts.cljs) writes PARAGRAPH-ATTR-U8-SIZE (16)
/// The shared CLJS writer writes PARAGRAPH-ATTR-U8-SIZE (20)
/// attr bytes after the u32 span count, and SPAN-ATTR-U8-SIZE (80)
/// attr bytes before the fills block. These sizes must match the struct
/// layouts.
#[test]
fn raw_struct_sizes_match_cljs_writer() {
const PARAGRAPH_ATTR_U8_SIZE: usize = 16;
const PARAGRAPH_ATTR_U8_SIZE: usize = 20;
const SPAN_ATTR_U8_SIZE: usize = 80;
assert_eq!(RAW_PARAGRAPH_DATA_SIZE, 4 + PARAGRAPH_ATTR_U8_SIZE);
assert_eq!(
@ -522,6 +530,26 @@ mod tests {
);
}
#[test]
fn language_decodes_after_base_and_ruby_text() {
let mut attrs = [0u8; RAW_PARAGRAPH_DATA_SIZE];
attrs[10] = 1; // Korean spaces-only POC; byte 6 after span count.
attrs[20..24].copy_from_slice(&2u32.to_le_bytes());
let mut span = [0u8; RAW_SPAN_DATA_SIZE];
span[68..72].copy_from_slice(&3u32.to_le_bytes());
span[72..76].copy_from_slice(&3u32.to_le_bytes());
let paragraph = shapes::Paragraph::from(RawParagraph {
attrs: RawParagraphData::from(attrs),
spans: vec![RawTextSpan::from(span)],
text_buffer: "韓한ko".as_bytes().to_vec(),
});
assert_eq!(paragraph.language(), "ko");
assert!(paragraph.korean_spaces_only());
assert_eq!(paragraph.children()[0].text, "韓");
assert_eq!(paragraph.children()[0].ruby, "한");
assert_eq!(paragraph.children()[0].language, "ko");
}
#[test]
fn raw_text_combine_upright_counts_deserialize() {
// Byte 5 of the span attr block; the counted digits variants map

73
render-wasm/src/wasm/text/helpers.rs
View File

@ -477,11 +477,8 @@ pub fn delete_selection_range(text_content: &mut TextContent, selection: &TextSe
delete_range_in_paragraph(&mut paragraphs[end_paragraph], 0, end.offset);
let end_para_children: Vec<_> =
paragraphs[end_paragraph].children_mut().drain(..).collect();
paragraphs[start.paragraph]
.children_mut()
.extend(end_para_children);
let end_para_children = std::mem::take(paragraphs[end_paragraph].children_mut());
paragraphs[start.paragraph].extend_children(end_para_children);
paragraphs.drain((start.paragraph + 1)..=end_paragraph);
@ -593,13 +590,8 @@ pub fn delete_char_before(
let paragraphs = text_content.paragraphs_mut();
let prev_para_len = paragraph_char_count(&paragraphs[prev_para_idx]);
let current_children: Vec<_> = paragraphs[cursor.paragraph]
.children_mut()
.drain(..)
.collect();
paragraphs[prev_para_idx]
.children_mut()
.extend(current_children);
let current_children = std::mem::take(paragraphs[cursor.paragraph].children_mut());
paragraphs[prev_para_idx].extend_children(current_children);
paragraphs.remove(cursor.paragraph);
@ -770,10 +762,8 @@ pub fn delete_char_after(text_content: &mut TextContent, cursor: &TextPositionWi
delete_range_in_paragraph(para, cursor.offset, end);
} else if cursor.paragraph < paragraphs.len() - 1 {
let next_para_idx = cursor.paragraph + 1;
let next_children: Vec<_> = paragraphs[next_para_idx].children_mut().drain(..).collect();
paragraphs[cursor.paragraph]
.children_mut()
.extend(next_children);
let next_children = std::mem::take(paragraphs[next_para_idx].children_mut());
paragraphs[cursor.paragraph].extend_children(next_children);
paragraphs.remove(next_para_idx);
}
@ -825,6 +815,9 @@ pub fn split_paragraph_at_cursor(
let text_transform = para.text_transform();
let writing_mode = para.writing_mode();
let text_orientation = para.text_orientation();
let language = para.language().to_owned();
let korean_spaces_only = para.korean_spaces_only();
let source_position = para.source_position();
let line_height = para.line_height();
let letter_spacing = para.letter_spacing();
@ -851,6 +844,9 @@ pub fn split_paragraph_at_cursor(
);
new_para.set_writing_mode(writing_mode);
new_para.set_text_orientation(text_orientation);
new_para.set_language(language);
new_para.set_korean_spaces_only(korean_spaces_only);
new_para.set_source_position(source_position);
paragraphs.insert(cursor.paragraph + 1, new_para);
@ -864,6 +860,51 @@ mod tests {
use crate::shapes::{FontFamily, FontStyle, GrowType, TextAlign, TextSpan};
use crate::uuid::Uuid;
#[test]
fn paragraph_split_keeps_language_and_korean_wrap() {
let mut para = paragraph(&["한국어"]);
para.set_language("ko".to_string());
para.set_korean_spaces_only(true);
let mut content = content(vec![para]);
let cursor = TextPositionWithAffinity {
paragraph: 0,
offset: 1,
..Default::default()
};
assert!(split_paragraph_at_cursor(&mut content, &cursor));
assert_eq!(content.paragraphs().len(), 2);
for para in content.paragraphs() {
assert_eq!(para.language(), "ko");
assert!(para.korean_spaces_only());
assert_eq!(para.source_position(), 0);
assert!(para.children().iter().all(|span| span.language == "ko"));
}
}
#[test]
fn paragraph_merge_inherits_the_surviving_paragraph_language() {
let mut first = paragraph(&["中"]);
first.set_language("ja".to_string());
let mut second = paragraph(&["中"]);
second.set_language("ko".to_string());
let mut content = content(vec![first, second]);
delete_char_after(
&mut content,
&TextPositionWithAffinity {
paragraph: 0,
offset: 1,
..Default::default()
},
);
assert_eq!(content.paragraphs().len(), 1);
let paragraph = &content.paragraphs()[0];
assert_eq!(paragraph.language(), "ja");
assert!(paragraph
.children()
.iter()
.all(|span| span.language == "ja"));
}
fn span(text: &str) -> TextSpan {
TextSpan::new(
text.to_string(),

7
render-wasm/src/wasm/text_editor.rs
View File

@ -984,8 +984,11 @@ pub extern "C" fn text_editor_export_content() -> *mut u8 {
.replace('\r', "\\r")
.replace('\t', "\\t");
span_parts.push(format!(
"{{\"p\":{},\"s\":{},\"t\":\"{}\"}}",
span.paragraph_position, span.span_position, escaped_text
"{{\"pp\":{},\"p\":{},\"s\":{},\"t\":\"{}\"}}",
para.source_position(),
span.paragraph_position,
span.span_position,
escaped_text
));
}
json_parts.push(format!("[{}]", span_parts.join(",")));