From 61059855f0f6ca5b4a8b45245863d1037bd3562c Mon Sep 17 00:00:00 2001 From: Alejandro Alonso Date: Thu, 1 Oct 2026 12:20:20 +0200 Subject: [PATCH] WIP --- backend/src/app/rpc/commands/files.clj | 55 ++ common/src/app/common/flags.cljc | 3 + .../app/common/render_wasm/api/upload.cljs | 448 +-------------- .../common/render_wasm/api/upload_encode.cljc | 530 ++++++++++++++++++ .../app/common/render_wasm/enums_data.cljc | 98 ++++ .../common/render_wasm/serialize_shape.cljs | 11 + .../common/render_wasm/serializers/color.cljc | 21 + .../common/render_wasm/serializers/color.cljs | 11 - .../render_wasm/upload_encode_test.cljc | 156 ++++++ common/test/common_tests/runner.cljc | 2 + frontend/src/app/main/data/workspace.cljs | 14 +- .../src/app/main/data/workspace/pages.cljs | 32 +- frontend/src/app/main/repo.cljs | 1 + frontend/src/app/render_wasm/api.cljs | 436 +++++++++----- 14 files changed, 1234 insertions(+), 584 deletions(-) create mode 100644 common/src/app/common/render_wasm/api/upload_encode.cljc create mode 100644 common/src/app/common/render_wasm/enums_data.cljc create mode 100644 common/src/app/common/render_wasm/serializers/color.cljc delete mode 100644 common/src/app/common/render_wasm/serializers/color.cljs create mode 100644 common/test/common_tests/render_wasm/upload_encode_test.cljc diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index e10dcd4b9e..06faca26dd 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -24,11 +24,13 @@ [app.common.types.file :as ctf] [app.common.types.tokens-lib :as ctob] [app.common.uri :as uri] + [app.common.render-wasm.api.upload-encode :as upload-encode] [app.config :as cf] [app.db :as db] [app.db.sql :as-alias sql] [app.features.fdata :as feat.fdata] [app.features.logical-deletion :as ldel] + [app.http :as-alias http] [app.http.sse :as sse] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] @@ -472,6 +474,59 @@ (check-read-permissions! cfg profile-id file-id share-id) (get-page cfg (assoc params :profile-id profile-id))))) +;; --- COMMAND QUERY: get-file-wasm-shapes-batch (prototype) + +(def schema:get-file-wasm-shapes-batch + [:map {:title "get-file-wasm-shapes-batch"} + [:file-id ::sm/uuid] + [:page-id {:optional true} ::sm/uuid] + [:share-id {:optional true} ::sm/uuid] + [:features {:optional true} ::cfeat/features]]) + +(defn- encode-page-wasm-shapes-batch + "Encode page objects to `_set_shapes_batch` bytes. Prototype helper." + [page] + ;; Prototype: pick up common encoder edits (e.g. UUID LE layout) without a + ;; full JVM restart. Remove once the path is no longer experimental. + (require 'app.common.render-wasm.api.upload-encode :reload) + (let [objects (:objects page) + t0 (System/nanoTime) + bytes (upload-encode/encode-page-objects objects) + elapsed (/ (- (System/nanoTime) t0) 1e6) + n (count objects)] + (l/info :hint "wasm-shapes-batch encoded" + :version upload-encode/PROTOCOL-VERSION + :shapes n + :bytes (alength ^bytes bytes) + :encode-ms (format "%.2f" elapsed)) + {:bytes bytes + :shape-count n + :encode-ms elapsed})) + +(sv/defmethod ::get-file-wasm-shapes-batch + "Prototype: returns a binary `_set_shapes_batch` buffer for a page. + Used to measure cold-load TTFP when the frontend ingests the buffer + instead of encoding shapes in the browser. Auth same as get-page." + {::doc/added "2.12" + ::sm/params schema:get-file-wasm-shapes-batch} + [cfg {:keys [::rpc/profile-id file-id share-id] :as params}] + (db/tx-run! cfg + (fn [cfg] + (check-read-permissions! cfg profile-id file-id share-id) + (let [page (get-page cfg (assoc params :profile-id profile-id)) + file (bfc/get-file cfg file-id :read-only? true) + result (encode-page-wasm-shapes-batch page) + bytes (:bytes result) + revn (:revn file)] + (rph/wrap + (java.io.ByteArrayInputStream. bytes) + {::http/headers + {"content-type" "application/octet-stream" + "x-penpot-wasm-batch-version" upload-encode/PROTOCOL-VERSION + "x-penpot-shape-count" (str (:shape-count result)) + "x-penpot-file-revn" (str revn) + "x-penpot-encode-ms" (format "%.2f" (:encode-ms result))}}))))) + ;; --- COMMAND QUERY: get-team-shared-files (defn- get-components-with-variants diff --git a/common/src/app/common/flags.cljc b/common/src/app/common/flags.cljc index 60353f730f..48ab9a5f29 100644 --- a/common/src/app/common/flags.cljc +++ b/common/src/app/common/flags.cljc @@ -151,6 +151,8 @@ :render-wasm-dpr ;; Show WASM renderer info label (hidden by default). :render-wasm-info + ;; Prototype: ingest server-encoded `_set_shapes_batch` on cold load. + :wasm-shapes-batch :render-switch :hide-release-modal :subscriptions @@ -225,6 +227,7 @@ :enable-token-import-from-library :enable-render-switch :enable-render-wasm-info + :enable-wasm-shapes-batch :enable-available-viewer-wasm :enable-background-blur :enable-stroke-path diff --git a/common/src/app/common/render_wasm/api/upload.cljs b/common/src/app/common/render_wasm/api/upload.cljs index f7a26712bc..07afe175bf 100644 --- a/common/src/app/common/render_wasm/api/upload.cljs +++ b/common/src/app/common/render_wasm/api/upload.cljs @@ -5,427 +5,33 @@ ;; Copyright (c) KALEIDOS SUBSIDIARY SL (ns app.common.render-wasm.api.upload - "Enlarged per-shape + multi-shape structural upload for WASM cold load. + "WASM heap flush for structural cold-load batches. - Writes a binary batch consumed by `_set_shapes_batch`. Remaining - host-specific attrs (image bytes, text, path, grid tracks) are applied - afterwards via the existing per-shape setters." + Pure encode lives in `app.common.render-wasm.api.upload-encode`." (:require [app.common.buffer :as buf] - [app.common.data :as d] - [app.common.data.macros :as dm] + [app.common.render-wasm.api.upload-encode :as encode] [app.common.render-wasm.helpers :as h] [app.common.render-wasm.mem :as mem] - [app.common.render-wasm.serializers :as sr] - [app.common.render-wasm.serializers.color :as sr-clr] - [app.common.render-wasm.wasm :as wasm] - [app.common.types.fills :as types.fills] - [app.common.types.fills.impl :as types.fills.impl] - [app.common.types.shape.layout :as ctl] - [app.common.uuid :as uuid])) + [app.common.render-wasm.wasm :as wasm])) -(def ^:const BASE-PROPS-SIZE 104) -(def ^:const FLAG-CLIP-CONTENT 0x01) -(def ^:const FLAG-HIDDEN 0x02) +(def write-shape-payload! encode/write-shape-payload!) +(def encode-shape-record encode/encode-shape-record) +(def encode-shapes-batch encode/encode-shapes-batch) +(def shapes-in-tree-order encode/shapes-in-tree-order) +(def encode-page-objects encode/encode-page-objects) +(def PROTOCOL-VERSION encode/PROTOCOL-VERSION) -(def ^:const SECTION-CHILDREN 0x01) -(def ^:const SECTION-BLUR-LAYER 0x02) -(def ^:const SECTION-BLUR-BG 0x04) -(def ^:const SECTION-SHADOWS 0x08) -(def ^:const SECTION-MASKED 0x10) -(def ^:const SECTION-BOOL-TYPE 0x20) -(def ^:const SECTION-GROW-TYPE 0x40) -(def ^:const SECTION-LAYOUT-ITEM 0x80) -(def ^:const SECTION-FLEX 0x100) -(def ^:const SECTION-FILLS 0x200) -(def ^:const SECTION-STROKES 0x400) - -;; Stroke header before RawFillData (must match upload_batch.rs). -(def ^:const STROKE-HEADER-U8-SIZE 36) -(def ^:const STROKE-ALIGN-CENTER 0) -(def ^:const STROKE-ALIGN-INNER 1) -(def ^:const STROKE-ALIGN-OUTER 2) - -(defn- write-uuid! - [dview offset id] - (buf/write-uuid dview offset id) - (+ offset 16)) - -(defn- write-base-props! - "Write the 104-byte RawBasePropsData at `offset`. Returns next offset." - [dview offset shape] - (let [id (dm/get-prop shape :id) - parent-id (get shape :parent-id) - shape-type (dm/get-prop shape :type) - clip-content (if (= shape-type :frame) - (not (get shape :show-content)) - false) - hidden (get shape :hidden false) - flags (cond-> 0 - clip-content (bit-or FLAG-CLIP-CONTENT) - hidden (bit-or FLAG-HIDDEN)) - blend-mode (sr/translate-blend-mode (get shape :blend-mode)) - constraint-h (sr/translate-constraint-h (or (get shape :constraints-h) :none)) - constraint-v (sr/translate-constraint-v (or (get shape :constraints-v) :none)) - opacity (d/nilv (get shape :opacity) 1.0) - rotation (d/nilv (get shape :rotation) 0.0) - transform (get shape :transform) - [ta tb tc td te tf] - (if (some? transform) - [(dm/get-prop transform :a) - (dm/get-prop transform :b) - (dm/get-prop transform :c) - (dm/get-prop transform :d) - (dm/get-prop transform :e) - (dm/get-prop transform :f)] - [1.0 0.0 0.0 1.0 0.0 0.0]) - selrect (get shape :selrect) - [sx1 sy1 sx2 sy2] - (if (some? selrect) - [(dm/get-prop selrect :x1) - (dm/get-prop selrect :y1) - (dm/get-prop selrect :x2) - (dm/get-prop selrect :y2)] - [0.0 0.0 0.0 0.0]) - r1 (d/nilv (get shape :r1) 0.0) - r2 (d/nilv (get shape :r2) 0.0) - r3 (d/nilv (get shape :r3) 0.0) - r4 (d/nilv (get shape :r4) 0.0)] - - (write-uuid! dview offset id) - (write-uuid! dview (+ offset 16) (d/nilv parent-id uuid/zero)) - (buf/write-u8 dview (+ offset 32) (sr/translate-shape-type shape-type)) - (buf/write-u8 dview (+ offset 33) flags) - (buf/write-u8 dview (+ offset 34) blend-mode) - (buf/write-u8 dview (+ offset 35) constraint-h) - (buf/write-u8 dview (+ offset 36) constraint-v) - (buf/write-f32 dview (+ offset 40) opacity) - (buf/write-f32 dview (+ offset 44) rotation) - (buf/write-f32 dview (+ offset 48) ta) - (buf/write-f32 dview (+ offset 52) tb) - (buf/write-f32 dview (+ offset 56) tc) - (buf/write-f32 dview (+ offset 60) td) - (buf/write-f32 dview (+ offset 64) te) - (buf/write-f32 dview (+ offset 68) tf) - (buf/write-f32 dview (+ offset 72) sx1) - (buf/write-f32 dview (+ offset 76) sy1) - (buf/write-f32 dview (+ offset 80) sx2) - (buf/write-f32 dview (+ offset 84) sy2) - (buf/write-f32 dview (+ offset 88) r1) - (buf/write-f32 dview (+ offset 92) r2) - (buf/write-f32 dview (+ offset 96) r3) - (buf/write-f32 dview (+ offset 100) r4) - (+ offset BASE-PROPS-SIZE))) - -(defn- write-blur! - [dview offset blur] - (buf/write-u8 dview offset (if (get blur :hidden) 1 0)) - (buf/write-f32 dview (+ offset 4) (get blur :value 0)) - (+ offset 8)) - -(defn- write-shadow! - [dview offset shadow] - (let [color (get shadow :color) - rgba (sr-clr/hex->u32argb (get color :color) - (get color :opacity))] - (buf/write-u32 dview offset rgba) - (buf/write-f32 dview (+ offset 4) (get shadow :blur 0)) - (buf/write-f32 dview (+ offset 8) (get shadow :spread 0)) - (buf/write-f32 dview (+ offset 12) (get shadow :offset-x 0)) - (buf/write-f32 dview (+ offset 16) (get shadow :offset-y 0)) - (buf/write-u8 dview (+ offset 20) (sr/translate-shadow-style (get shadow :style))) - (buf/write-u8 dview (+ offset 21) (if (get shadow :hidden) 1 0)) - (+ offset 24))) - -(defn- write-flex! - [dview offset shape] - (let [dir (-> (get shape :layout-flex-dir :row) - (sr/translate-layout-flex-dir)) - gap (get shape :layout-gap) - row-gap (get gap :row-gap 0) - column-gap (get gap :column-gap 0) - align-items (-> (get shape :layout-align-items) sr/translate-layout-align-items) - align-content (-> (get shape :layout-align-content) sr/translate-layout-align-content) - justify-items (-> (get shape :layout-justify-items) sr/translate-layout-justify-items) - justify-content (-> (get shape :layout-justify-content) sr/translate-layout-justify-content) - wrap-type (-> (get shape :layout-wrap-type) sr/translate-layout-wrap-type) - padding (get shape :layout-padding) - padding-top (get padding :p1 0) - padding-right (get padding :p2 0) - padding-bottom (get padding :p3 0) - padding-left (get padding :p4 0)] - (buf/write-u8 dview offset dir) - (buf/write-u8 dview (+ offset 1) align-items) - (buf/write-u8 dview (+ offset 2) align-content) - (buf/write-u8 dview (+ offset 3) justify-items) - (buf/write-u8 dview (+ offset 4) justify-content) - (buf/write-u8 dview (+ offset 5) wrap-type) - (buf/write-f32 dview (+ offset 8) row-gap) - (buf/write-f32 dview (+ offset 12) column-gap) - (buf/write-f32 dview (+ offset 16) padding-top) - (buf/write-f32 dview (+ offset 20) padding-right) - (buf/write-f32 dview (+ offset 24) padding-bottom) - (buf/write-f32 dview (+ offset 28) padding-left) - (+ offset 32))) - -(defn- write-layout-item! - [dview offset shape] - (let [margins (get shape :layout-item-margin) - margin-top (get margins :m1 0) - margin-right (get margins :m2 0) - margin-bottom (get margins :m3 0) - margin-left (get margins :m4 0) - h-sizing (-> (get shape :layout-item-h-sizing) sr/translate-layout-sizing) - v-sizing (-> (get shape :layout-item-v-sizing) sr/translate-layout-sizing) - align-self (-> (get shape :layout-item-align-self) sr/translate-align-self) - max-h (get shape :layout-item-max-h) - min-h (get shape :layout-item-min-h) - max-w (get shape :layout-item-max-w) - min-w (get shape :layout-item-min-w) - is-absolute (boolean (get shape :layout-item-absolute)) - z-index (get shape :layout-item-z-index) - flags (cond-> 0 - (some? max-h) (bit-or 0x01) - (some? min-h) (bit-or 0x02) - (some? max-w) (bit-or 0x04) - (some? min-w) (bit-or 0x08) - is-absolute (bit-or 0x10))] - (buf/write-f32 dview offset margin-top) - (buf/write-f32 dview (+ offset 4) margin-right) - (buf/write-f32 dview (+ offset 8) margin-bottom) - (buf/write-f32 dview (+ offset 12) margin-left) - (buf/write-u8 dview (+ offset 16) (d/nilv h-sizing 0)) - (buf/write-u8 dview (+ offset 17) (d/nilv v-sizing 0)) - (buf/write-u8 dview (+ offset 18) flags) - (buf/write-u8 dview (+ offset 19) (d/nilv align-self 0)) - (buf/write-f32 dview (+ offset 20) (d/nilv max-h 0)) - (buf/write-f32 dview (+ offset 24) (d/nilv min-h 0)) - (buf/write-f32 dview (+ offset 28) (d/nilv max-w 0)) - (buf/write-f32 dview (+ offset 32) (d/nilv min-w 0)) - (buf/write-i32 dview (+ offset 36) (d/nilv z-index 0)) - (+ offset 40))) - -(defn- write-fills-section! - "Write fills in the same layout as `_set_shape_fills`: - [u8 n][u8;3 pad][n × FILL-U8-SIZE]. Returns next offset." - [dview offset fills] - (let [fills (types.fills/coerce (or fills [])) - byte-size (types.fills/get-byte-size fills) - ;; write-to expects a Uint32Array heap + u32 element offset - heap-u32 (js/Uint32Array. (.-buffer dview)) - u32-off (quot offset 4)] - (types.fills/write-to fills heap-u32 u32-off) - (+ offset byte-size))) - -(defn- write-stroke-fill! - [dview offset stroke] - (let [opacity (or (:stroke-opacity stroke) 1.0) - color (:stroke-color stroke) - gradient (:stroke-color-gradient stroke) - image (:stroke-image stroke)] - (cond - (some? gradient) - (types.fills.impl/write-gradient-fill offset dview opacity gradient) - - (some? image) - (types.fills.impl/write-image-fill offset dview opacity image) - - (some? color) - (types.fills.impl/write-solid-fill offset dview opacity color) - - :else - (types.fills.impl/write-solid-fill offset dview 0.0 "#000000")))) - -(defn- write-stroke! - [dview offset stroke] - (let [width (or (:stroke-width stroke) 1.0) - style (-> stroke :stroke-style sr/translate-stroke-style) - align (case (:stroke-alignment stroke) - :inner STROKE-ALIGN-INNER - :outer STROKE-ALIGN-OUTER - STROKE-ALIGN-CENTER) - cap-start (-> stroke :stroke-cap-start sr/translate-stroke-cap) - cap-end (-> stroke :stroke-cap-end sr/translate-stroke-cap) - dash (or (:stroke-dash stroke) -1) - gap (or (:stroke-gap stroke) -1) - - top (or (:stroke-width-top stroke) width) - right (or (:stroke-width-right stroke) width) - bottom (or (:stroke-width-bottom stroke) width) - left (or (:stroke-width-left stroke) width) - - has-sides? (not= top right bottom left)] - - (buf/write-f32 dview offset width) - (buf/write-u8 dview (+ offset 4) style) - (buf/write-u8 dview (+ offset 5) align) - (buf/write-u8 dview (+ offset 6) (d/nilv cap-start 0)) - (buf/write-u8 dview (+ offset 7) (d/nilv cap-end 0)) - (buf/write-f32 dview (+ offset 8) dash) - (buf/write-f32 dview (+ offset 12) gap) - (buf/write-u8 dview (+ offset 16) (if has-sides? 1 0)) - (buf/write-f32 dview (+ offset 20) top) - (buf/write-f32 dview (+ offset 24) right) - (buf/write-f32 dview (+ offset 28) bottom) - (buf/write-f32 dview (+ offset 32) left) - (write-stroke-fill! dview (+ offset STROKE-HEADER-U8-SIZE) stroke) - (+ offset STROKE-HEADER-U8-SIZE types.fills.impl/FILL-U8-SIZE))) - -(defn- visible-strokes - [shape] - (let [type (dm/get-prop shape :type)] - (if (= type :group) - [] - (into [] (remove :hidden) (or (get shape :strokes) []))))) - -(defn- write-strokes-section! - [dview offset strokes] - (buf/write-u32 dview offset (count strokes)) - (reduce (fn [o s] (write-stroke! dview o s)) - (+ offset 4) - strokes)) - -(defn write-shape-payload! - "Serialize one shape's structural payload into `dview` starting at `offset` - (payload only — no length prefix). Returns the offset after the payload. - - Options: - - `:include-layout?` — when true, emit FLEX + LAYOUT-ITEM (workspace cold load). - - `:include-fills-strokes?` — when true, emit FILLS + STROKES sections." - [dview offset shape {:keys [include-layout? include-fills-strokes?] - :or {include-layout? false - include-fills-strokes? false}}] - (let [shape-type (dm/get-prop shape :type) - children (into [] (filter uuid?) (get shape :shapes)) - blur (get shape :blur) - bg-blur (get shape :background-blur) - shadows (or (get shape :shadow) []) - masked? (and (= shape-type :group) (boolean (get shape :masked-group))) - bool-type (when (= shape-type :bool) (get shape :bool-type)) - grow-type (when (= shape-type :text) (get shape :grow-type)) - flex? (and include-layout? (ctl/flex-layout? shape)) - layout-item? include-layout? - strokes (when include-fills-strokes? (visible-strokes shape)) - - mask (cond-> 0 - true (bit-or SECTION-CHILDREN) - (some? blur) (bit-or SECTION-BLUR-LAYER) - (some? bg-blur) (bit-or SECTION-BLUR-BG) - (seq shadows) (bit-or SECTION-SHADOWS) - (= shape-type :group) (bit-or SECTION-MASKED) - (some? bool-type) (bit-or SECTION-BOOL-TYPE) - (some? grow-type) (bit-or SECTION-GROW-TYPE) - flex? (bit-or SECTION-FLEX) - layout-item? (bit-or SECTION-LAYOUT-ITEM) - include-fills-strokes? (bit-or SECTION-FILLS) - include-fills-strokes? (bit-or SECTION-STROKES)) - - offset (write-base-props! dview offset shape) - _ (buf/write-u32 dview offset mask) - offset (+ offset 4) - - offset (let [o offset] - (buf/write-u32 dview o (count children)) - (reduce (fn [o id] (write-uuid! dview o id)) - (+ o 4) - children)) - - offset (cond-> offset - (some? blur) - (as-> o (write-blur! dview o blur))) - - offset (cond-> offset - (some? bg-blur) - (as-> o (write-blur! dview o bg-blur))) - - offset (cond-> offset - (seq shadows) - (as-> o - (do - (buf/write-u32 dview o (count shadows)) - (reduce (fn [o s] (write-shadow! dview o s)) - (+ o 4) - shadows)))) - - offset (cond-> offset - (= shape-type :group) - (as-> o - (do (buf/write-u8 dview o (if masked? 1 0)) - (+ o 4)))) - - offset (cond-> offset - (some? bool-type) - (as-> o - (do (buf/write-u8 dview o (sr/translate-bool-type bool-type)) - (+ o 4)))) - - offset (cond-> offset - (some? grow-type) - (as-> o - (do (buf/write-u8 dview o (sr/translate-grow-type grow-type)) - (+ o 4)))) - - ;; FLEX before LAYOUT-ITEM (Rust clears layout on flex) - offset (cond-> offset - flex? - (as-> o (write-flex! dview o shape))) - - offset (cond-> offset - layout-item? - (as-> o (write-layout-item! dview o shape))) - - offset (cond-> offset - include-fills-strokes? - (as-> o (write-fills-section! dview o (get shape :fills)))) - - offset (cond-> offset - include-fills-strokes? - (as-> o (write-strokes-section! dview o strokes)))] - offset)) - -(defn- payload-byte-size - [shape {:keys [include-layout? include-fills-strokes?] - :or {include-layout? false include-fills-strokes? false}}] - (let [children (into [] (filter uuid?) (get shape :shapes)) - shadows (or (get shape :shadow) []) - shape-type (dm/get-prop shape :type) - blur (get shape :blur) - bg-blur (get shape :background-blur) - flex? (and include-layout? (ctl/flex-layout? shape)) - fills-size (if include-fills-strokes? - (types.fills/get-byte-size (types.fills/coerce (or (get shape :fills) []))) - 0) - strokes (when include-fills-strokes? (visible-strokes shape)) - strokes-size (if include-fills-strokes? - (+ 4 (* (count strokes) - (+ STROKE-HEADER-U8-SIZE types.fills.impl/FILL-U8-SIZE))) - 0)] - (+ BASE-PROPS-SIZE - 4 ;; mask - (+ 4 (* 16 (count children))) - (if (some? blur) 8 0) - (if (some? bg-blur) 8 0) - (if (seq shadows) (+ 4 (* 24 (count shadows))) 0) - (if (= shape-type :group) 4 0) - (if (and (= shape-type :bool) (some? (get shape :bool-type))) 4 0) - (if (and (= shape-type :text) (some? (get shape :grow-type))) 4 0) - (if flex? 32 0) - (if include-layout? 40 0) - fills-size - strokes-size))) - -(defn- encode-shape-record - "Returns a Uint8Array: [u32 payload_len][payload]." - [shape opts] - (let [capacity (+ 4 (payload-byte-size shape opts)) - buffer (js/ArrayBuffer. capacity) - dview (js/DataView. buffer) - end (write-shape-payload! dview 4 shape opts) - payload-len (- end 4)] - (assert (= end capacity) - (str "upload record size mismatch: wrote " end " expected " capacity)) - (buf/write-u32 dview 0 payload-len) - (js/Uint8Array. buffer 0 end))) +(defn flush-shapes-batch-bytes! + "Upload a pre-encoded `_set_shapes_batch` buffer (Uint8Array)." + [batch-bytes] + (when (and (wasm/live?) batch-bytes) + (let [total (.-byteLength ^js batch-bytes) + offset (mem/alloc total) + heap (mem/get-heap-u8)] + (.set heap batch-bytes offset) + (h/call wasm/internal-module "_set_shapes_batch") + nil))) (defn flush-shapes-batch! "Upload `shapes` as one `_set_shapes_batch` call. @@ -433,19 +39,7 @@ `:include-fills-strokes?`)." [shapes opts] (when (and (wasm/live?) (seq shapes)) - (let [records (mapv #(encode-shape-record % opts) shapes) - total (reduce (fn [acc ^js u8] (+ acc (.-byteLength u8))) 4 records) - offset (mem/alloc total) - heap (mem/get-heap-u8) - dview (js/DataView. (.-buffer heap))] - (buf/write-u32 dview offset (count records)) - (reduce (fn [o ^js u8] - (.set heap u8 o) - (+ o (.-byteLength u8))) - (+ offset 4) - records) - (h/call wasm/internal-module "_set_shapes_batch") - nil))) + (flush-shapes-batch-bytes! (encode/encode-shapes-batch shapes opts)))) (defn set-shape-upload! "Single-shape structural upload (enlarged blob, one FFI)." diff --git a/common/src/app/common/render_wasm/api/upload_encode.cljc b/common/src/app/common/render_wasm/api/upload_encode.cljc new file mode 100644 index 0000000000..0f510ca927 --- /dev/null +++ b/common/src/app/common/render_wasm/api/upload_encode.cljc @@ -0,0 +1,530 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.common.render-wasm.api.upload-encode + "Pure structural encode for `_set_shapes_batch` (JVM + JS). + + No WASM / Emscripten dependency. Browser flush lives in + `app.common.render-wasm.api.upload`." + (:require + [app.common.buffer :as buf] + [app.common.data :as d] + [app.common.data.macros :as dm] + [app.common.files.helpers :as cfh] + [app.common.render-wasm.enums-data :as enums] + [app.common.render-wasm.serializers.color :as sr-clr] + [app.common.types.fills :as types.fills] + [app.common.types.fills.impl :as types.fills.impl] + [app.common.types.shape.layout :as ctl] + [app.common.uuid :as uuid])) + +;; v2: JVM UUID layout fixed to LE u32 quartet (matches CLJS/WASM). +(def ^:const PROTOCOL-VERSION "wasm-shapes-batch/v2") + +(def ^:const BASE-PROPS-SIZE 104) +(def ^:const FLAG-CLIP-CONTENT 0x01) +(def ^:const FLAG-HIDDEN 0x02) + +(def ^:const SECTION-CHILDREN 0x01) +(def ^:const SECTION-BLUR-LAYER 0x02) +(def ^:const SECTION-BLUR-BG 0x04) +(def ^:const SECTION-SHADOWS 0x08) +(def ^:const SECTION-MASKED 0x10) +(def ^:const SECTION-BOOL-TYPE 0x20) +(def ^:const SECTION-GROW-TYPE 0x40) +(def ^:const SECTION-LAYOUT-ITEM 0x80) +(def ^:const SECTION-FLEX 0x100) +(def ^:const SECTION-FILLS 0x200) +(def ^:const SECTION-STROKES 0x400) + +(def ^:const STROKE-HEADER-U8-SIZE 36) +(def ^:const STROKE-ALIGN-CENTER 0) +(def ^:const STROKE-ALIGN-INNER 1) +(def ^:const STROKE-ALIGN-OUTER 2) + +(defn ->bytes + "Return a platform byte array / Uint8Array for a buffer written with + `app.common.buffer`." + [buffer] + #?(:clj (let [^java.nio.ByteBuffer bb buffer + arr (byte-array (.capacity bb))] + (.position bb 0) + (.get bb arr) + (.position bb 0) + arr) + :cljs (js/Uint8Array. (.-buffer ^js buffer) + (.-byteOffset ^js buffer) + (.-byteLength ^js buffer)))) + +(defn- copy-u8! + [dst-view dst-offset src-view src-offset nbytes] + #?(:clj + (buf/copy-bytes src-view src-offset nbytes dst-view dst-offset) + :cljs + (let [src (js/Uint8Array. (.-buffer ^js src-view) + (+ (.-byteOffset ^js src-view) src-offset) + nbytes) + dst (js/Uint8Array. (.-buffer ^js dst-view) + (+ (.-byteOffset ^js dst-view) dst-offset) + nbytes)] + (.set dst src)))) + +(defn- write-uuid! + "Write a UUID in the WASM `_set_shapes_batch` layout: four little-endian + u32 words matching `uuid/get-u32` / `uuid_from_u32_quartet`. + + Do not use `buf/write-uuid` on the JVM here: that helper stores two + big-endian longs (canonical UUID bytes). CLJS `buf/write-uuid` already + emits the LE u32 quartet WASM expects; a BE long layout makes every + shape id diverge from host `use-shape`, so text/path content never + attaches to the shapes the batch tree renders." + [dview offset id] + #?(:cljs + (buf/write-uuid dview offset id) + :clj + (let [msb (.getMostSignificantBits ^java.util.UUID id) + lsb (.getLeastSignificantBits ^java.util.UUID id) + a (unsigned-bit-shift-right msb 32) + b (bit-and msb 0xffffffff) + c (unsigned-bit-shift-right lsb 32) + d (bit-and lsb 0xffffffff)] + (buf/write-u32 dview offset a) + (buf/write-u32 dview (+ offset 4) b) + (buf/write-u32 dview (+ offset 8) c) + (buf/write-u32 dview (+ offset 12) d))) + (+ offset 16)) + +(defn- write-base-props! + [dview offset shape] + (let [id (dm/get-prop shape :id) + parent-id (get shape :parent-id) + shape-type (dm/get-prop shape :type) + clip-content (if (= shape-type :frame) + (not (get shape :show-content)) + false) + hidden (get shape :hidden false) + flags (cond-> 0 + clip-content (bit-or FLAG-CLIP-CONTENT) + hidden (bit-or FLAG-HIDDEN)) + blend-mode (enums/translate-blend-mode (get shape :blend-mode)) + constraint-h (enums/translate-constraint-h (or (get shape :constraints-h) :none)) + constraint-v (enums/translate-constraint-v (or (get shape :constraints-v) :none)) + opacity (d/nilv (get shape :opacity) 1.0) + rotation (d/nilv (get shape :rotation) 0.0) + transform (get shape :transform) + [ta tb tc td te tf] + (if (some? transform) + [(dm/get-prop transform :a) + (dm/get-prop transform :b) + (dm/get-prop transform :c) + (dm/get-prop transform :d) + (dm/get-prop transform :e) + (dm/get-prop transform :f)] + [1.0 0.0 0.0 1.0 0.0 0.0]) + selrect (get shape :selrect) + [sx1 sy1 sx2 sy2] + (if (some? selrect) + [(dm/get-prop selrect :x1) + (dm/get-prop selrect :y1) + (dm/get-prop selrect :x2) + (dm/get-prop selrect :y2)] + [0.0 0.0 0.0 0.0]) + r1 (d/nilv (get shape :r1) 0.0) + r2 (d/nilv (get shape :r2) 0.0) + r3 (d/nilv (get shape :r3) 0.0) + r4 (d/nilv (get shape :r4) 0.0)] + + (write-uuid! dview offset id) + (write-uuid! dview (+ offset 16) (d/nilv parent-id uuid/zero)) + (buf/write-u8 dview (+ offset 32) (enums/translate-shape-type shape-type)) + (buf/write-u8 dview (+ offset 33) flags) + (buf/write-u8 dview (+ offset 34) blend-mode) + (buf/write-u8 dview (+ offset 35) constraint-h) + (buf/write-u8 dview (+ offset 36) constraint-v) + (buf/write-f32 dview (+ offset 40) opacity) + (buf/write-f32 dview (+ offset 44) rotation) + (buf/write-f32 dview (+ offset 48) ta) + (buf/write-f32 dview (+ offset 52) tb) + (buf/write-f32 dview (+ offset 56) tc) + (buf/write-f32 dview (+ offset 60) td) + (buf/write-f32 dview (+ offset 64) te) + (buf/write-f32 dview (+ offset 68) tf) + (buf/write-f32 dview (+ offset 72) sx1) + (buf/write-f32 dview (+ offset 76) sy1) + (buf/write-f32 dview (+ offset 80) sx2) + (buf/write-f32 dview (+ offset 84) sy2) + (buf/write-f32 dview (+ offset 88) r1) + (buf/write-f32 dview (+ offset 92) r2) + (buf/write-f32 dview (+ offset 96) r3) + (buf/write-f32 dview (+ offset 100) r4) + (+ offset BASE-PROPS-SIZE))) + +(defn- write-blur! + [dview offset blur] + (buf/write-u8 dview offset (if (get blur :hidden) 1 0)) + (buf/write-f32 dview (+ offset 4) (get blur :value 0)) + (+ offset 8)) + +(defn- write-shadow! + [dview offset shadow] + (let [color (get shadow :color) + rgba (sr-clr/hex->u32argb (get color :color) + (get color :opacity))] + (buf/write-u32 dview offset rgba) + (buf/write-f32 dview (+ offset 4) (get shadow :blur 0)) + (buf/write-f32 dview (+ offset 8) (get shadow :spread 0)) + (buf/write-f32 dview (+ offset 12) (get shadow :offset-x 0)) + (buf/write-f32 dview (+ offset 16) (get shadow :offset-y 0)) + (buf/write-u8 dview (+ offset 20) (enums/translate-shadow-style (get shadow :style))) + (buf/write-u8 dview (+ offset 21) (if (get shadow :hidden) 1 0)) + (+ offset 24))) + +(defn- write-flex! + [dview offset shape] + (let [dir (-> (get shape :layout-flex-dir :row) + (enums/translate-layout-flex-dir)) + gap (get shape :layout-gap) + row-gap (get gap :row-gap 0) + column-gap (get gap :column-gap 0) + align-items (-> (get shape :layout-align-items) + enums/translate-layout-align-items) + align-content (-> (get shape :layout-align-content) + enums/translate-layout-align-content) + justify-items (-> (get shape :layout-justify-items) + enums/translate-layout-justify-items) + justify-content (-> (get shape :layout-justify-content) + enums/translate-layout-justify-content) + wrap-type (-> (get shape :layout-wrap-type) + enums/translate-layout-wrap-type) + padding (get shape :layout-padding) + padding-top (get padding :p1 0) + padding-right (get padding :p2 0) + padding-bottom (get padding :p3 0) + padding-left (get padding :p4 0)] + (buf/write-u8 dview offset dir) + (buf/write-u8 dview (+ offset 1) align-items) + (buf/write-u8 dview (+ offset 2) align-content) + (buf/write-u8 dview (+ offset 3) justify-items) + (buf/write-u8 dview (+ offset 4) justify-content) + (buf/write-u8 dview (+ offset 5) wrap-type) + (buf/write-f32 dview (+ offset 8) row-gap) + (buf/write-f32 dview (+ offset 12) column-gap) + (buf/write-f32 dview (+ offset 16) padding-top) + (buf/write-f32 dview (+ offset 20) padding-right) + (buf/write-f32 dview (+ offset 24) padding-bottom) + (buf/write-f32 dview (+ offset 28) padding-left) + (+ offset 32))) + +(defn- write-layout-item! + [dview offset shape] + (let [margins (get shape :layout-item-margin) + margin-top (get margins :m1 0) + margin-right (get margins :m2 0) + margin-bottom (get margins :m3 0) + margin-left (get margins :m4 0) + h-sizing (-> (get shape :layout-item-h-sizing) + enums/translate-layout-sizing) + v-sizing (-> (get shape :layout-item-v-sizing) + enums/translate-layout-sizing) + align-self (-> (get shape :layout-item-align-self) + enums/translate-align-self) + max-h (get shape :layout-item-max-h) + min-h (get shape :layout-item-min-h) + max-w (get shape :layout-item-max-w) + min-w (get shape :layout-item-min-w) + is-absolute (boolean (get shape :layout-item-absolute)) + z-index (get shape :layout-item-z-index) + flags (cond-> 0 + (some? max-h) (bit-or 0x01) + (some? min-h) (bit-or 0x02) + (some? max-w) (bit-or 0x04) + (some? min-w) (bit-or 0x08) + is-absolute (bit-or 0x10))] + (buf/write-f32 dview offset margin-top) + (buf/write-f32 dview (+ offset 4) margin-right) + (buf/write-f32 dview (+ offset 8) margin-bottom) + (buf/write-f32 dview (+ offset 12) margin-left) + (buf/write-u8 dview (+ offset 16) (d/nilv h-sizing 0)) + (buf/write-u8 dview (+ offset 17) (d/nilv v-sizing 0)) + (buf/write-u8 dview (+ offset 18) flags) + (buf/write-u8 dview (+ offset 19) (d/nilv align-self 0)) + (buf/write-f32 dview (+ offset 20) (d/nilv max-h 0)) + (buf/write-f32 dview (+ offset 24) (d/nilv min-h 0)) + (buf/write-f32 dview (+ offset 28) (d/nilv max-w 0)) + (buf/write-f32 dview (+ offset 32) (d/nilv min-w 0)) + (buf/write-i32 dview (+ offset 36) (d/nilv z-index 0)) + (+ offset 40))) + +(defn- fills-data-byte-size + [fills] + (+ 4 (* (count fills) types.fills.impl/FILL-U8-SIZE))) + +(defn- write-fills-section! + [dview offset fills] + (let [fills (types.fills/coerce (or fills [])) + nbytes (fills-data-byte-size fills) + src (.-dbuffer fills)] + (copy-u8! dview offset src 0 nbytes) + (+ offset nbytes))) + +(defn- write-stroke-fill! + [dview offset stroke] + (let [opacity (or (:stroke-opacity stroke) 1.0) + color (:stroke-color stroke) + gradient (:stroke-color-gradient stroke) + image (:stroke-image stroke)] + (cond + (some? gradient) + (types.fills.impl/write-gradient-fill offset dview opacity gradient) + + (some? image) + (types.fills.impl/write-image-fill offset dview opacity image) + + (some? color) + (types.fills.impl/write-solid-fill offset dview opacity color) + + :else + (types.fills.impl/write-solid-fill offset dview 0.0 "#000000")))) + +(defn- write-stroke! + [dview offset stroke] + (let [width (or (:stroke-width stroke) 1.0) + style (-> stroke :stroke-style enums/translate-stroke-style) + align (case (:stroke-alignment stroke) + :inner STROKE-ALIGN-INNER + :outer STROKE-ALIGN-OUTER + STROKE-ALIGN-CENTER) + cap-start (-> stroke :stroke-cap-start enums/translate-stroke-cap) + cap-end (-> stroke :stroke-cap-end enums/translate-stroke-cap) + dash (or (:stroke-dash stroke) -1) + gap (or (:stroke-gap stroke) -1) + top (or (:stroke-width-top stroke) width) + right (or (:stroke-width-right stroke) width) + bottom (or (:stroke-width-bottom stroke) width) + left (or (:stroke-width-left stroke) width) + has-sides? (not= top right bottom left)] + + (buf/write-f32 dview offset width) + (buf/write-u8 dview (+ offset 4) style) + (buf/write-u8 dview (+ offset 5) align) + (buf/write-u8 dview (+ offset 6) (d/nilv cap-start 0)) + (buf/write-u8 dview (+ offset 7) (d/nilv cap-end 0)) + (buf/write-f32 dview (+ offset 8) dash) + (buf/write-f32 dview (+ offset 12) gap) + (buf/write-u8 dview (+ offset 16) (if has-sides? 1 0)) + (buf/write-f32 dview (+ offset 20) top) + (buf/write-f32 dview (+ offset 24) right) + (buf/write-f32 dview (+ offset 28) bottom) + (buf/write-f32 dview (+ offset 32) left) + (write-stroke-fill! dview (+ offset STROKE-HEADER-U8-SIZE) stroke) + (+ offset STROKE-HEADER-U8-SIZE types.fills.impl/FILL-U8-SIZE))) + +(defn- visible-strokes + [shape] + (let [type (dm/get-prop shape :type)] + (if (= type :group) + [] + (into [] (remove :hidden) (or (get shape :strokes) []))))) + +(defn- write-strokes-section! + [dview offset strokes] + (buf/write-u32 dview offset (count strokes)) + (reduce (fn [o s] (write-stroke! dview o s)) + (+ offset 4) + strokes)) + +(defn write-shape-payload! + "Serialize one shape's structural payload into `dview` starting at `offset` + (payload only — no length prefix). Returns the offset after the payload." + [dview offset shape {:keys [include-layout? include-fills-strokes?] + :or {include-layout? false + include-fills-strokes? false}}] + (let [shape-type (dm/get-prop shape :type) + children (into [] (filter uuid?) (get shape :shapes)) + blur (get shape :blur) + bg-blur (get shape :background-blur) + shadows (or (get shape :shadow) []) + masked? (and (= shape-type :group) (boolean (get shape :masked-group))) + bool-type (when (= shape-type :bool) (get shape :bool-type)) + grow-type (when (= shape-type :text) (get shape :grow-type)) + flex? (and include-layout? (ctl/flex-layout? shape)) + layout-item? include-layout? + strokes (when include-fills-strokes? (visible-strokes shape)) + + mask (cond-> 0 + true (bit-or SECTION-CHILDREN) + (some? blur) (bit-or SECTION-BLUR-LAYER) + (some? bg-blur) (bit-or SECTION-BLUR-BG) + (seq shadows) (bit-or SECTION-SHADOWS) + (= shape-type :group) (bit-or SECTION-MASKED) + (some? bool-type) (bit-or SECTION-BOOL-TYPE) + (some? grow-type) (bit-or SECTION-GROW-TYPE) + flex? (bit-or SECTION-FLEX) + layout-item? (bit-or SECTION-LAYOUT-ITEM) + include-fills-strokes? (bit-or SECTION-FILLS) + include-fills-strokes? (bit-or SECTION-STROKES)) + + offset (write-base-props! dview offset shape) + _ (buf/write-u32 dview offset mask) + offset (+ offset 4) + + offset (let [o offset] + (buf/write-u32 dview o (count children)) + (reduce (fn [o id] (write-uuid! dview o id)) + (+ o 4) + children)) + + offset (cond-> offset + (some? blur) + (as-> o (write-blur! dview o blur))) + + offset (cond-> offset + (some? bg-blur) + (as-> o (write-blur! dview o bg-blur))) + + offset (cond-> offset + (seq shadows) + (as-> o + (do + (buf/write-u32 dview o (count shadows)) + (reduce (fn [o s] (write-shadow! dview o s)) + (+ o 4) + shadows)))) + + offset (cond-> offset + (= shape-type :group) + (as-> o + (do (buf/write-u8 dview o (if masked? 1 0)) + (+ o 4)))) + + offset (cond-> offset + (some? bool-type) + (as-> o + (do (buf/write-u8 dview o (enums/translate-bool-type bool-type)) + (+ o 4)))) + + offset (cond-> offset + (some? grow-type) + (as-> o + (do (buf/write-u8 dview o (enums/translate-grow-type grow-type)) + (+ o 4)))) + + offset (cond-> offset + flex? + (as-> o (write-flex! dview o shape))) + + offset (cond-> offset + layout-item? + (as-> o (write-layout-item! dview o shape))) + + offset (cond-> offset + include-fills-strokes? + (as-> o (write-fills-section! dview o (get shape :fills)))) + + offset (cond-> offset + include-fills-strokes? + (as-> o (write-strokes-section! dview o strokes)))] + offset)) + +(defn- payload-byte-size + [shape {:keys [include-layout? include-fills-strokes?] + :or {include-layout? false include-fills-strokes? false}}] + (let [children (into [] (filter uuid?) (get shape :shapes)) + shadows (or (get shape :shadow) []) + shape-type (dm/get-prop shape :type) + blur (get shape :blur) + bg-blur (get shape :background-blur) + flex? (and include-layout? (ctl/flex-layout? shape)) + fills-size (if include-fills-strokes? + (fills-data-byte-size + (types.fills/coerce (or (get shape :fills) []))) + 0) + strokes (when include-fills-strokes? (visible-strokes shape)) + strokes-size (if include-fills-strokes? + (+ 4 (* (count strokes) + (+ STROKE-HEADER-U8-SIZE types.fills.impl/FILL-U8-SIZE))) + 0)] + (+ BASE-PROPS-SIZE + 4 + (+ 4 (* 16 (count children))) + (if (some? blur) 8 0) + (if (some? bg-blur) 8 0) + (if (seq shadows) (+ 4 (* 24 (count shadows))) 0) + (if (= shape-type :group) 4 0) + (if (and (= shape-type :bool) (some? (get shape :bool-type))) 4 0) + (if (and (= shape-type :text) (some? (get shape :grow-type))) 4 0) + (if flex? 32 0) + (if include-layout? 40 0) + fills-size + strokes-size))) + +(defn encode-shape-record + "Returns platform bytes for `[u32 payload_len][payload]`." + [shape opts] + (let [capacity (+ 4 (payload-byte-size shape opts)) + dview (buf/allocate capacity) + end (write-shape-payload! dview 4 shape opts) + payload-len (- end 4)] + (assert (= end capacity) + (str "upload record size mismatch: wrote " end " expected " capacity)) + (buf/write-u32 dview 0 payload-len) + (->bytes dview))) + +(defn encode-shapes-batch + "Encode `shapes` as one `_set_shapes_batch` buffer. Returns platform bytes. + Opts: `:include-layout?`, `:include-fills-strokes?`." + [shapes opts] + (if-not (seq shapes) + (->bytes (let [b (buf/allocate 4)] + (buf/write-u32 b 0 0) + b)) + (let [records (mapv #(encode-shape-record % opts) shapes) + total (reduce (fn [acc u8] + (+ acc #?(:clj (alength ^bytes u8) + :cljs (.-byteLength ^js u8)))) + 4 + records) + dview (buf/allocate total)] + (buf/write-u32 dview 0 (count records)) + (reduce (fn [o u8] + (let [len #?(:clj (alength ^bytes u8) + :cljs (.-byteLength ^js u8))] + #?(:clj + (let [src (java.nio.ByteBuffer/wrap ^bytes u8)] + (.order src java.nio.ByteOrder/LITTLE_ENDIAN) + (buf/copy-bytes src 0 len dview o)) + :cljs + (.set (js/Uint8Array. (.-buffer ^js dview) + (+ (.-byteOffset ^js dview) o) + len) + u8)) + (+ o len))) + 4 + records) + (->bytes dview)))) + +(defn shapes-in-tree-order + "Shapes sorted parents-before-children (workspace cold-load order)." + [objects] + (if (contains? objects uuid/zero) + (let [ordered-ids (cfh/get-children-ids-with-self objects uuid/zero)] + (into [] (keep #(get objects %)) ordered-ids)) + (let [top-level-ids (->> (vals objects) + (filter (fn [shape] + (not (contains? objects (:parent-id shape))))) + (map :id)) + all-ordered-ids (into [] + (mapcat #(cfh/get-children-ids-with-self objects %)) + top-level-ids)] + (into [] (keep #(get objects %)) all-ordered-ids)))) + +(defn encode-page-objects + "Encode page `:objects` map for cold-load (layout + fills/strokes)." + [objects] + (encode-shapes-batch + (shapes-in-tree-order objects) + {:include-layout? true + :include-fills-strokes? true})) diff --git a/common/src/app/common/render_wasm/enums_data.cljc b/common/src/app/common/render_wasm/enums_data.cljc new file mode 100644 index 0000000000..50370a40c7 --- /dev/null +++ b/common/src/app/common/render_wasm/enums_data.cljc @@ -0,0 +1,98 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.common.render-wasm.enums-data + "Pure enum discriminant tables for WASM batch encode (JVM + JS). + + Values match `render-wasm` `Raw*` enums consumed by `_set_shapes_batch`. + Kept separate from `shared.js` so the backend can encode without WASM." + (:require + [app.common.data :as d])) + +(def shape-type + {:frame 0 :group 1 :bool 2 :rect 3 :path 4 :text 5 :circle 6 :svg-raw 7}) + +(def blend-mode + {:normal 3 :screen 14 :overlay 15 :darken 16 :lighten 17 + :color-dodge 18 :color-burn 19 :hard-light 20 :soft-light 21 + :difference 22 :exclusion 23 :multiply 24 :hue 25 :saturation 26 + :color 27 :luminosity 28}) + +;; Matches Rust RawConstraintH/V (None = 0). +(def constraint-h + {:none 0 :left 1 :right 2 :leftright 3 :center 4 :scale 5}) + +(def constraint-v + {:none 0 :top 1 :bottom 2 :topbottom 3 :center 4 :scale 5}) + +(def shadow-style + {:drop-shadow 0 :inner-shadow 1}) + +(def bool-type + {:union 0 :difference 1 :intersection 2 :exclude 3}) + +(def grow-type + {:fixed 0 :auto-width 1 :auto-height 2}) + +(def sizing + {:fill 0 :fix 1 :auto 2}) + +(def flex-direction + {:row 0 :row-reverse 1 :column 2 :column-reverse 3}) + +(def wrap-type + {:wrap 0 :nowrap 1}) + +(def align-items + {:start 0 :end 1 :center 2 :stretch 3}) + +(def align-content + {:start 0 :end 1 :center 2 + :space-between 3 :space-around 4 :space-evenly 5 :stretch 6}) + +(def justify-items + {:start 0 :end 1 :center 2 :stretch 3}) + +(def justify-content + {:start 0 :end 1 :center 2 + :space-between 3 :space-around 4 :space-evenly 5 :stretch 6}) + +(def align-self + {:none 0 :auto 1 :start 2 :end 3 :center 4 :stretch 5}) + +(def stroke-style + {:solid 0 :dotted 1 :dashed 2 :mixed 3}) + +(def stroke-cap + {:none 0 :line-arrow 1 :triangle-arrow 2 :square-marker 3 + :circle-marker 4 :diamond-marker 5 :round 6 :square 7}) + +(defn- lookup + [table value default-key] + (let [k (cond + (keyword? value) value + (string? value) (keyword value) + :else value) + default (get table default-key)] + (d/nilv (get table k) default))) + +(defn translate-shape-type [type] (lookup shape-type type :rect)) +(defn translate-blend-mode [mode] (lookup blend-mode mode :normal)) +(defn translate-constraint-h [type] (lookup constraint-h type :none)) +(defn translate-constraint-v [type] (lookup constraint-v type :none)) +(defn translate-shadow-style [style] (lookup shadow-style style :drop-shadow)) +(defn translate-bool-type [type] (lookup bool-type type :union)) +(defn translate-grow-type [type] (lookup grow-type type :fixed)) +(defn translate-layout-sizing [type] (lookup sizing type :fix)) +(defn translate-layout-flex-dir [type] (lookup flex-direction type :row)) +(defn translate-layout-wrap-type [type] (lookup wrap-type type :nowrap)) +(defn translate-layout-align-items [type] (lookup align-items type :start)) +(defn translate-layout-align-content [type] (lookup align-content type :stretch)) +(defn translate-layout-justify-items [type] (lookup justify-items type :start)) +(defn translate-layout-justify-content [type] (lookup justify-content type :stretch)) +(defn translate-align-self [type] (lookup align-self type :none)) +(defn translate-stroke-style [type] (lookup stroke-style type :solid)) +(defn translate-stroke-cap [type] (lookup stroke-cap type :none)) diff --git a/common/src/app/common/render_wasm/serialize_shape.cljs b/common/src/app/common/render_wasm/serialize_shape.cljs index 1df4a84505..8aa41c402e 100644 --- a/common/src/app/common/render_wasm/serialize_shape.cljs +++ b/common/src/app/common/render_wasm/serialize_shape.cljs @@ -75,3 +75,14 @@ (wselect/use-shape (:id shape)) (write-shape-tail! shape)))) prepared)) + +(defn apply-batch-tails! + "Apply svg-attrs/path tails after a pre-encoded structural batch ingest. + Returns the svg-derived prepared shapes vector." + [shapes] + (let [prepared (mapv svg-derived/apply-svg-derived shapes)] + (doseq [shape prepared] + (when (needs-shape-tail? shape) + (wselect/use-shape (:id shape)) + (write-shape-tail! shape))) + prepared)) diff --git a/common/src/app/common/render_wasm/serializers/color.cljc b/common/src/app/common/render_wasm/serializers/color.cljc new file mode 100644 index 0000000000..ddf55783c5 --- /dev/null +++ b/common/src/app/common/render_wasm/serializers/color.cljc @@ -0,0 +1,21 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.common.render-wasm.serializers.color + (:require + [app.common.math :as mth])) + +(defn hex->u32argb + "Takes a hex color in #rrggbb format, and an opacity value from 0 to 1 and + returns its 32-bit argb representation." + [hex opacity] + (let [hex (or hex "#000000") + rgb #?(:cljs (js/parseInt (subs hex 1) 16) + :clj (Integer/parseInt (subs hex 1) 16)) + ;; `mth/floor` returns a double on the JVM; bit ops need an int. + a (int (mth/floor (* (or opacity 1.0) 0xff)))] + #?(:cljs (unsigned-bit-shift-right (bit-or (bit-shift-left a 24) rgb) 0) + :clj (bit-and (bit-or (bit-shift-left a 24) rgb) 0xffffffff)))) diff --git a/common/src/app/common/render_wasm/serializers/color.cljs b/common/src/app/common/render_wasm/serializers/color.cljs deleted file mode 100644 index c3ef27fdfc..0000000000 --- a/common/src/app/common/render_wasm/serializers/color.cljs +++ /dev/null @@ -1,11 +0,0 @@ -(ns app.common.render-wasm.serializers.color - (:require - [app.common.math :as mth])) - -(defn hex->u32argb - "Takes a hex color in #rrggbb format, and an opacity value from 0 to 1 and returns its 32-bit argb representation" - [hex opacity] - (let [rgb (js/parseInt (subs hex 1) 16) - a (mth/floor (* (or opacity 1) 0xff))] - ;; rgba >>> 0 so we have an unsigned representation - (unsigned-bit-shift-right (bit-or (bit-shift-left a 24) rgb) 0))) \ No newline at end of file diff --git a/common/test/common_tests/render_wasm/upload_encode_test.cljc b/common/test/common_tests/render_wasm/upload_encode_test.cljc new file mode 100644 index 0000000000..91d05263b7 --- /dev/null +++ b/common/test/common_tests/render_wasm/upload_encode_test.cljc @@ -0,0 +1,156 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns common-tests.render-wasm.upload-encode-test + (:require + [app.common.render-wasm.api.upload-encode :as encode] + [app.common.render-wasm.enums-data :as enums] + [app.common.render-wasm.serializers.color :as sr-clr] + [app.common.uuid :as uuid] + [clojure.test :as t])) + +(defn- sample-rect + [id parent-id] + {:id id + :parent-id parent-id + :type :rect + :name "r" + :frame-id parent-id + :shapes [] + :opacity 1 + :rotation 0 + :selrect {:x1 0 :y1 0 :x2 100 :y2 50 :x 0 :y 0 :width 100 :height 50} + :points [] + :fills [{:fill-color "#ff0000" :fill-opacity 1}] + :strokes [] + :hidden false}) + +(defn- sample-root + [child-id] + {:id uuid/zero + :type :frame + :name "Root" + :parent-id nil + :frame-id uuid/zero + :shapes [child-id] + :selrect {:x1 0 :y1 0 :x2 100 :y2 50 :x 0 :y 0 :width 100 :height 50} + :fills [] + :strokes [] + :opacity 1 + :rotation 0 + :show-content true + :hidden false}) + +(t/deftest enums-shape-type-defaults + (t/is (= 3 (enums/translate-shape-type :rect))) + (t/is (= 0 (enums/translate-shape-type :frame))) + (t/is (= 3 (enums/translate-shape-type :unknown-type))) + (t/is (= 0 (enums/translate-constraint-h :none))) + (t/is (= 1 (enums/translate-constraint-h :left)))) + +(t/deftest encode-empty-batch + (let [bytes (encode/encode-shapes-batch [] {:include-layout? true + :include-fills-strokes? true})] + (t/is (= 4 #?(:clj (alength ^bytes bytes) + :cljs (.-byteLength bytes)))) + (t/is (= 0 (bit-and #?(:clj (aget ^bytes bytes 0) + :cljs (aget bytes 0)) + 0xff))))) + +(t/deftest encode-page-objects-smoke + (let [child-id (uuid/next) + objects {uuid/zero (sample-root child-id) + child-id (sample-rect child-id uuid/zero)} + bytes (encode/encode-page-objects objects) + len #?(:clj (alength ^bytes bytes) + :cljs (.-byteLength bytes))] + (t/is (< 4 len)) + ;; shape_count == 2 (root + rect), little-endian u32 at 0 + (t/is (= 2 (+ #?(:clj (aget ^bytes bytes 0) + :cljs (aget bytes 0)) + (bit-shift-left #?(:clj (aget ^bytes bytes 1) + :cljs (aget bytes 1)) + 8) + (bit-shift-left #?(:clj (aget ^bytes bytes 2) + :cljs (aget bytes 2)) + 16) + (bit-shift-left #?(:clj (aget ^bytes bytes 3) + :cljs (aget bytes 3)) + 24)))))) + +(t/deftest hex-u32argb-accepts-double-opacity + ;; Shadows often carry opacity as a double; JVM bit-shift needs an int. + (let [c (sr-clr/hex->u32argb "#ff0000" 0.5)] + (t/is (int? c)) + (t/is (pos? c)))) + +(t/deftest encode-shape-with-shadow + (let [child-id (uuid/next) + rect (-> (sample-rect child-id uuid/zero) + (assoc :shadow + [{:id (uuid/next) + :style :drop-shadow + :color {:color "#000000" :opacity 0.5} + :offset-x 2 + :offset-y 2 + :blur 4 + :spread 0 + :hidden false}])) + objects {uuid/zero (sample-root child-id) + child-id rect} + bytes (encode/encode-page-objects objects)] + (t/is (< 4 #?(:clj (alength ^bytes bytes) + :cljs (.-byteLength bytes)))))) + +(defn- u8-at + [bytes i] + (bit-and #?(:clj (aget ^bytes bytes i) + :cljs (aget bytes i)) + 0xff)) + +(t/deftest encode-uuid-matches-wasm-le-u32-quartet + ;; Canonical UUID bytes a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8, written as + ;; four LE u32s (same layout CLJS buf/write-uuid / get-u32 emit). + (let [id #uuid "a1a2a3a4-b1b2-c1c2-d1d2-d3d4d5d6d7d8" + shape (-> (sample-rect id uuid/zero) + (assoc :parent-id uuid/zero + :fills [] + :strokes [])) + rec (encode/encode-shape-record + shape + {:include-layout? false + :include-fills-strokes? false}) + ;; Record: [u32 payload_len][104 base…]; id starts at byte 4. + expected [0xa4 0xa3 0xa2 0xa1 + 0xc2 0xc1 0xb2 0xb1 + 0xd4 0xd3 0xd2 0xd1 + 0xd8 0xd7 0xd6 0xd5]] + (doseq [i (range 16)] + (t/is (= (nth expected i) (u8-at rec (+ 4 i))) + (str "uuid byte " i))))) + +(t/deftest encode-text-shape-type-and-grow + (let [id (uuid/next) + shape {:id id + :parent-id uuid/zero + :type :text + :name "t" + :frame-id uuid/zero + :shapes [] + :grow-type :auto-width + :selrect {:x1 0 :y1 0 :x2 40 :y2 20 :x 0 :y 0 :width 40 :height 20} + :fills [] + :strokes [] + :opacity 1 + :rotation 0 + :hidden false} + rec (encode/encode-shape-record + shape + {:include-layout? true + :include-fills-strokes? true})] + ;; shape_type at base offset 32 → record byte 4+32 + (t/is (= 5 (u8-at rec (+ 4 32)))) + (t/is (= 1 (enums/translate-grow-type :auto-width))))) diff --git a/common/test/common_tests/runner.cljc b/common/test/common_tests/runner.cljc index 8cab260e59..50a80363ae 100644 --- a/common/test/common_tests/runner.cljc +++ b/common/test/common_tests/runner.cljc @@ -14,6 +14,7 @@ [clojure.test :as t] [common-tests.attrs-test] [common-tests.buffer-test] + [common-tests.render-wasm.upload-encode-test] [common-tests.colors-test] [common-tests.data-test] [common-tests.files-builder-test] @@ -99,6 +100,7 @@ 'common-tests.attrs-test 'common-tests.buffer-test 'common-tests.colors-test + 'common-tests.render-wasm.upload-encode-test 'common-tests.data-test 'common-tests.files-changes-test 'common-tests.files-builder-test diff --git a/frontend/src/app/main/data/workspace.cljs b/frontend/src/app/main/data/workspace.cljs index b04bf5394b..d9151f2c15 100644 --- a/frontend/src/app/main/data/workspace.cljs +++ b/frontend/src/app/main/data/workspace.cljs @@ -357,10 +357,20 @@ [file-id features] (ptk/reify ::fetch-bundle ptk/WatchEvent - (watch [_ _ stream] + (watch [_ state stream] (log/debug :hint "fetch bundle" :file-id (dm/str file-id)) - (let [stopper-s (rx/filter (ptk/type? ::finalize-workspace) stream)] + (let [stopper-s (rx/filter (ptk/type? ::finalize-workspace) stream) + page-id (or (some-> (rt/get-params state) :page-id uuid/parse*) + (:current-page-id state) + (some-> (dsh/lookup-file-data state file-id) + (get :pages) + (first)))] + + ;; Prototype: overlap binary shapes-batch download with Transit fetch. + (when (uuid? page-id) + (wasm.api/prefetch-cold-load-batch! file-id page-id)) + (->> (rx/zip (rp/cmd! :get-file {:id file-id :features features}) (get-file-object-thumbnails file-id)) (rx/take 1) diff --git a/frontend/src/app/main/data/workspace/pages.cljs b/frontend/src/app/main/data/workspace/pages.cljs index c8b8180c41..e70a48f075 100644 --- a/frontend/src/app/main/data/workspace/pages.cljs +++ b/frontend/src/app/main/data/workspace/pages.cljs @@ -32,6 +32,7 @@ [app.main.features :as features] [app.main.router :as rt] [app.main.worker :as mw] + [app.render-wasm.api :as wasm.api] [app.render-wasm.shape :as wasm.shape] [app.util.http :as http] [app.util.i18n :as i18n :refer [tr]] @@ -104,22 +105,27 @@ ptk/WatchEvent (watch [_ state _] (if (dsh/lookup-page state file-id page-id) - (rx/concat - (rx/of (initialize-page* file-id page-id) - (fdf/fix-deleted-fonts-for-page file-id page-id)) + (do + ;; Prototype: prefetch server shapes-batch on page open/switch + ;; so cold ingest can skip client structural encode. + (wasm.api/prefetch-cold-load-batch! file-id page-id) - ;; Disable thumbnail generation in wasm renderer - (if (features/active-feature? state "render-wasm/v1") - (rx/empty) - (rx/of (dwth/watch-state-changes file-id page-id))) + (rx/concat + (rx/of (initialize-page* file-id page-id) + (fdf/fix-deleted-fonts-for-page file-id page-id)) - (rx/of (dwl/watch-component-changes)) - (rx/of (dwl/watch-token-changes)) + ;; Disable thumbnail generation in wasm renderer + (if (features/active-feature? state "render-wasm/v1") + (rx/empty) + (rx/of (dwth/watch-state-changes file-id page-id))) - (let [profile (:profile state) - props (get profile :props)] - (when (not (:workspace-visited props)) - (rx/of (select-frame-tool file-id page-id))))) + (rx/of (dwl/watch-component-changes)) + (rx/of (dwl/watch-token-changes)) + + (let [profile (:profile state) + props (get profile :props)] + (when (not (:workspace-visited props)) + (rx/of (select-frame-tool file-id page-id)))))) ;; NOTE: this redirect is necessary for cases where user ;; explicitly passes an non-existing page-id on the url diff --git a/frontend/src/app/main/repo.cljs b/frontend/src/app/main/repo.cljs index 01ce9e4c01..9193314575 100644 --- a/frontend/src/app/main/repo.cljs +++ b/frontend/src/app/main/repo.cljs @@ -159,6 +159,7 @@ {:stream? true} :export-binfile {:response-type :blob} + :get-file-wasm-shapes-batch {:response-type :buffer} :retrieve-list-of-builtin-templates {:query-params :all}}) (defn- send! diff --git a/frontend/src/app/render_wasm/api.cljs b/frontend/src/app/render_wasm/api.cljs index dd5d75f4da..ad3f8217a8 100644 --- a/frontend/src/app/render_wasm/api.cljs +++ b/frontend/src/app/render_wasm/api.cljs @@ -18,6 +18,7 @@ [app.common.math :as mth] [app.common.render-wasm.api.props :as props] [app.common.render-wasm.api.select :as wselect] + [app.common.render-wasm.api.upload :as upload] [app.common.render-wasm.helpers :as h] [app.common.render-wasm.mem :as mem] [app.common.render-wasm.mem.heap32 :as mem.h32] @@ -38,6 +39,7 @@ [app.main.data.render-wasm :as drw] [app.main.data.workspace.texts-v3 :as texts] [app.main.refs :as refs] + [app.main.repo :as rp] [app.main.router :as rt] [app.main.store :as st] [app.main.ui.shapes.text] @@ -67,6 +69,58 @@ (def use-dpr? (contains? cf/flags :render-wasm-dpr)) +(defn wasm-shapes-batch-enabled? + "Prototype: server-encoded cold-load batch (on by default). + Disable with `disable-wasm-shapes-batch` in penpotFlags / PENPOT_FLAGS." + [] + (contains? cf/flags :wasm-shapes-batch)) + +;; Prefetched `_set_shapes_batch` bytes for the next cold load. +;; Shape: {:file-id uuid :page-id uuid :promise Promise} +(defonce ^:private cold-load-batch* (atom nil)) + +(defn clear-cold-load-batch! + [] + (reset! cold-load-batch* nil)) + +(defn prefetch-cold-load-batch! + "Start downloading the server-encoded shapes batch for `file-id`/`page-id`. + Safe to call multiple times; only the latest pair is kept." + [file-id page-id] + (when (and (wasm-shapes-batch-enabled?) (uuid? file-id) (uuid? page-id)) + (let [t0 (js/performance.now) + prom + (p/create + (fn [resolve reject] + (->> (rp/cmd! :get-file-wasm-shapes-batch + {:file-id file-id + :page-id page-id}) + (rx/take 1) + (rx/subs! + (fn [buf] + (let [u8 (js/Uint8Array. buf) + ms (- (js/performance.now) t0)] + (js/console.info + "[wasm-batch] download" + (.-byteLength u8) "bytes in" + (.toFixed ms 1) "ms") + (resolve u8))) + (fn [err] + (js/console.warn "[wasm-batch] download failed" err) + (resolve nil))))))] + (reset! cold-load-batch* + {:file-id file-id + :page-id page-id + :promise prom}) + prom))) + +(defn- take-cold-load-batch-promise + [file-id page-id] + (when-let [entry @cold-load-batch*] + (when (and (= file-id (:file-id entry)) + (= page-id (:page-id entry))) + (:promise entry)))) + (defn- wasm-get-numeric-value "Read a positive numeric query param (e.g. `?dpr=2`)." [name] @@ -1788,24 +1842,15 @@ (def ^:private ^:const BATCH_MAX_SHAPES 512) -(defn- process-shapes-chunk - "Process up to `BATCH_MAX_SHAPES` shapes starting at `start-index`. - - Structural attrs are uploaded in one `_set_shapes_batch` FFI per chunk; - host-specific attrs (fills/strokes/text/grid/path) stay per-shape. - - Returns {:thumbnails [...] :full [...] :text-font-state {...} :next-index n}" - [shapes start-index thumbnails-acc full-acc text-font-state-acc] - (let [total (count shapes) +(defn- process-host-attrs-chunk + "Process host-specific attrs for up to `BATCH_MAX_SHAPES` already-uploaded + shapes. `prepared` must already be svg-derived." + [prepared start-index thumbnails-acc full-acc text-font-state-acc] + (let [total (count prepared) end-index (min total (+ start-index BATCH_MAX_SHAPES)) - chunk (into [] (subvec (if (vector? shapes) shapes (vec shapes)) - start-index end-index)) - prepared (serialize-shape/serialize-shapes-batch! - chunk - {:include-layout? true - :include-fills-strokes? true})] - - (loop [xs prepared + chunk (into [] (subvec (if (vector? prepared) prepared (vec prepared)) + start-index end-index))] + (loop [xs chunk t-acc (transient thumbnails-acc) f-acc (transient full-acc) font-state-acc text-font-state-acc] @@ -1824,6 +1869,114 @@ :text-font-state font-state-acc :next-index end-index})))) +(defn- process-shapes-chunk + "Process up to `BATCH_MAX_SHAPES` shapes starting at `start-index`. + + Structural attrs are uploaded in one `_set_shapes_batch` FFI per chunk; + host-specific attrs (fills/strokes/text/grid/path) stay per-shape. + + Returns {:thumbnails [...] :full [...] :text-font-state {...} :next-index n}" + [shapes start-index thumbnails-acc full-acc text-font-state-acc] + (let [total (count shapes) + end-index (min total (+ start-index BATCH_MAX_SHAPES)) + chunk (into [] (subvec (if (vector? shapes) shapes (vec shapes)) + start-index end-index)) + prepared (serialize-shape/serialize-shapes-batch! + chunk + {:include-layout? true + :include-fills-strokes? true})] + (-> (process-host-attrs-chunk prepared 0 thumbnails-acc full-acc text-font-state-acc) + (assoc :next-index end-index)))) + +(defn- read-batch-shape-id-at + "Decode shape id at `byte-offset` from a `_set_shapes_batch` buffer + (LE u32 quartet)." + [u8 byte-offset] + (when (and u8 (>= (.-byteLength ^js u8) (+ byte-offset 16))) + (let [view (js/DataView. (.-buffer ^js u8) + (.-byteOffset ^js u8) + (.-byteLength ^js u8)) + a (.getUint32 view byte-offset true) + b (.getUint32 view (+ byte-offset 4) true) + c (.getUint32 view (+ byte-offset 8) true) + d (.getUint32 view (+ byte-offset 12) true)] + (uuid/from-unsigned-parts a b c d)))) + +(defn- find-batch-child-shape-id + "Walk the batch and return the first shape id that is not `uuid/zero`. + Root is all-zero so it matches under both BE and LE layouts — useless + for detecting the JVM UUID endian bug." + [u8] + (when (and u8 (>= (.-byteLength ^js u8) 4)) + (let [view (js/DataView. (.-buffer ^js u8) + (.-byteOffset ^js u8) + (.-byteLength ^js u8)) + count (.getUint32 view 0 true)] + (loop [i 0 + off 4] + (when (and (< i count) (<= (+ off 4 16) (.-byteLength ^js u8))) + (let [plen (.getUint32 view off true) + id (read-batch-shape-id-at u8 (+ off 4))] + (if (and id (not= id uuid/zero)) + id + (recur (inc i) (+ off 4 plen))))))))) + +(defn- server-batch-ids-match? + "True when a non-root id from the server buffer appears in `shapes`. + A mismatch means the JVM encoder still emits BE UUID longs — host text + content would attach to orphan shapes and texts stay invisible." + [u8 shapes] + (let [batch-id (find-batch-child-shape-id u8) + shape-ids (into #{} (map :id) shapes)] + (cond + (nil? batch-id) + (do (js/console.warn "[wasm-batch] no non-root id in buffer") + false) + + (contains? shape-ids batch-id) + (do (js/console.info "[wasm-batch] uuid parity ok" (str batch-id)) + true) + + :else + (do (js/console.error "[wasm-batch] uuid parity FAIL" + "batch-child=" (str batch-id) + "not in page shapes — refusing ingest" + "(backend still on BE uuid layout?)") + false)))) + +(defn- maybe-ingest-server-batch! + "If a prefetched server batch is available for the current page, flush it + and apply path/svg tails. Returns a promise of + `{:mode :server :prepared [...]}` or `{:mode :client}`." + [shapes] + (let [state @st/state + file-id (:current-file-id state) + page-id (:current-page-id state) + batch-p (take-cold-load-batch-promise file-id page-id)] + (if-not batch-p + (p/resolved {:mode :client}) + (-> batch-p + (p/then + (fn [u8] + (clear-cold-load-batch!) + (cond + (nil? u8) + {:mode :client} + + (not (server-batch-ids-match? u8 shapes)) + {:mode :client} + + :else + (let [t0 (js/performance.now)] + (upload/flush-shapes-batch-bytes! u8) + (let [prepared (serialize-shape/apply-batch-tails! shapes) + ms (- (js/performance.now) t0)] + (js/console.info "[wasm-batch] structural ingest" + (.-byteLength u8) "bytes in" + (.toFixed ms 1) "ms" + "protocol" upload/PROTOCOL-VERSION) + {:mode :server :prepared prepared}))))))))) + (defn- set-objects-async "Asynchronously process shapes in time-budgeted chunks, yielding to the browser between chunks so the UI stays responsive. @@ -1832,72 +1985,78 @@ (let [total-shapes (count shapes)] (p/create (fn [resolve _reject] - (letfn [(process-next-chunk [index thumbnails-acc full-acc text-font-state-acc] + (letfn [(finalize! [thumbnails-acc full-acc text-font-state-acc prepared-shapes] + (perf/end-measure "set-objects") + (when on-shapes-ready (on-shapes-ready)) + (if-not (wasm/live?) + (do + (end-shapes-loading!) + (resolve nil)) + (do + (h/call wasm/internal-module "_end_loading") + (end-shapes-loading!) + (h/call wasm/internal-module "_set_view_end") + (reset! view-interaction-active? false) + (let [text-ids (into [] (comp (filter cfh/text-shape?) (map :id)) prepared-shapes)] + (when (seq text-ids) + (update-text-layouts text-ids))) + (if render-callback + (render-callback) + (request-render "set-objects-complete")) + (ug/dispatch! (ug/event "penpot:wasm:set-objects")) + (resolve nil) + (let [pending-thumbnails (d/index-by :key :callback thumbnails-acc) + pending-full (d/index-by :key :callback full-acc)] + (when (or (seq pending-thumbnails) (seq pending-full)) + (->> (rx/concat + (->> (rx/from (vals pending-thumbnails)) + (rx/merge-map + (fn [callback] + (if (fn? callback) (callback) (rx/empty)))) + (rx/reduce conj [])) + (->> (rx/from (vals pending-full)) + (rx/mapcat + (fn [callback] + (if (fn? callback) (callback) (rx/empty)))) + (rx/reduce conj []))) + (rx/subs! + noop-fn + noop-fn + (fn [] + (relayout-after-fonts! prepared-shapes text-font-state-acc) + (request-render "images-loaded"))))))))) + + (process-next-client-chunk [index thumbnails-acc full-acc text-font-state-acc] (if (< index total-shapes) - ;; Process one time-budgeted chunk (let [{:keys [thumbnails full text-font-state next-index]} (process-shapes-chunk shapes index thumbnails-acc full-acc text-font-state-acc)] - ;; Yield to browser, then continue with next chunk (-> (yield-to-browser) (p/then (fn [_] - (process-next-chunk next-index thumbnails full text-font-state))))) - ;; All chunks done - finalize - (do - (perf/end-measure "set-objects") + (process-next-client-chunk next-index thumbnails full text-font-state))))) + (finalize! thumbnails-acc full-acc text-font-state-acc shapes))) - ;; Notify that shapes are loaded and tiles rebuilt - (when on-shapes-ready (on-shapes-ready)) - (if-not (wasm/live?) - (do - (end-shapes-loading!) - (resolve nil)) - (do - ;; Show shapes immediately: end loading overlay + unblock rendering - (h/call wasm/internal-module "_end_loading") - (end-shapes-loading!) + (process-next-server-chunk [prepared index thumbnails-acc full-acc text-font-state-acc] + (if (< index (count prepared)) + (let [{:keys [thumbnails full text-font-state next-index]} + (process-host-attrs-chunk prepared index + thumbnails-acc full-acc text-font-state-acc)] + (-> (yield-to-browser) + (p/then (fn [_] + (process-next-server-chunk prepared next-index + thumbnails full text-font-state))))) + (finalize! thumbnails-acc full-acc text-font-state-acc prepared)))] - ;; Rebuild the tile index so _render knows which shapes - ;; map to which tiles after a page switch. - (h/call wasm/internal-module "_set_view_end") - (reset! view-interaction-active? false) - - ;; Text layouts must run after _end_loading (they - ;; depend on state that is only correct when loading - ;; is false). Each call touch_shape → touched_ids. - (let [text-ids (into [] (comp (filter cfh/text-shape?) (map :id)) shapes)] - (when (seq text-ids) - (update-text-layouts text-ids))) - (if render-callback - (render-callback) - (request-render "set-objects-complete")) - (ug/dispatch! (ug/event "penpot:wasm:set-objects")) - (resolve nil) - - ;; Kick off image fetches in the background. - ;; The promise is already resolved so these don't - ;; block the caller. - (let [pending-thumbnails (d/index-by :key :callback thumbnails-acc) - pending-full (d/index-by :key :callback full-acc)] - (when (or (seq pending-thumbnails) (seq pending-full)) - (->> (rx/concat - (->> (rx/from (vals pending-thumbnails)) - (rx/merge-map - (fn [callback] - (if (fn? callback) (callback) (rx/empty)))) - (rx/reduce conj [])) - (->> (rx/from (vals pending-full)) - (rx/mapcat - (fn [callback] - (if (fn? callback) (callback) (rx/empty)))) - (rx/reduce conj []))) - (rx/subs! - noop-fn - noop-fn - (fn [] - (relayout-after-fonts! shapes text-font-state-acc) - (request-render "images-loaded")))))))))))] - (process-next-chunk 0 [] [] empty-text-font-state)))))) + (-> (maybe-ingest-server-batch! shapes) + (p/then + (fn [{:keys [mode prepared]}] + (if (= mode :server) + (process-next-server-chunk prepared 0 [] [] empty-text-font-state) + (process-next-client-chunk 0 [] [] empty-text-font-state)))) + (p/catch + (fn [err] + (js/console.error "[wasm-batch] ingest failed, falling back" err) + (process-next-client-chunk 0 [] [] empty-text-font-state))))))))) ;; This is a version of process-pending that doesn't have sideffects @@ -1947,45 +2106,61 @@ (process-pending-no-sideffects thumbnails full set-objects-cb))) (defn- set-objects-sync - "Synchronously process all shapes (for small shape counts)." + "Process all shapes for a small page. Still async at the edges: server + batch ingest is a promise (prefetch), so callers must have + `shapes-loading?` set before the pool is cleared." [shapes render-callback on-shapes-ready] - (let [prepared (serialize-shape/serialize-shapes-batch! - shapes - {:include-layout? true - :include-fills-strokes? true}) - total-shapes (count prepared) - {:keys [thumbnails full text-font-state]} - (loop [index 0 - thumbnails-acc (transient []) - full-acc (transient []) - font-state-acc empty-text-font-state] - (if (< index total-shapes) - (let [shape (nth prepared index) - {:keys [thumbnails full font-face-keys pending-font-face-keys]} - (set-object-host-attrs shape true :skip-fills-strokes? true)] - (recur (inc index) - (reduce conj! thumbnails-acc thumbnails) - (reduce conj! full-acc full) - (acc-text-font-state font-state-acc - (:id shape) - font-face-keys - pending-font-face-keys))) - {:thumbnails (persistent! thumbnails-acc) - :full (persistent! full-acc) - :text-font-state font-state-acc}))] - (perf/end-measure "set-objects") - (when on-shapes-ready (on-shapes-ready)) - (when (wasm/live?) - ;; Rebuild the tile index so _render knows which shapes - ;; map to which tiles after a page switch. - (h/call wasm/internal-module "_set_view_end") - (reset! view-interaction-active? false) - (process-pending shapes thumbnails full text-font-state - (fn [] - (if render-callback - (render-callback) - (request-render "set-objects-sync-complete")) - (ug/dispatch! (ug/event "penpot:wasm:set-objects"))))))) + (-> (maybe-ingest-server-batch! shapes) + (p/then + (fn [{:keys [mode prepared]}] + (let [prepared (if (= mode :server) + prepared + (serialize-shape/serialize-shapes-batch! + shapes + {:include-layout? true + :include-fills-strokes? true})) + total-shapes (count prepared) + {:keys [thumbnails full text-font-state]} + (loop [index 0 + thumbnails-acc (transient []) + full-acc (transient []) + font-state-acc empty-text-font-state] + (if (< index total-shapes) + (let [shape (nth prepared index) + {:keys [thumbnails full font-face-keys pending-font-face-keys]} + (set-object-host-attrs shape true :skip-fills-strokes? true)] + (recur (inc index) + (reduce conj! thumbnails-acc thumbnails) + (reduce conj! full-acc full) + (acc-text-font-state font-state-acc + (:id shape) + font-face-keys + pending-font-face-keys))) + {:thumbnails (persistent! thumbnails-acc) + :full (persistent! full-acc) + :text-font-state font-state-acc}))] + (perf/end-measure "set-objects") + (when on-shapes-ready (on-shapes-ready)) + (if-not (wasm/live?) + (end-shapes-loading!) + (do + (h/call wasm/internal-module "_end_loading") + (end-shapes-loading!) + (h/call wasm/internal-module "_set_view_end") + (reset! view-interaction-active? false) + (process-pending prepared thumbnails full text-font-state + (fn [] + (if render-callback + (render-callback) + (request-render "set-objects-sync-complete")) + (ug/dispatch! (ug/event "penpot:wasm:set-objects"))))))))) + (p/catch + (fn [error] + (when (wasm/live?) + (h/call wasm/internal-module "_end_loading")) + (end-shapes-loading!) + (js/console.error "[wasm-batch] sync ingest failed" error)))) + nil) (defn- shapes-in-tree-order "Returns shapes sorted in tree order (parents before children). @@ -2035,26 +2210,25 @@ (perf/begin-measure "set-objects") (let [shapes (shapes-in-tree-order objects) total-shapes (count shapes)] + ;; Gate rAF while the pool may be empty or mid-ingest. Server batch + ;; ingest is always a promise (even for "sync" small pages), so both + ;; paths need this — otherwise page switches panic in `_render`. + (begin-shapes-loading!) + (h/call wasm/internal-module "_begin_loading") (if (or force-sync (< total-shapes ASYNC_THRESHOLD)) (set-objects-sync shapes render-callback on-shapes-ready) - (do - (begin-shapes-loading!) - (h/call wasm/internal-module "_begin_loading") - ;; NOTE: to render a loading overlay in the future - ;; (when-not on-shapes-ready - ;; (h/call wasm/internal-module "_render_loading_overlay")) - (try - (-> (set-objects-async shapes render-callback on-shapes-ready) - (p/catch (fn [error] - (h/call wasm/internal-module "_end_loading") - (end-shapes-loading!) - (js/console.error "Async WASM shape loading failed" error)))) - (catch :default error - (h/call wasm/internal-module "_end_loading") - (end-shapes-loading!) - (js/console.error "Async WASM shape loading failed" error) - (throw error))) - nil)))))) + (try + (-> (set-objects-async shapes render-callback on-shapes-ready) + (p/catch (fn [error] + (h/call wasm/internal-module "_end_loading") + (end-shapes-loading!) + (js/console.error "Async WASM shape loading failed" error)))) + (catch :default error + (h/call wasm/internal-module "_end_loading") + (end-shapes-loading!) + (js/console.error "Async WASM shape loading failed" error) + (throw error)))) + nil)))) (defn clear-focus-mode []