From 318d3d1120d656ad48bedba3bf422f7cf48429ce Mon Sep 17 00:00:00 2001 From: Elena Torro Date: Fri, 10 Jul 2026 13:14:20 +0200 Subject: [PATCH] :sparkles: Support multiple page PDF export over headless exporter --- backend/scripts/_env | 6 + exporter/deps.edn | 6 + exporter/src/app/config.cljs | 7 +- exporter/src/app/renderer.cljs | 28 +- exporter/src/app/renderer/wasm.cljs | 438 +++++++++++++++++++++++++++ exporter/src/app/wasm.cljs | 229 ++++++++++++++ exporter/src/app/wasm/gfonts.cljs | 45 +++ exporter/src/app/wasm/serialize.cljs | 57 ++++ exporter/src/app/wasm/text.cljs | 58 ++++ 9 files changed, 866 insertions(+), 8 deletions(-) create mode 100644 exporter/src/app/renderer/wasm.cljs create mode 100644 exporter/src/app/wasm.cljs create mode 100644 exporter/src/app/wasm/gfonts.cljs create mode 100644 exporter/src/app/wasm/serialize.cljs create mode 100644 exporter/src/app/wasm/text.cljs diff --git a/backend/scripts/_env b/backend/scripts/_env index 58ea02ede0..9c3ebb7c3f 100644 --- a/backend/scripts/_env +++ b/backend/scripts/_env @@ -72,6 +72,12 @@ export PENPOT_USER_FEEDBACK_DESTINATION="support@example.com" export PENPOT_NITRATE_BACKEND_URI=http://localhost:3000/admin-console +# Headless WASM export: render `:is-wasm` exports via the in-process Skia/WASM +# pipeline (no browser). Reads the render-wasm artifact from PENPOT_WASM_DIR, +# defaulting to ../frontend/resources/public/js (render-wasm.js/.wasm). +# export PENPOT_WASM_HEADLESS=true +# export PENPOT_WASM_DIR=../frontend/resources/public/js + export JAVA_OPTS="\ -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \ -Djdk.attach.allowAttachSelf \ diff --git a/exporter/deps.edn b/exporter/deps.edn index 08e3a3cdd8..87eac71c4c 100644 --- a/exporter/deps.edn +++ b/exporter/deps.edn @@ -1,6 +1,12 @@ {:paths ["src" "vendor" "resources" "test"] :deps {penpot/common {:local/root "../common"} + ;; The exporter reuses the *portable* render-wasm serialization leaves from + ;; the frontend source tree (app.render-wasm.{wasm,mem,helpers,serializers, + ;; api.shapes,...}), so the CLJS↔WASM binary protocol cannot drift from the + ;; editor's. shadow-cljs compiles only the required dependency graph, so the + ;; browser-coupled namespaces (api, api.webgl, api.fonts) are never pulled in. + penpot/frontend {:local/root "../frontend"} org.clojure/clojure {:mvn/version "1.12.2"} binaryage/devtools {:mvn/version "1.0.7"} metosin/reitit-core {:mvn/version "0.9.1"} diff --git a/exporter/src/app/config.cljs b/exporter/src/app/config.cljs index 127a28fa99..91d878589b 100644 --- a/exporter/src/app/config.cljs +++ b/exporter/src/app/config.cljs @@ -42,7 +42,12 @@ [:redis-uri {:optional true} :string] [:tempdir {:optional true} :string] [:browser-pool-max {:optional true} ::sm/int] - [:browser-pool-min {:optional true} ::sm/int]]) + [:browser-pool-min {:optional true} ::sm/int] + ;; Headless WASM export (no browser). When true, `:is-wasm` exports render + ;; via the in-process Skia/WASM pipeline. `:wasm-dir` points at the built + ;; render-wasm artifact (render-wasm.js/.wasm). + [:wasm-headless {:optional true} :boolean] + [:wasm-dir {:optional true} :string]]) (def ^:private decode-config (sm/decoder schema:config sm/string-transformer)) diff --git a/exporter/src/app/renderer.cljs b/exporter/src/app/renderer.cljs index 32555c6b19..ae54133dcd 100644 --- a/exporter/src/app/renderer.cljs +++ b/exporter/src/app/renderer.cljs @@ -7,10 +7,13 @@ (ns app.renderer "Common renderer interface." (:require + [app.common.logging :as l] [app.common.spec :as us] + [app.config :as cf] [app.renderer.bitmap :as rb] [app.renderer.pdf :as rp] [app.renderer.svg :as rs] + [app.renderer.wasm :as rw] [cljs.spec.alpha :as s])) (s/def ::name ::us/string) @@ -36,13 +39,24 @@ :opt-un [::is-wasm])) (defn render - [{:keys [type] :as params} on-object] + [{:keys [type is-wasm] :as params} on-object] (us/verify ::render-params params) (us/verify fn? on-object) - (case type - :png (rb/render params on-object) - :jpeg (rb/render params on-object) - :webp (rb/render params on-object) - :pdf (rp/render params on-object) - :svg (rs/render params on-object))) + ;; Opt-in headless path: when an export is flagged `:is-wasm` AND the + ;; `:wasm-headless` config is enabled, render with the in-process Skia/WASM + ;; pipeline (no browser). Off by default, so existing behavior is unchanged. + (let [headless? (and is-wasm (cf/get :wasm-headless))] + (l/info :hint "render" + :type type + :is-wasm (boolean is-wasm) + :wasm-headless (boolean (cf/get :wasm-headless)) + :backend (if headless? "wasm" "browser")) + (if headless? + (rw/render params on-object) + (case type + :png (rb/render params on-object) + :jpeg (rb/render params on-object) + :webp (rb/render params on-object) + :pdf (rp/render params on-object) + :svg (rs/render params on-object))))) diff --git a/exporter/src/app/renderer/wasm.cljs b/exporter/src/app/renderer/wasm.cljs new file mode 100644 index 0000000000..6d447ca9f3 --- /dev/null +++ b/exporter/src/app/renderer/wasm.cljs @@ -0,0 +1,438 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.renderer.wasm + "Headless renderer backend: renders exports with the render-wasm Skia + pipeline running in this Node process, with no browser and no WebGL. + + Drop-in alternative to `app.renderer.bitmap`/`pdf` (the Playwright + backends), selected from `app.renderer` when an export is flagged + headless. Pipeline per request: + + init module (once) -> fetch shape bundle (backend RPC) + -> serialize scene + -> provision resources (team/google fonts + fallback fonts + images, + enumerated by the host-agnostic app.render-wasm.{resources,fallback-fonts}) + -> relayout text (real font metrics) -> render each object -> tempfile + + NOTE (current slice): single shared WASM design state, so requests are + serialized one at a time (no pooling yet). PNG + PDF are wired; jpeg/webp + still need the PNG->format conversion that `bitmap` does." + (:require + ["node:fs" :as fs] + ["undici" :as http] + [app.common.geom.matrix] + [app.common.geom.point] + [app.common.geom.rect] + [app.common.types.fills.impl] + [app.common.types.objects-map] + [app.common.types.path.impl] + [app.common.types.shape] + [app.common.data :as d] + [app.common.logging :as l] + [app.common.transit :as t] + [app.common.uri :as u] + [app.common.uuid :as uuid] + [app.config :as cf] + [app.render-wasm.fallback-fonts :as fbf] + [app.render-wasm.resources :as resources] + [app.util.mime :as mime] + [app.util.shell :as sh] + [app.wasm :as wasm] + [app.wasm.gfonts :as gfonts] + [app.wasm.serialize :as serialize] + [cuerdas.core :as str] + [promesa.core :as p])) + +;; --- module lifecycle (one shared, lazily-initialized instance) + +(defonce ^:private module* (atom nil)) + +(defn- ensure-module! + [] + (or @module* + (reset! module* (wasm/init!)))) + +;; --- serialize access to the single shared WASM module --- +;; +;; There is one shared WASM instance (one design state, one global mem buffer), +;; but `handle-multiple-export` fans out partitions concurrently (`p/all`). This +;; promise queue runs module-touching work one task at a time so concurrent +;; exports can't interleave their serialize/render/alloc on the shared state. + +(defonce ^:private queue (atom (p/resolved nil))) + +(defn- enqueue! + "Runs `thunk` (0-arg, returns a promise) only after all previously enqueued + work has settled. Returns `thunk`'s promise. A task's failure is isolated: + it doesn't break the chain for the next task." + [thunk] + (let [result (p/handle @queue (fn [_ _] (thunk)))] + (reset! queue (p/handle result (fn [_ _] nil))) + result)) + +;; --- shape bundle fetch (backend RPC) + +(defn- fetch-objects + "Fetches the page's `objects` map from the backend via the `get-page` RPC, + using the same auth the exporter uses elsewhere (management key + bearer)." + [{:keys [file-id page-id share-id token]}] + (let [agent (new http/Agent #js {:connect #js {:rejectUnauthorized false}}) + headers #js {"Content-Type" "application/transit+json" + "X-Shared-Key" (str "exporter " cf/management-key) + "Authorization" (str "Bearer " token)} + ;; share-id is an OPTIONAL uuid on the backend; it must be omitted when + ;; absent, not sent as nil (nil fails the uuid schema). + body (t/encode-str (cond-> {:file-id file-id + :page-id page-id} + share-id (assoc :share-id share-id))) + uri (-> (cf/get :public-uri) + (u/ensure-path-slash) + (u/join "api/rpc/command/get-page") + (str))] + (l/info :hint "wasm render: get-page" + :uri uri + :file-id (str file-id) + :page-id (str page-id) + :token-len (count (str token))) + (->> (http/fetch uri #js {:method "POST" :headers headers :body body :dispatcher agent}) + (p/mcat (fn [^js resp] + (if (= 200 (.-status resp)) + (.text resp) + ;; Surface the backend's actual error body so we can tell + ;; auth (401) from bad params (400/404). + (->> (.text resp) + (p/mcat (fn [resp-body] + (l/error :hint "wasm render: get-page failed" + :status (.-status resp) + :body resp-body) + (p/rejected (ex-info "get-page failed" + {:status (.-status resp) + :body resp-body})))))))) + (p/fmap t/decode-str) + (p/fmap :objects)))) + +;; --- font resolution +;; +;; Custom (team) fonts: the exporter's text serializer keeps their real font +;; uuid, so `wasm/fonts-for-shape` reports it. We fetch the file's team font +;; variants once (get-font-variants RPC), map uuid+weight+style -> ttf asset id, +;; and download the TTF from `assets/by-id/`. Google/builtin fonts still fall +;; back to the bundled default (they need the gfonts catalog — a later slice). + +(defn- fetch-font-variants + "Fetches the file's team (custom) font variants via the get-font-variants RPC. + Returns a promise of the variant vector (or nil on failure — fonts then just + fall back, they don't fail the export)." + [{:keys [file-id share-id token]}] + (let [agent (new http/Agent #js {:connect #js {:rejectUnauthorized false}}) + headers #js {"Content-Type" "application/transit+json" + "X-Shared-Key" (str "exporter " cf/management-key) + "Authorization" (str "Bearer " token)} + body (t/encode-str (cond-> {:file-id file-id} + share-id (assoc :share-id share-id))) + uri (-> (cf/get :public-uri) + (u/ensure-path-slash) + (u/join "api/rpc/command/get-font-variants") + (str))] + (->> (http/fetch uri #js {:method "POST" :headers headers :body body :dispatcher agent}) + (p/mcat (fn [^js resp] + (if (= 200 (.-status resp)) + (.text resp) + (p/resolved nil)))) + (p/fmap (fn [s] (when s (t/decode-str s)))) + (p/merr (fn [cause] + (l/warn :hint "wasm render: get-font-variants failed" :cause cause) + (p/resolved nil)))))) + +(defn- fetch-asset-bytes + "Downloads a stored asset (font TTF) by id, returning a promise of an + ArrayBuffer (or nil)." + [asset-id {:keys [token]}] + (let [agent (new http/Agent #js {:connect #js {:rejectUnauthorized false}}) + ;; Cookie, not Bearer: /assets/* redirects to a presigned S3/minio URL, + ;; and a Bearer Authorization header makes S3 400 ("multiple + ;; authentication types"). + headers #js {"X-Shared-Key" (str "exporter " cf/management-key) + "Cookie" (str "auth-token=" token)} + uri (-> (cf/get :public-uri) + (u/ensure-path-slash) + (u/join (str "assets/by-id/" asset-id)) + (str))] + (->> (http/fetch uri #js {:method "GET" :headers headers :dispatcher agent}) + (p/mcat (fn [^js resp] + (if (= 200 (.-status resp)) + (.arrayBuffer resp) + (p/resolved nil)))) + (p/merr (fn [cause] + (l/warn :hint "wasm render: font asset fetch failed" + :asset-id (str asset-id) :cause cause) + (p/resolved nil)))))) + +(defn- gfont-proxy-url + "Rewrites a gstatic ttf url to the local gfonts proxy (mirrors the browser's + `google-font-ttf-url`)." + [ttf-url] + (let [proxy (-> (cf/get :public-uri) + (u/ensure-path-slash) + (u/join "internal/gfonts/font/") + (str))] + (str/replace ttf-url "https://fonts.gstatic.com/s/" proxy))) + +(defn- fetch-gfont-bytes + "Downloads a google font TTF through the local gfonts proxy." + [ttf-url] + (let [agent (new http/Agent #js {:connect #js {:rejectUnauthorized false}}) + uri (gfont-proxy-url ttf-url)] + (->> (http/fetch uri #js {:method "GET" :dispatcher agent}) + (p/mcat (fn [^js resp] + (if (= 200 (.-status resp)) + (.arrayBuffer resp) + (p/resolved nil)))) + (p/merr (fn [cause] + (l/warn :hint "wasm render: gfont fetch failed" :url uri :cause cause) + (p/resolved nil)))))) + +(defn- make-resolve-font + "Builds a `resolve-font` fn (family map -> promise of TTF bytes). Tries the + file's custom (team) font variants first — matching uuid + weight + style, + degrading to uuid+weight then uuid — then the shared google catalog. Returns + nil for fonts not found (builtin falls back to the bundled default)." + [variants params] + (fn [{:keys [id weight style]}] + (let [font-uuid (uuid/from-unsigned-parts (aget id 0) (aget id 1) (aget id 2) (aget id 3)) + style-str (if (zero? style) "normal" "italic") + variant (or (d/seek (fn [v] (and (= (:font-id v) font-uuid) + (= (:font-weight v) weight) + (= (name (:font-style v)) style-str))) + variants) + (d/seek (fn [v] (and (= (:font-id v) font-uuid) + (= (:font-weight v) weight))) + variants) + (d/seek (fn [v] (= (:font-id v) font-uuid)) variants))] + (if-let [ttf-id (:ttf-file-id variant)] + (fetch-asset-bytes ttf-id params) + (if-let [gurl (gfonts/resolve-ttf-url font-uuid weight style)] + (fetch-gfont-bytes gurl) + (p/resolved nil)))))) + +;; --- fallback fonts (emoji + per-script noto fonts) +;; +;; Emoji and non-latin scripts render through fallback families in the wasm +;; font store, not through any span's font family — so `wasm/fonts-for-shape` +;; never reports them and the per-object provisioning above never uploads +;; them. The browser workspace uploads them as a side effect of serializing +;; text (`add-emoji-font` / `add-noto-fonts`), which is why client-side single +;; exports show emoji/CJK while headless exports dropped them. Which fonts a +;; scene needs is computed by the host-agnostic +;; `app.render-wasm.fallback-fonts` (same data the browser uses). +;; `clear-fonts!` resets the store every request, so this must run per +;; request; the TTF bytes are cached per font for the process lifetime. + +(defn- scene-fallback-fonts + "Fallback font descriptors needed by the scene's text content." + [scene] + (let [texts (for [shape (vals scene) + :when (= :text (:type shape)) + node (or (some->> (:content shape) (tree-seq :children :children)) []) + :let [text (:text node)] + :when (string? text)] + text) + emoji? (boolean (some fbf/contains-emoji? texts)) + langs (reduce fbf/collect-used-languages #{} texts)] + (cond-> (fbf/add-noto-fonts [] langs) + emoji? (fbf/add-emoji-font)))) + +(defonce ^:private fallback-font-bytes* (atom {})) + +(defn- fetch-fallback-font-bytes + "Downloads (and caches for the process lifetime) one fallback font's TTF." + [{:keys [font-id weight style]}] + (if-let [bytes (get @fallback-font-bytes* font-id)] + (p/resolved bytes) + (let [font-uuid (gfonts/gfont-id->uuid font-id) + ttf-url (some-> font-uuid (gfonts/resolve-ttf-url weight style))] + (if ttf-url + (->> (fetch-gfont-bytes ttf-url) + (p/fmap (fn [buf] + (when buf (swap! fallback-font-bytes* assoc font-id buf)) + buf))) + (p/resolved nil))))) + +(defn- provision-fallback-fonts! + [scene] + (->> (scene-fallback-fonts scene) + (map (fn [{:keys [font-id weight style is-emoji is-fallback] :as font}] + (if-let [font-uuid (gfonts/gfont-id->uuid font-id)] + (->> (fetch-fallback-font-bytes font) + (p/fmap (fn [buf] + (if buf + (wasm/store-font! {:id (uuid/get-u32 font-uuid) + :weight weight + :style style + :emoji? (boolean is-emoji) + :fallback? (boolean is-fallback)} + buf) + (l/warn :hint "wasm render: fallback font unavailable" + :font-id font-id))))) + (p/resolved nil)))) + (p/all))) + +;; --- image resolution +;; +;; Image fills reference file-media ids. We collect them from the scene, fetch +;; the encoded bytes from `assets/by-file-media-id/`, and hand them to +;; `_store_image` (Skia decodes them; no WebGL). Keyed by media uuid, so this is +;; done once per request rather than per rendered object. + +(defn- fetch-file-media-bytes + "Downloads an image fill's encoded bytes by file-media id." + [media-id {:keys [token]}] + (let [agent (new http/Agent #js {:connect #js {:rejectUnauthorized false}}) + ;; Cookie, not Bearer: /assets/* redirects to a presigned S3/minio URL, + ;; and a Bearer Authorization header makes S3 400 ("multiple + ;; authentication types"). + headers #js {"X-Shared-Key" (str "exporter " cf/management-key) + "Cookie" (str "auth-token=" token)} + uri (-> (cf/get :public-uri) + (u/ensure-path-slash) + (u/join (str "assets/by-file-media-id/" media-id)) + (str))] + (->> (http/fetch uri #js {:method "GET" :headers headers :dispatcher agent}) + (p/mcat (fn [^js resp] + (if (= 200 (.-status resp)) + (.arrayBuffer resp) + (do + (l/warn :hint "wasm render: image fetch non-200" + :media-id (str media-id) + :uri uri + :status (.-status resp)) + (p/resolved nil))))) + (p/merr (fn [cause] + (l/warn :hint "wasm render: image fetch failed" + :media-id (str media-id) :uri uri :cause cause) + (p/resolved nil)))))) + +(defn- provision-images! + "Fetches and stores every image the scene references — shape fills, stroke + image fills, and text-span image fills (enumerated by the host-agnostic + `app.render-wasm.resources`, same source the workspace uses). Images the + module already holds are skipped: unlike fonts, the image store is not + reset per request, so repeated exports of the same file reuse them. + NOTE: that also means the store grows with every distinct image the + process ever exports; if exporter memory becomes a problem, add an + eviction policy on the Rust side rather than clearing per request." + [scene params] + (let [all-ids (resources/scene-image-ids scene) + new-ids (remove wasm/image-cached? all-ids)] + (l/info :hint "wasm render: provisioning images" + :total (count all-ids) + :cached (- (count all-ids) (count new-ids))) + (->> new-ids + (map (fn [image-id] + (->> (fetch-file-media-bytes image-id params) + (p/fmap (fn [buf] + (if buf + (do + (l/info :hint "wasm render: image stored" + :media-id (str image-id) + :bytes (.-byteLength ^js buf)) + (wasm/store-image! image-id buf)) + (l/warn :hint "wasm render: image unavailable" + :media-id (str image-id)))))))) + (p/all)))) + +(defn- relayout-text! + "Recomputes layout for every text shape in the scene. Called after fonts are + provisioned so metrics use the real fonts (serialize-time layout used the + fallback)." + [scene] + (doseq [shape (vals scene) + :when (= :text (:type shape))] + (wasm/update-text-layout! (:id shape)))) + +;; --- render + +(defn- render-object-bytes + [type id scale] + (case type + :pdf (let [bytes (wasm/render-shape-pdf id scale)] + (l/info :hint "PDF generated via Skia (render-wasm headless)" + :object-id (str id) + :backend "skia-wasm" + :bytes (.-length bytes)) + bytes) + (wasm/render-shape-raster id scale))) + +(defn- render* + [{:keys [scale type objects] :as params} on-object] + (l/info :hint "wasm render: start" :type type :scale scale :objects (count objects)) + (->> (ensure-module!) + (p/mcat (fn [_] + (l/info :hint "wasm render: module ready, fetching scene" + :file-id (str (:file-id params)) :page-id (str (:page-id params))) + (fetch-objects params))) + (p/mcat (fn [scene] + (let [sample (first (vals scene))] + (l/info :hint "wasm render: scene fetched" + :shapes (count scene) + :map? (map? scene) + :first-key (str (first (keys scene))) + :sample-id (str (:id sample)) + :sample-type (str (:type sample)) + :sample-keys (pr-str (when (map? sample) (vec (keys sample)))))) + (serialize/serialize-scene! scene) + (l/info :hint "wasm render: scene serialized") + ;; Reset the shared module's font store so fonts from a previous + ;; request don't accumulate / leak into this one. + (wasm/clear-fonts!) + ;; Fetch the file's custom (team) font variants once, provision + ;; every referenced image once, then resolve/provision fonts per + ;; rendered object. + (->> (p/all [(fetch-font-variants params) + (provision-images! scene params) + (provision-fallback-fonts! scene)]) + (p/mcat + (fn [[variants _]] + (let [resolve-font (make-resolve-font (or variants []) params)] + ;; Provision every object's fonts BEFORE rendering, so + ;; the text relayout below sees real font metrics. + (p/all (map (fn [{:keys [id]}] + (wasm/provision-fonts! id resolve-font)) + objects))))) + (p/mcat + (fn [_] + ;; Serialize-time layout used the fallback font (fonts + ;; weren't uploaded yet); recompute now that they are, or + ;; text metrics/line breaks are wrong. + (relayout-text! scene) + (p/run + (fn [{:keys [id] :as object}] + (let [bytes (render-object-bytes type id scale) + path (sh/tempfile :prefix "penpot.tmp.wasm." + :suffix (mime/get-extension type))] + (l/info :hint "wasm render: object rendered" + :object-id (str id) :bytes (.-length bytes)) + (fs/writeFileSync path bytes) + ;; `on-object` may return a plain value (zip append + ;; returns the archiver instance) or a promise (single + ;; export's file move); `p/do` normalizes both to a + ;; thenable so `p/mcat` doesn't throw "expected thenable". + (p/do (on-object (assoc object :path path))))) + objects)))))) + (p/merr (fn [cause] + (l/error :hint "wasm render: failed" :cause cause) + ;; A panic/abort can leave the shared module's buffer allocated + ;; or the wasm instance aborted; drop it so the next request + ;; rebuilds a fresh module instead of inheriting the bad state. + (reset! module* nil) + (p/rejected cause))))) + +(defn render + "Public entry. Serializes module access through `enqueue!` so concurrent + exports (multi-partition zip) run one at a time on the shared WASM instance." + [params on-object] + (enqueue! (fn [] (render* params on-object)))) diff --git a/exporter/src/app/wasm.cljs b/exporter/src/app/wasm.cljs new file mode 100644 index 0000000000..17b9b7c680 --- /dev/null +++ b/exporter/src/app/wasm.cljs @@ -0,0 +1,229 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.wasm + "Headless driver for the render-wasm module under Node. + + This is the GPU-free counterpart of the browser's + `app.render-wasm.api`: it loads the emscripten artifact in the Node + process (no WebGL), boots it via `init_headless`, and exposes the + minimal surface the exporter needs — font provisioning and shape + rendering to PNG/PDF bytes. + + The shape *serialization* is reused from the portable render-wasm leaves + (`app.render-wasm.{wasm,mem,helpers,serializers,api.shapes,...}`); this + namespace only owns the Node runtime + the headless render/export calls. + The CLJS↔WASM binary protocol therefore stays identical to the editor — + no second implementation, no drift. + + Requires render-wasm built with `-sENVIRONMENT=web,node`." + (:require + ["node:fs" :as fs] + ["node:path" :as path] + [app.common.logging :as l] + [app.common.uuid :as uuid] + [app.config :as cf] + [app.render-wasm.helpers :as h] + [app.render-wasm.mem :as mem] + [app.render-wasm.wasm :as wasm] + [promesa.core :as p] + [shadow.esm :refer [dynamic-import]])) + +(def ^:private default-viewport-width 1920) +(def ^:private default-viewport-height 1080) + +;; render_shape_raster / render_shape_pixels result header: [len u32][w u32][h u32]. +(def ^:private RASTER-HEADER-BYTES 12) +;; get_fonts_for_shape entry: [uuid 16 bytes][weight u32][style u32]. +(def ^:private FONT-ENTRY-BYTES 24) + +(defn- artifact-dir + [] + (cf/get :wasm-dir "../frontend/resources/public/js")) + +(defn- read-result-bytes + "Reads `len` bytes from the WASM heap starting at `offset`, copying them out + (via `.slice`) before the buffer is freed." + [offset len] + (.slice (mem/get-heap-u8) offset (+ offset len))) + +;; --- MODULE LIFECYCLE + +(defn init! + "Loads the render-wasm artifact under Node and boots it headless. Sets the + shared `wasm/internal-module` so the portable serialization leaves work. + Idempotent-ish: callers should hold the returned module." + ([] (init! default-viewport-width default-viewport-height)) + ([width height] + (let [dir (artifact-dir) + js-path (path/resolve dir "render-wasm.js") + wasm-path (path/resolve dir "render-wasm.wasm") + wasm-bytes (fs/readFileSync wasm-path)] + (l/info :hint "loading render-wasm (headless)" :js js-path) + ;; shadow-cljs :esm — use its dynamic-import helper (raw `js/import` + ;; compiles to an undefined `import$`). + (->> (dynamic-import (str "file://" js-path)) + (p/mcat + (fn [mod] + (let [factory (unchecked-get mod "default")] + (factory + #js {;; Bypass the web fetch loader: instantiate from local bytes. + :instantiateWasm + (fn [imports success] + (-> (js/WebAssembly.instantiate wasm-bytes imports) + (.then (fn [result] (success (.-instance result))))) + #js {}) + :locateFile (fn [p] (path/resolve dir p)) + :printErr (fn [s] (l/warn :wasm s))})))) + (p/fmap + (fn [module] + (set! wasm/internal-module module) + (h/call module "_init_headless" width height) + (set! wasm/context-initialized? true) + (l/info :hint "render-wasm headless module ready" :width width :height height) + module)))))) + +;; --- FONT PROVISIONING (on demand, mirrors the browser) + +(defn fonts-for-shape + "Returns the distinct font families needed to render the subtree rooted at + `shape-id` as a vector of {:id :weight :style}. Equivalent to + the browser's `get-content-fonts`, but read from the loaded WASM tree." + [shape-id] + (let [module wasm/internal-module + buf (uuid/get-u32 shape-id) ;; resolved from app.render-wasm leaves + offset (h/call module "_get_fonts_for_shape" + (aget buf 0) (aget buf 1) (aget buf 2) (aget buf 3)) + heap32 (mem/get-heap-u32) + n (aget heap32 (mem/->offset-32 offset))] + (let [entries + (vec + (for [i (range n)] + (let [base (+ offset 4 (* i FONT-ENTRY-BYTES)) + u32 (fn [o] (aget heap32 (mem/->offset-32 (+ base o))))] + {:id #js [(u32 0) (u32 4) (u32 8) (u32 12)] + :weight (u32 16) + :style (u32 20)})))] + (mem/free) + entries))) + +(defn store-font! + "Uploads one font's TTF bytes into the WASM font store, keyed by the family + (uuid quartet + weight + style). `font-bytes` is a Uint8Array/Buffer." + [{:keys [id weight style emoji? fallback?]} font-bytes] + (let [module wasm/internal-module + size (.-byteLength font-bytes) + ptr (h/call module "_alloc_bytes" size) + heap (mem/get-heap-u8)] + (.set heap (js/Uint8Array. font-bytes) ptr) + (h/call module "_store_font" + (aget id 0) (aget id 1) (aget id 2) (aget id 3) + weight style (boolean emoji?) (boolean fallback?)))) + +(defn clear-fonts! + "Resets the WASM font store. Must be called once per render request because + the shared module would otherwise accumulate fonts across requests." + [] + (h/call wasm/internal-module "_clear_fonts")) + +(defn update-text-layout! + "Recomputes a text shape's layout with the currently provisioned fonts. Text is + laid out at serialize time using the fallback font (real fonts aren't uploaded + yet), so this must run again after `provision-fonts!` or glyph metrics/line + breaks are wrong." + [shape-id] + (let [buf (uuid/get-u32 shape-id)] + (h/call wasm/internal-module "_update_shape_text_layout_for" + (aget buf 0) (aget buf 1) (aget buf 2) (aget buf 3)))) + +(defn image-cached? + "True when the module's image store already holds this image (full size). + The store is NOT reset between requests, so previously provisioned images + can be reused instead of refetched." + [image-id] + (let [buf (uuid/get-u32 image-id)] + (not (zero? (h/call wasm/internal-module "_is_image_cached" + (aget buf 0) (aget buf 1) (aget buf 2) (aget buf 3) + false))))) + +(defn store-image! + "Uploads one image's *encoded* bytes (PNG/JPEG — Skia decodes, no WebGL) into + the WASM image store via `_store_image`. Buffer layout matches the Rust reader: + [shape uuid 16][image uuid 16][is_thumbnail u32][encoded bytes]. Images are + keyed by image uuid, so the shape uuid is left zero. `image-bytes` is an + ArrayBuffer/Buffer/Uint8Array." + [image-id image-bytes] + (let [module wasm/internal-module + img-u8 (js/Uint8Array. image-bytes) + size (.-byteLength img-u8) + total (+ 36 size) + ptr (h/call module "_alloc_bytes" total) + heap (mem/get-heap-u8) + dview (js/DataView. (.-buffer heap)) + quart (uuid/get-u32 image-id)] + ;; shape uuid [0..16) = 0 (images are keyed by image uuid only) + (.setUint32 dview (+ ptr 0) 0 true) + (.setUint32 dview (+ ptr 4) 0 true) + (.setUint32 dview (+ ptr 8) 0 true) + (.setUint32 dview (+ ptr 12) 0 true) + ;; image uuid [16..32) — 4 LE u32 (matches common `buffer/write-uuid`, which + ;; the fill path uses, so it hashes to the same key the fill references) + (.setUint32 dview (+ ptr 16) (aget quart 0) true) + (.setUint32 dview (+ ptr 20) (aget quart 1) true) + (.setUint32 dview (+ ptr 24) (aget quart 2) true) + (.setUint32 dview (+ ptr 28) (aget quart 3) true) + ;; is_thumbnail [32..36) = 0 + (.setUint32 dview (+ ptr 32) 0 true) + ;; encoded bytes [36..) + (.set heap img-u8 (+ ptr 36)) + (h/call module "_store_image"))) + +(defn provision-fonts! + "Resolves and uploads every font needed by `shape-id`. `resolve-font` is an + injected fn of the family map -> promise of TTF bytes (or nil to skip). This + keeps the font *source* (gfonts proxy / custom assets / backend) out of the + driver." + [shape-id resolve-font] + (->> (fonts-for-shape shape-id) + (map (fn [family] + (->> (resolve-font family) + (p/fmap (fn [bytes] (when bytes (store-font! family bytes))))))) + (p/all))) + +;; --- RENDER + +(defn- render-call + [fn-name shape-id scale] + (let [module wasm/internal-module + buf (uuid/get-u32 shape-id) + offset (h/call module fn-name + (aget buf 0) (aget buf 1) (aget buf 2) (aget buf 3) + scale) + heap32 (mem/get-heap-u32) + len (aget heap32 (mem/->offset-32 offset)) + bytes (read-result-bytes (+ offset RASTER-HEADER-BYTES) len)] + (mem/free) + bytes)) + +(defn render-shape-raster + "Renders the shape subtree to PNG bytes (Uint8Array) on a CPU surface." + [shape-id scale] + (render-call "_render_shape_raster" shape-id scale)) + +(defn render-shape-pdf + "Renders the shape subtree to PDF bytes (Uint8Array)." + [shape-id scale] + ;; PDF result header is [len u32] only (no w/h); handled separately. + (let [module wasm/internal-module + buf (uuid/get-u32 shape-id) + offset (h/call module "_render_shape_pdf" + (aget buf 0) (aget buf 1) (aget buf 2) (aget buf 3) + scale) + heap32 (mem/get-heap-u32) + len (aget heap32 (mem/->offset-32 offset)) + bytes (read-result-bytes (+ offset 4) len)] + (mem/free) + bytes)) diff --git a/exporter/src/app/wasm/gfonts.cljs b/exporter/src/app/wasm/gfonts.cljs new file mode 100644 index 0000000000..1f3ae3ce0d --- /dev/null +++ b/exporter/src/app/wasm/gfonts.cljs @@ -0,0 +1,45 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.wasm.gfonts + "Compile-time Google Fonts catalog for the headless exporter. + + Reuses the same `gfonts.json` + parser the browser uses, via the portable + `preload-gfonts` macro (a `.clj` macro — it does NOT pull the browser-coupled + `app.main.fonts` cljs runtime). `parse-gfont` assigns each font a `uuid/random` + at macro-expansion time, so the catalog MUST be defined once here and shared: + text serialization (`app.wasm.text`) maps `gfont-` -> this uuid, and font + provisioning (`app.renderer.wasm`) maps the same uuid back to the variant's + ttf url. Consistency only holds because both read this single instance." + (:require-macros [app.main.fonts :refer [preload-gfonts]])) + +(def ^:private catalog + (preload-gfonts "fonts/gfonts.2025.11.28.json")) + +(def ^:private by-id + (reduce (fn [m font] (assoc m (:id font) font)) {} catalog)) + +(def ^:private by-uuid + (reduce (fn [m font] (assoc m (:uuid font) font)) {} catalog)) + +(defn gfont-id->uuid + "Maps a `gfont-` id to its (build-stable) catalog uuid, or nil." + [gfont-id] + (:uuid (get by-id gfont-id))) + +(defn resolve-ttf-url + "Given a google font uuid + numeric weight + style int (0 = normal, else + italic), returns the variant's gstatic ttf url (or nil if not a google font / + no variants). Degrades weight+style -> weight -> first variant." + [font-uuid weight style] + (when-let [font (get by-uuid font-uuid)] + (let [w (str weight) + s (if (zero? style) "normal" "italic") + variants (:variants font) + variant (or (some (fn [v] (when (and (= (:weight v) w) (= (:style v) s)) v)) variants) + (some (fn [v] (when (= (:weight v) w) v)) variants) + (first variants))] + (:ttf-url variant)))) diff --git a/exporter/src/app/wasm/serialize.cljs b/exporter/src/app/wasm/serialize.cljs new file mode 100644 index 0000000000..85c171a9ca --- /dev/null +++ b/exporter/src/app/wasm/serialize.cljs @@ -0,0 +1,57 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.wasm.serialize + "Browser-free shape serialization for the headless exporter. + + This is the headless counterpart of the browser orchestrator + `app.render-wasm.api/set-object`. It cannot reuse that fn directly (its + namespace pulls React/DOM/store), so it owns the *call sequencing* here — + but every byte layout is reused from the shared, identical sources: + + - `app.render-wasm.api.shapes/set-shape-base-props` (the scalar struct), + - `app.common.types.fills` / `…fills.impl` (fills & stroke fills), + - `app.render-wasm.{mem,serializers,helpers}` leaves. + + So the bytes sent to WASM are the same as the editor's; only the (simple) + orchestration lives here. + + COVERAGE: base props (incl. corners/opacity/transform/clip/constraints), + children, fills (solid/gradient/image), strokes (solid/gradient/image), + layer + background blur, shadows, masks, path/bool geometry, text. Image + BYTES and fonts are provisioned separately by `app.renderer.wasm`. + NOT YET: svg-raw" + (:require + [app.render-wasm.api.props :as props] + [app.render-wasm.helpers :as h] + [app.render-wasm.serialize-shape :as serialize-shape] + [app.render-wasm.wasm :as wasm] + [app.wasm.text :as text])) + +(defn set-shape! + "Serializes a single shape into the WASM design state. The host-independent + properties (base props, children, blur, shadows, svg-attrs, mask, bool-type, + path geometry, grow-type) go through the shared `serialize-shape!` — the same + code the workspace's `set-object` uses, so the two can't drift. Only the + host-specific parts are handled here: fills/strokes (image bytes are provisioned + separately) and text content (fonts provisioned separately)." + [shape] + (let [type (get shape :type)] + (serialize-shape/serialize-shape! shape) + (props/write-shape-fills! (get shape :fills)) + (when-not (= type :group) + (props/write-shape-strokes! (get shape :strokes))) + (when (= type :text) + (text/set-shape-text! (get shape :content))))) + +(defn serialize-scene! + "Loads every shape of an `objects` map into the WASM design state. Resets the + shapes pool first so repeated exports don't accumulate into the shared + state. Order is irrelevant: shapes reference each other by id and the tree + is resolved at render time." + [objects] + (h/call wasm/internal-module "_init_shapes_pool" (count objects)) + (run! set-shape! (vals objects))) diff --git a/exporter/src/app/wasm/text.cljs b/exporter/src/app/wasm/text.cljs new file mode 100644 index 0000000000..223784c711 --- /dev/null +++ b/exporter/src/app/wasm/text.cljs @@ -0,0 +1,58 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.wasm.text + "Browser-free text-content serialization for the headless exporter. + + The binary text layout is shared with the workspace via + `app.render-wasm.text-content`; only *font-id resolution* is local here, + because the exporter has no fonts DB: custom fonts keep their uuid + (`custom-`), google fonts (`gfont-`) map via the compile-time + `app.wasm.gfonts` catalog, and builtin falls back to the default (uuid/zero)." + (:require + [app.common.uuid :as uuid] + [app.render-wasm.helpers :as h] + [app.render-wasm.serializers :as sr] + [app.render-wasm.text-content :as tc] + [app.render-wasm.wasm :as wasm] + [app.wasm.gfonts :as gfonts] + [cuerdas.core :as str])) + +(defn- normalize-font-id + "Maps a content font-id to its wasm uuid. The provisioning side keys on the + same uuid." + [font-id] + (try + (cond + (str/starts-with? font-id "gfont-") + (or (gfonts/gfont-id->uuid font-id) uuid/zero) + + (str/includes? font-id "-") + (let [no-prefix (subs font-id (inc (str/index-of font-id "-")))] + (if (str/blank? no-prefix) uuid/zero (uuid/parse no-prefix))) + + :else uuid/zero) + (catch :default _ uuid/zero))) + +(defn set-shape-text! + "Serializes a text shape's content into the current WASM shape. Mirrors the + editor's sequence: clear -> vertical-align -> append each paragraph -> layout. + Byte writing is the shared `text-content/write-shape-text!`; only font-id + resolution is injected." + [content] + (when content + (h/call wasm/internal-module "_clear_shape_text") + (h/call wasm/internal-module "_set_shape_vertical_align" + (sr/translate-vertical-align (get content :vertical-align))) + (let [paragraph-set (first (get content :children)) + paragraphs (get paragraph-set :children)] + (doseq [paragraph paragraphs] + (let [spans (get paragraph :children)] + (when (seq spans) + (let [text (apply str (map :text spans))] + (tc/write-shape-text! spans paragraph text + {:normalize-font-id normalize-font-id})))))) + (h/call wasm/internal-module "_update_shape_text_layout")))