diff --git a/backend/scripts/_env b/backend/scripts/_env index 9b33950016..53272b4c4e 100644 --- a/backend/scripts/_env +++ b/backend/scripts/_env @@ -81,9 +81,15 @@ export PENPOT_INTERNAL_URI=http://localhost:3450 # defaulting to ../frontend/resources/public/js (render-wasm.js/.wasm) — which # is where render-wasm/build leaves it in devenv. The docker image ships the # artifact inside the exporter bundle and points PENPOT_WASM_DIR at it. -# export PENPOT_WASM_HEADLESS=true +export PENPOT_WASM_HEADLESS=true # export PENPOT_WASM_DIR=../frontend/resources/public/js +# Per worker, so the process can hold up to POOL_SIZE times this. # export PENPOT_WASM_IMAGE_CACHE_MB=256 +# Render worker threads. Each owns a wasm module, so this costs memory, not +# just cpu. Renders are synchronous, so they run off the event loop. +# export PENPOT_WASM_POOL_SIZE=2 +# Milliseconds without progress before a worker is treated as wedged. +# export PENPOT_WASM_RENDER_TIMEOUT=120000 export JAVA_OPTS="\ -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \ diff --git a/exporter/package.json b/exporter/package.json index fce5a1fad9..4b5ca6caf7 100644 --- a/exporter/package.json +++ b/exporter/package.json @@ -30,9 +30,9 @@ }, "scripts": { "clear:shadow-cache": "rm -rf .shadow-cljs && rm -rf target", - "watch:app": "pnpm run clear:shadow-cache && clojure -M:dev:shadow-cljs watch main", + "watch:app": "pnpm run clear:shadow-cache && clojure -M:dev:shadow-cljs watch main wasm-worker", "watch": "pnpm run watch:app", - "build:app": "clojure -M:dev:shadow-cljs release main", + "build:app": "clojure -M:dev:shadow-cljs release main wasm-worker", "build": "pnpm run clear:shadow-cache && pnpm run build:app", "fmt": "cljfmt fix --parallel=true src/", "check-fmt": "cljfmt check --parallel=true src/", diff --git a/exporter/scripts/build b/exporter/scripts/build index 7b2e4f7a63..7bf1f0d33e 100755 --- a/exporter/scripts/build +++ b/exporter/scripts/build @@ -40,4 +40,4 @@ EOF chmod +x target/setup; -sed -i -re "s/\%version\%/$CURRENT_VERSION/g" ./target/app.js; +sed -i -re "s/\%version\%/$CURRENT_VERSION/g" ./target/app.js ./target/worker/wasm-worker.js; diff --git a/exporter/shadow-cljs.edn b/exporter/shadow-cljs.edn index ae963cf311..aacc1a65d5 100644 --- a/exporter/shadow-cljs.edn +++ b/exporter/shadow-cljs.edn @@ -23,6 +23,39 @@ {:output-feature-set :es2020 :output-wrapper false} + :release + {:compiler-options + {:fn-invoke-direct true + :source-map true + :optimizations #shadow/env ["PENPOT_BUILD_OPTIMIZATIONS" :as :keyword :default :simple] + :pseudo-names true + :pretty-print true + :anon-fn-naming-policy :off + :source-map-detail-level :all}}} + + ;; Headless render worker (see app.wasm.pool). A separate build rather than a + ;; second module of :main, so the bundle node loads through `new Worker(path)` + ;; is self-contained — code splitting would leave it importing shared chunks + ;; relative to a different base. Its own :output-dir for the same reason. + :wasm-worker + {:target :esm + :runtime :node + :output-dir "target/worker/" + + :modules + {:wasm-worker + {:entries [] + :init-fn app.wasm.worker/main}} + + :js-options + {:entry-keys ["module" "browser" "main"] + :export-conditions ["module" "import", "browser" "require" "default"] + :js-provider :import} + + :compiler-options + {:output-feature-set :es2020 + :output-wrapper false} + :release {:compiler-options {:fn-invoke-direct true diff --git a/exporter/src/app/config.cljs b/exporter/src/app/config.cljs index 9d5b9a9fff..dfd153c642 100644 --- a/exporter/src/app/config.cljs +++ b/exporter/src/app/config.cljs @@ -49,8 +49,18 @@ [:wasm-headless {:optional true} :boolean] [:wasm-dir {:optional true} :string] ;; Byte budget (in MB) for the WASM image cache; least-recently-used - ;; images are evicted between requests once the store exceeds it. - [:wasm-image-cache-mb {:optional true} ::sm/int]]) + ;; images are evicted between requests once the store exceeds it. Applies + ;; per worker, so the process can hold up to pool-size times this. + [:wasm-image-cache-mb {:optional true} ::sm/int] + ;; Number of render worker threads. Each owns a full wasm module, so this + ;; trades memory for concurrency. + [:wasm-pool-size {:optional true} ::sm/int] + ;; Milliseconds a worker may go without reporting progress before it is + ;; considered wedged, terminated and replaced. + [:wasm-render-timeout {:optional true} ::sm/int] + ;; Escape hatch for the compiled worker bundle path; normally derived from + ;; the running script. + [:wasm-worker-script {:optional true} :string]]) (def ^:private decode-config (sm/decoder schema:config sm/string-transformer)) diff --git a/exporter/src/app/core.cljs b/exporter/src/app/core.cljs index 35ebd830bd..80ad7b9018 100644 --- a/exporter/src/app/core.cljs +++ b/exporter/src/app/core.cljs @@ -13,6 +13,7 @@ [app.http :as http] [app.redis :as redis] [app.wasm :as wasm] + [app.wasm.pool :as wasm.pool] [promesa.core :as p])) (enable-console-print!) @@ -25,11 +26,10 @@ :internal-uri (str (cf/get-internal-uri)) :version (:full cf/version)) (when (cf/get :wasm-headless) - (l/warn :msg "headless wasm export enabled (experimental)" - :hint (str "renders run in-process on a single shared wasm module " - "and are serialized one at a time, so exports do not " - "run concurrently; not recommended for busy instances") + (l/info :msg "headless wasm export enabled" + :hint "each worker holds its own wasm module and image cache" :wasm-dir (wasm/artifact-dir) + :workers (wasm.pool/pool-size) :image-cache-mb (wasm/image-cache-mb))) (p/do! (bwr/init) @@ -48,6 +48,7 @@ (bwr/stop) (redis/stop) (http/stop) + (wasm.pool/stop) (done))) (.on proc/default "uncaughtException" diff --git a/exporter/src/app/handlers/export_frames.cljs b/exporter/src/app/handlers/export_frames.cljs index cf263d8e00..faf8deea63 100644 --- a/exporter/src/app/handlers/export_frames.cljs +++ b/exporter/src/app/handlers/export_frames.cljs @@ -85,12 +85,16 @@ :cause (ex-message cause)}] (redis/pub! topic data))) + ;; Keyed by `:index` (the frame's position in the request), NOT append + ;; order: partitions render concurrently, so objects arrive finished in + ;; whatever order they complete. Appending here would hand `pdfunite` + ;; its pages shuffled. result-cache - (atom []) + (atom {}) on-object - (fn [{:keys [path] :as object}] - (let [res (swap! result-cache conj path)] + (fn [{:keys [path index]}] + (let [res (swap! result-cache assoc index path)] (on-progress (count res)))) procs @@ -98,7 +102,7 @@ (map #(rd/render (assoc % :is-wasm is-wasm) on-object)))] (->> (p/all procs) - (p/fmap (fn [] @result-cache)) + (p/fmap (fn [] (->> @result-cache (sort-by key) (mapv val)))) (p/mcat (partial join-pdf file-id)) (p/mcat (partial move-file resource)) (p/fmap (constantly resource)) diff --git a/exporter/src/app/handlers/export_shapes.cljs b/exporter/src/app/handlers/export_shapes.cljs index 6213281453..9ee6e8eb02 100644 --- a/exporter/src/app/handlers/export_shapes.cljs +++ b/exporter/src/app/handlers/export_shapes.cljs @@ -180,11 +180,17 @@ {:id (:object-id entry) :filename (:filename entry) :name (:name entry) - :suffix (:suffix entry)})] + :suffix (:suffix entry) + :index (:index entry)})] + ;; `:index` is the object's position in the *requested* order. Renderers + ;; hand objects back as they finish, and partitions run concurrently, so + ;; completion order is not submission order — anything order-sensitive + ;; (`export-frames` joining a multi-page pdf) has to sort by this. (let [xform (comp (map #(assoc % :token token)) - (assoc-file-name))] + (assoc-file-name) + (map-indexed (fn [index params] (assoc params :index index))))] (->> (sequence xform exports) (d/group-by (juxt :scale :type)) (map second) diff --git a/exporter/src/app/renderer/wasm.cljs b/exporter/src/app/renderer/wasm.cljs index f4594d8bd2..f2319efb18 100644 --- a/exporter/src/app/renderer/wasm.cljs +++ b/exporter/src/app/renderer/wasm.cljs @@ -6,456 +6,20 @@ (ns app.renderer.wasm "Headless renderer backend: renders exports with the render-wasm Skia - pipeline running in this Node process, with no browser and no WebGL. + pipeline, with no browser and no WebGL. Drop-in alternative to `app.renderer.bitmap`/`pdf` (the Playwright - backends), selected from `app.renderer` when an export is flagged - headless. Pipeline per request: + backends), selected from `app.renderer` when an export is flagged headless. + Handles png/jpeg/webp (Skia encodes all three natively) and pdf; `:svg` is + routed to the browser backend by `app.renderer` — it needs vector markup, + not a raster. - init module (once) -> fetch shape bundle (backend RPC) - -> serialize scene - -> provision resources (team/google fonts + fallback fonts + images, - enumerated by the host-agnostic app.render-wasm.{resources,fallback-fonts}) - -> relayout text (real font metrics) -> render each object -> tempfile - - Handles png/jpeg/webp (Skia encodes all three natively) and pdf. `:svg` is - routed to the browser backend by `app.renderer` — it needs vector markup, not - a raster. - - NOTE (current slice): single shared WASM design state, so requests are - serialized one at a time (no pooling yet)." + Only the entry point lives here. The render itself is synchronous wasm work, + so it runs in a worker thread rather than on the event loop: `app.wasm.pool` + schedules it and `app.wasm.render` is the pipeline." (:require - ["node:fs" :as fs] - ["undici" :as http] - [app.common.data :as d] - ;; Required for side effects: these register the transit read handlers and - ;; deftype impls the `get-page` response is decoded into. - [app.common.geom.matrix] - [app.common.geom.point] - [app.common.geom.rect] - [app.common.logging :as l] - [app.common.transit :as t] - [app.common.types.fills.impl] - [app.common.types.objects-map] - [app.common.types.path.impl] - [app.common.types.shape] - [app.common.uri :as u] - [app.common.uuid :as uuid] - [app.config :as cf] - [app.render-wasm.fallback-fonts :as fbf] - [app.render-wasm.resources :as resources] - [app.util.mime :as mime] - [app.util.shell :as sh] - [app.wasm :as wasm] - [app.wasm.gfonts :as gfonts] - [app.wasm.serialize :as serialize] - [cuerdas.core :as str] - [promesa.core :as p])) - -;; --- module lifecycle (one shared, lazily-initialized instance) - -(defonce ^:private module* (atom nil)) - -(defn- ensure-module! - [] - (or @module* - (reset! module* (wasm/init!)))) - -;; --- serialize access to the single shared WASM module --- -;; -;; There is one shared WASM instance (one design state, one global mem buffer), -;; but `handle-multiple-export` fans out partitions concurrently (`p/all`). This -;; promise queue runs module-touching work one task at a time so concurrent -;; exports can't interleave their serialize/render/alloc on the shared state. - -(defonce ^:private queue (atom (p/resolved nil))) - -(defn- enqueue! - "Runs `thunk` (0-arg, returns a promise) only after all previously enqueued - work has settled. Returns `thunk`'s promise. A task's failure is isolated: - it doesn't break the chain for the next task." - [thunk] - (let [result (p/handle @queue (fn [_ _] (thunk)))] - (reset! queue (p/handle result (fn [_ _] nil))) - result)) - -;; --- backend endpoints -;; -;; Every fetch below targets the *internal* endpoint (`internal-uri`, falling -;; back to `public-uri`), same as the Playwright backends and -;; `handlers.resources`: in a real deployment the exporter reaches the backend -;; over the container network, not through the public ingress. - -(defn- internal-uri - "Absolute URI for `path` on the internal (backend) endpoint." - [path] - (-> (cf/get-internal-uri) - (u/ensure-path-slash) - (u/join path) - (str))) - -(defn- rpc-headers - "Auth headers for backend RPC calls (management key + bearer)." - [token] - #js {"Content-Type" "application/transit+json" - "X-Shared-Key" (str "exporter " cf/management-key) - "Authorization" (str "Bearer " token)}) - -(defn- asset-headers - "Auth headers for `/assets/*` downloads. Cookie, not Bearer: those endpoints - redirect to a presigned S3/minio URL, and a Bearer Authorization header makes - S3 400 (\"multiple authentication types\")." - [token] - #js {"X-Shared-Key" (str "exporter " cf/management-key) - "Cookie" (str "auth-token=" token)}) - -;; --- shape bundle fetch (backend RPC) - -(defn- fetch-objects - "Fetches the page's `objects` map from the backend via the `get-page` RPC, - using the same auth the exporter uses elsewhere (management key + bearer)." - [{:keys [file-id page-id share-id token]}] - (let [headers (rpc-headers token) - ;; share-id is an OPTIONAL uuid on the backend; it must be omitted when - ;; absent, not sent as nil (nil fails the uuid schema). - body (t/encode-str (cond-> {:file-id file-id - :page-id page-id} - share-id (assoc :share-id share-id))) - uri (internal-uri "api/rpc/command/get-page")] - (l/dbg :hint "wasm render: get-page" - :uri uri - :file-id (str file-id) - :page-id (str page-id)) - (->> (http/fetch uri #js {:method "POST" :headers headers :body body}) - (p/mcat (fn [^js resp] - (if (= 200 (.-status resp)) - (.text resp) - ;; Surface the backend's actual error body so we can tell - ;; auth (401) from bad params (400/404). - (->> (.text resp) - (p/mcat (fn [resp-body] - (l/error :hint "wasm render: get-page failed" - :status (.-status resp) - :body resp-body) - (p/rejected (ex-info "get-page failed" - {:status (.-status resp) - :body resp-body})))))))) - (p/fmap t/decode-str) - (p/fmap :objects)))) - -;; --- font resolution -;; -;; Custom (team) fonts: the exporter's text serializer keeps their real font -;; uuid, so `wasm/fonts-for-shape` reports it. We fetch the file's team font -;; variants once (get-font-variants RPC), map uuid+weight+style -> ttf asset id, -;; and download the TTF from `assets/by-id/`. Google/builtin fonts still fall -;; back to the bundled default (they need the gfonts catalog — a later slice). - -(defn- fetch-font-variants - "Fetches the file's team (custom) font variants via the get-font-variants RPC. - Returns a promise of the variant vector (or nil on failure — fonts then just - fall back, they don't fail the export)." - [{:keys [file-id share-id token]}] - (let [headers (rpc-headers token) - body (t/encode-str (cond-> {:file-id file-id} - share-id (assoc :share-id share-id))) - uri (internal-uri "api/rpc/command/get-font-variants")] - (->> (http/fetch uri #js {:method "POST" :headers headers :body body}) - (p/mcat (fn [^js resp] - (if (= 200 (.-status resp)) - (.text resp) - (p/resolved nil)))) - (p/fmap (fn [s] (when s (t/decode-str s)))) - (p/merr (fn [cause] - (l/warn :hint "wasm render: get-font-variants failed" :cause cause) - (p/resolved nil)))))) - -(defn- fetch-asset-bytes - "Downloads a stored asset (font TTF) by id, returning a promise of an - ArrayBuffer (or nil)." - [asset-id {:keys [token]}] - (let [headers (asset-headers token) - uri (internal-uri (str "assets/by-id/" asset-id))] - (->> (http/fetch uri #js {:method "GET" :headers headers}) - (p/mcat (fn [^js resp] - (if (= 200 (.-status resp)) - (.arrayBuffer resp) - (p/resolved nil)))) - (p/merr (fn [cause] - (l/warn :hint "wasm render: font asset fetch failed" - :asset-id (str asset-id) :cause cause) - (p/resolved nil)))))) - -(defn- gfont-proxy-url - "Rewrites a gstatic ttf url to the local gfonts proxy (mirrors the browser's - `google-font-ttf-url`)." - [ttf-url] - (let [proxy (internal-uri "internal/gfonts/font/")] - (str/replace ttf-url "https://fonts.gstatic.com/s/" proxy))) - -(defn- fetch-gfont-bytes - "Downloads a google font TTF through the local gfonts proxy." - [ttf-url] - (let [uri (gfont-proxy-url ttf-url)] - (->> (http/fetch uri #js {:method "GET"}) - (p/mcat (fn [^js resp] - (if (= 200 (.-status resp)) - (.arrayBuffer resp) - (p/resolved nil)))) - (p/merr (fn [cause] - (l/warn :hint "wasm render: gfont fetch failed" :url uri :cause cause) - (p/resolved nil)))))) - -(defn- make-resolve-font - "Builds a `resolve-font` fn (family map -> promise of TTF bytes). Tries the - file's custom (team) font variants first — matching uuid + weight + style, - degrading to uuid+weight then uuid — then the shared google catalog. Returns - nil for fonts not found (builtin falls back to the bundled default)." - [variants params] - (fn [{:keys [id weight style]}] - (let [font-uuid (uuid/from-unsigned-parts (aget id 0) (aget id 1) (aget id 2) (aget id 3)) - style-str (if (zero? style) "normal" "italic") - variant (or (d/seek (fn [v] (and (= (:font-id v) font-uuid) - (= (:font-weight v) weight) - (= (name (:font-style v)) style-str))) - variants) - (d/seek (fn [v] (and (= (:font-id v) font-uuid) - (= (:font-weight v) weight))) - variants) - (d/seek (fn [v] (= (:font-id v) font-uuid)) variants))] - (if-let [ttf-id (:ttf-file-id variant)] - (fetch-asset-bytes ttf-id params) - (if-let [gurl (gfonts/resolve-ttf-url font-uuid weight style)] - (fetch-gfont-bytes gurl) - (p/resolved nil)))))) - -;; --- fallback fonts (emoji + per-script noto fonts) -;; -;; Emoji and non-latin scripts render through fallback families in the wasm -;; font store, not through any span's font family — so `wasm/fonts-for-shape` -;; never reports them and the per-object provisioning above never uploads -;; them. The browser workspace uploads them as a side effect of serializing -;; text (`add-emoji-font` / `add-noto-fonts`), which is why client-side single -;; exports show emoji/CJK while headless exports dropped them. Which fonts a -;; scene needs is computed by the host-agnostic -;; `app.render-wasm.fallback-fonts` (same data the browser uses). -;; `clear-fonts!` resets the store every request, so this must run per -;; request; the TTF bytes are cached per font for the process lifetime. - -(defn- scene-fallback-fonts - "Fallback font descriptors needed by the scene's text content. Deduped: - several languages share a noto family (`gfont-noto-sans` covers cyrillic, - greek, devanagari, latin-ext and vietnamese), and the descriptors are - provisioned concurrently — without this they'd all miss the byte cache at - once and download the same TTF several times." - [scene] - (let [texts (for [shape (vals scene) - :when (= :text (:type shape)) - node (or (some->> (:content shape) (tree-seq :children :children)) []) - :let [text (:text node)] - :when (string? text)] - text) - emoji? (boolean (some fbf/contains-emoji? texts)) - langs (reduce fbf/collect-used-languages #{} texts)] - (distinct - (cond-> (fbf/add-noto-fonts [] langs) - emoji? (fbf/add-emoji-font))))) - -(defonce ^:private fallback-font-bytes* (atom {})) - -(defn- fetch-fallback-font-bytes - "Downloads (and caches for the process lifetime) one fallback font's TTF." - [{:keys [font-id weight style]}] - (if-let [bytes (get @fallback-font-bytes* font-id)] - (p/resolved bytes) - (let [font-uuid (gfonts/gfont-id->uuid font-id) - ttf-url (some-> font-uuid (gfonts/resolve-ttf-url weight style))] - (if ttf-url - (->> (fetch-gfont-bytes ttf-url) - (p/fmap (fn [buf] - (when buf (swap! fallback-font-bytes* assoc font-id buf)) - buf))) - (p/resolved nil))))) - -(defn- provision-fallback-fonts! - [scene] - (->> (scene-fallback-fonts scene) - (map (fn [{:keys [font-id weight style is-emoji is-fallback] :as font}] - (if-let [font-uuid (gfonts/gfont-id->uuid font-id)] - (->> (fetch-fallback-font-bytes font) - (p/fmap (fn [buf] - (if buf - (wasm/store-font! {:id (uuid/get-u32 font-uuid) - :weight weight - :style style - :emoji? (boolean is-emoji) - :fallback? (boolean is-fallback)} - buf) - (l/warn :hint "wasm render: fallback font unavailable" - :font-id font-id))))) - (p/resolved nil)))) - (p/all))) - -;; --- image resolution -;; -;; Image fills reference file-media ids. We collect them from the scene, fetch -;; the encoded bytes from `assets/by-file-media-id/`, and hand them to -;; `_store_image` (Skia decodes them; no WebGL). Keyed by media uuid, so this is -;; done once per request rather than per rendered object. - -(defn- fetch-file-media-bytes - "Downloads an image fill's encoded bytes by file-media id." - [media-id {:keys [token]}] - (let [headers (asset-headers token) - uri (internal-uri (str "assets/by-file-media-id/" media-id))] - (->> (http/fetch uri #js {:method "GET" :headers headers}) - (p/mcat (fn [^js resp] - (if (= 200 (.-status resp)) - (.arrayBuffer resp) - (do - (l/warn :hint "wasm render: image fetch non-200" - :media-id (str media-id) - :uri uri - :status (.-status resp)) - (p/resolved nil))))) - (p/merr (fn [cause] - (l/warn :hint "wasm render: image fetch failed" - :media-id (str media-id) :uri uri :cause cause) - (p/resolved nil)))))) - -(defn- provision-images! - "Fetches and stores every image the scene references — shape fills, stroke - image fills, and text-span image fills (enumerated by the host-agnostic - `app.render-wasm.resources`, same source the workspace uses). Images the - module already holds are skipped: unlike fonts, the image store is not - reset per request, so repeated exports of the same file reuse them. - NOTE: that also means the store grows with every distinct image the - process ever exports; if exporter memory becomes a problem, add an - eviction policy on the Rust side rather than clearing per request." - [scene params] - (let [all-ids (resources/scene-image-ids scene) - new-ids (remove wasm/image-cached? all-ids)] - (l/dbg :hint "wasm render: provisioning images" - :total (count all-ids) - :cached (- (count all-ids) (count new-ids))) - (->> new-ids - (map (fn [image-id] - (->> (fetch-file-media-bytes image-id params) - (p/fmap (fn [buf] - (if buf - (do - (l/dbg :hint "wasm render: image stored" - :media-id (str image-id) - :bytes (.-byteLength ^js buf)) - (wasm/store-image! image-id buf)) - (l/warn :hint "wasm render: image unavailable" - :media-id (str image-id)))))))) - (p/all)))) - -(defn- relayout-text! - "Recomputes layout for every text shape in the scene. Called after fonts are - provisioned so metrics use the real fonts (serialize-time layout used the - fallback)." - [scene] - (doseq [shape (vals scene) - :when (= :text (:type shape))] - (wasm/update-text-layout! (:id shape)))) - -;; --- render - -(defn- render-object-bytes - [type id scale] - (if (= :pdf type) - (let [bytes (wasm/render-shape-pdf id scale)] - (l/dbg :hint "PDF generated via Skia (render-wasm headless)" - :object-id (str id) - :backend "skia-wasm" - :bytes (.-length bytes)) - bytes) - ;; The export type doubles as the encoder format: Skia encodes png/jpeg/webp - ;; natively, so — unlike the Playwright backend — webp needs no imagemagick - ;; conversion pass. `:svg` never reaches here; it needs vector markup, so - ;; `app.renderer` keeps it on the browser path. - (wasm/render-shape-raster id scale type))) - -;; NOTE: `handlers.export-shapes/prepare-exports` splits an export into -;; partitions of 50 objects, and each partition arrives here as its own -;; request. Since the unit of work is the partition, every one of them -;; re-fetches and re-serializes the *whole* page scene, and re-provisions its -;; fonts. Exporting 200 frames therefore pays that setup 4x. Acceptable while -;; the module is a single shared instance (`enqueue!` serializes requests -;; anyway); revisit together with module pooling. - -(defn- render* - [{:keys [scale type objects] :as params} on-object] - (l/dbg :hint "wasm render: start" - :type type - :scale scale - :objects (count objects) - :file-id (str (:file-id params)) - :page-id (str (:page-id params))) - (->> (ensure-module!) - (p/mcat (fn [_] (fetch-objects params))) - (p/mcat (fn [scene] - (l/dbg :hint "wasm render: scene fetched" :shapes (count scene)) - (serialize/serialize-scene! scene) - (l/dbg :hint "wasm render: scene serialized") - ;; Reset the shared module's font store so fonts from a previous - ;; request don't accumulate / leak into this one. - (wasm/clear-fonts!) - ;; Fetch the file's custom (team) font variants once, provision - ;; every referenced image once, then resolve/provision the - ;; deduped font set for all rendered objects. - (->> (p/all [(fetch-font-variants params) - (provision-images! scene params) - (provision-fallback-fonts! scene)]) - (p/mcat - (fn [[variants _]] - (let [resolve-font (make-resolve-font (or variants []) params)] - ;; Provision every object's fonts BEFORE rendering, so - ;; the text relayout below sees real font metrics. - ;; Deduped across objects: a 50-frame partition that - ;; shares one family downloads its TTF once, not 50 - ;; times. - (wasm/provision-fonts! (map :id objects) resolve-font)))) - (p/mcat - (fn [_] - ;; Serialize-time layout used the fallback font (fonts - ;; weren't uploaded yet); recompute now that they are, or - ;; text metrics/line breaks are wrong. - (relayout-text! scene) - (p/run - (fn [{:keys [id] :as object}] - (let [bytes (render-object-bytes type id scale) - path (sh/tempfile :prefix "penpot.tmp.wasm." - :suffix (mime/get-extension type))] - (l/dbg :hint "wasm render: object rendered" - :object-id (str id) :bytes (.-length bytes)) - (fs/writeFileSync path bytes) - ;; `on-object` may return a plain value (zip append - ;; returns the archiver instance) or a promise (single - ;; export's file move); `p/do` normalizes both to a - ;; thenable so `p/mcat` doesn't throw "expected thenable". - (p/do (on-object (assoc object :path path))))) - objects)))))) - (p/fmap (fn [result] - ;; Trim the image store AFTER the request (never mid-render, so - ;; an image can't disappear under a running export). Images the - ;; next request needs again are simply re-provisioned. - (let [evicted (wasm/evict-images! (wasm/image-cache-mb))] - (when (pos? evicted) - (l/info :hint "wasm render: evicted cached images" :count evicted))) - result)) - (p/merr (fn [cause] - (l/error :hint "wasm render: failed" :cause cause) - ;; A panic/abort can leave the shared module's buffer allocated - ;; or the wasm instance aborted; drop it so the next request - ;; rebuilds a fresh module instead of inheriting the bad state. - (reset! module* nil) - (p/rejected cause))))) + [app.wasm.pool :as pool])) (defn render - "Public entry. Serializes module access through `enqueue!` so concurrent - exports (multi-partition zip) run one at a time on the shared WASM instance." [params on-object] - (enqueue! (fn [] (render* params on-object)))) + (pool/render params on-object)) diff --git a/exporter/src/app/wasm/pool.cljs b/exporter/src/app/wasm/pool.cljs new file mode 100644 index 0000000000..256571728a --- /dev/null +++ b/exporter/src/app/wasm/pool.cljs @@ -0,0 +1,287 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.wasm.pool + "Main-thread pool of headless render workers. + + A render is synchronous wasm work; running it here would block the node + event loop for the whole export, stalling the http server and every other + request. So each render runs in a `node:worker_threads` worker owning its + own wasm module, and this namespace does the scheduling. + + Round-robin: any idle worker takes the next request, so one large export + spreads across the pool. Two consequences worth knowing: + + - Each worker holds its own image/font cache, so partitions of the same + export do not share a warm cache and may refetch the same images. + - Objects no longer complete in submission order across partitions. Callers + that care about order must sort by `:index` (see + `handlers.export-frames`); appending in completion order gives a PDF with + shuffled pages. + + Because the render is off-thread, a stuck request can finally be bounded: + the watchdog below terminates a worker that stops reporting progress, and a + fresh one replaces it." + (:require + ["node:path" :as path] + ["node:process" :as proc] + ["node:worker_threads" :as wt] + [app.common.logging :as l] + [app.common.transit :as t] + [app.config :as cf] + [promesa.core :as p])) + +(def ^:private default-pool-size + "Conservative: every worker holds a full wasm heap plus its own image cache, + so memory grows with the pool, not just cpu usage." + 2) + +(def ^:private default-timeout-ms + "Watchdog budget between two progress reports, not for the whole request: a + 50-object partition is legitimately slow, a wedged one reports nothing." + 120000) + +(def ^:private crash-window-ms + "A worker dying sooner than this after spawning never got as far as doing + work, so it is a startup failure (bad config, missing wasm artifact) that + restarting cannot fix — not a crash on some particular file." + 5000) + +(def ^:private max-restarts 5) +(def ^:private restart-backoff-ms 1000) + +(defn pool-size [] (max 1 (cf/get :wasm-pool-size default-pool-size))) +(defn timeout-ms [] (cf/get :wasm-render-timeout default-timeout-ms)) + +(defn- worker-script + "Absolute path of the compiled worker bundle. Derived from the running + script (`app.js`) so it resolves both in devenv (`node target/app.js`) and + in the docker image (`/opt/penpot/exporter/app.js`)." + [] + (or (cf/get :wasm-worker-script) + (-> (aget (.-argv ^js proc/default) 1) + (path/dirname) + (path/resolve "worker" "wasm-worker.js")))) + +;; --- pool state +;; +;; One atom per worker so a single worker's lifecycle can be swapped without +;; touching the others. Each holds {:worker :job :timer}; `:job` is nil when +;; idle, and a worker is never given a second job while one is in flight. + +(defonce ^:private workers* (atom [])) +(defonce ^:private pending* (atom #queue [])) +(defonce ^:private req-id* (atom 0)) + +(declare pump! respawn!) + +(defn- pool-dead? + [] + (let [workers @workers*] + (and (seq workers) (every? (fn [w] (:dead? @w)) workers)))) + +(defn- drain-pending! + "Fails everything still queued. Called when the last worker gives up: without + it those requests would wait for a worker that is never coming back." + [cause] + (let [queued @pending*] + (reset! pending* #queue []) + (doseq [job queued] + ((:reject job) cause)))) + +;; --- watchdog + +(defn- clear-timer! + [wstate] + (when-let [timer (:timer @wstate)] + (js/clearTimeout timer)) + (swap! wstate dissoc :timer)) + +(defn- arm-timer! + [wstate] + (clear-timer! wstate) + (let [timer (js/setTimeout + (fn [] + (when-let [job (:job @wstate)] + (l/error :hint "wasm pool: worker stopped reporting progress, terminating" + :request-id (:id job) + :timeout (timeout-ms)) + (swap! wstate assoc :job nil) + ((:reject job) (ex-info "headless render timed out" + {:request-id (:id job) + :timeout (timeout-ms)})) + (respawn! wstate))) + (timeout-ms))] + (swap! wstate assoc :timer timer))) + +;; --- job lifecycle + +(defn- settle-ok! + "Resolves the job, but only once every `on-object` call has settled: those + run as a promise chain so the caller's zip append / file move keep the + order the worker emitted them in." + [wstate] + (clear-timer! wstate) + (when-let [job (:job @wstate)] + (swap! wstate assoc :job nil) + (->> @(:chain job) + (p/fmap (fn [_] ((:resolve job) nil))) + (p/merr (fn [cause] ((:reject job) cause) (p/resolved nil))))) + (pump!)) + +(defn- settle-error! + [wstate cause] + (clear-timer! wstate) + (when-let [job (:job @wstate)] + (swap! wstate assoc :job nil) + ((:reject job) cause)) + (pump!)) + +(defn- append-object! + "Chains one `on-object` call after the previous one. The worker may run ahead + of the caller (it writes tempfiles as fast as it renders); chaining keeps the + callbacks themselves strictly ordered." + [job object path] + (swap! (:chain job) + (fn [prev] + (->> prev + (p/mcat (fn [_] (p/do ((:on-object job) (assoc object :path path))))))))) + +(defn- on-message + [wstate ^js msg] + (let [type (unchecked-get msg "type") + job (:job @wstate)] + ;; A reply can outlive the job it belongs to (watchdog fired, worker was + ;; already terminated); drop anything that no longer matches. + (when (and job (= (:id job) (unchecked-get msg "id"))) + (case type + "object" (do (arm-timer! wstate) + (append-object! job + (t/decode-str (unchecked-get msg "object")) + (unchecked-get msg "path"))) + "done" (settle-ok! wstate) + "error" (settle-error! wstate (ex-info (unchecked-get msg "message") {})) + nil)))) + +;; --- worker lifecycle + +(defn- note-death! + "Counts consecutive startup crashes. A worker that survived past the crash + window did real work before dying, so its counter resets." + [wstate] + (let [alive-ms (- (js/Date.now) (:spawned-at @wstate 0))] + (swap! wstate update :failures + (fn [n] (if (< alive-ms crash-window-ms) (inc (or n 0)) 0))))) + +(defn- spawn! + [wstate] + (let [worker (new wt/Worker (worker-script))] + (swap! wstate assoc :spawned-at (js/Date.now)) + (.on ^js worker "message" (fn [msg] (on-message wstate msg))) + (.on ^js worker "error" + (fn [cause] + (l/error :hint "wasm pool: worker crashed" :cause cause) + (settle-error! wstate cause) + (respawn! wstate))) + (.on ^js worker "exit" + (fn [code] + ;; A clean exit only happens on `stop`; anything else means the + ;; worker died under a request, so fail it and rebuild. + (when-not (:stopping? @wstate) + (when (:job @wstate) + (l/error :hint "wasm pool: worker exited mid-render" :code code) + (settle-error! wstate (ex-info "headless render worker exited" {:code code}))) + (respawn! wstate)))) + (swap! wstate assoc :worker worker) + wstate)) + +(defn- respawn! + [wstate] + (when-let [worker (:worker @wstate)] + (swap! wstate dissoc :worker) + (.terminate ^js worker)) + (note-death! wstate) + (when-not (:stopping? @wstate) + (if (>= (:failures @wstate 0) max-restarts) + ;; Restarting is not going to help: it died on startup this many times in + ;; a row, so the cause is the environment (config, missing artifact). + ;; Give up on this worker and, once none are left, stop accepting work + ;; instead of queueing requests nobody will ever pick up. + (do + (swap! wstate assoc :dead? true) + (l/error :hint "wasm pool: worker keeps failing at startup, giving up" + :restarts max-restarts + :script (worker-script)) + (when (pool-dead?) + (l/error :hint "wasm pool: no workers left, failing headless exports") + (drain-pending! (ex-info "headless render pool is unavailable" {})))) + (js/setTimeout (fn [] + (when-not (:stopping? @wstate) + (spawn! wstate) + (pump!))) + (* restart-backoff-ms (inc (:failures @wstate 0))))))) + +(defn- ensure-pool! + [] + (when (empty? @workers*) + (let [size (pool-size)] + (l/info :hint "wasm pool: starting" :workers size :script (worker-script)) + (reset! workers* (vec (for [_ (range size)] + (spawn! (atom {})))))))) + +;; --- dispatch + +(defn- dispatch! + [wstate job] + (swap! wstate assoc :job job) + (arm-timer! wstate) + (.postMessage ^js (:worker @wstate) + #js {:type "render" + :id (:id job) + :params (t/encode-str (:params job))})) + +(defn- pump! + "Hands queued requests to idle workers, oldest request first." + [] + (loop [] + (when-let [wstate (->> @workers* + (filter (fn [w] (and (:worker @w) (nil? (:job @w))))) + (first))] + (when-let [job (peek @pending*)] + (swap! pending* pop) + (dispatch! wstate job) + (recur))))) + +(defn render + "Runs one export request on a pool worker. `on-object` is called with each + rendered object plus its tempfile `:path`, in the order the worker produced + them. Resolves once the request is done and every `on-object` has settled." + [params on-object] + (ensure-pool!) + (p/create + (fn [resolve reject] + (if (pool-dead?) + (reject (ex-info "headless render pool is unavailable" {})) + (do + (swap! pending* conj {:id (swap! req-id* inc) + :params params + :on-object on-object + :chain (atom (p/resolved nil)) + :resolve resolve + :reject reject}) + (pump!)))))) + +(defn stop + "Terminates every worker. Needed on devenv hot reload, where otherwise each + reload would leak a full pool." + [] + (let [workers @workers*] + (reset! workers* []) + (p/all (for [wstate workers] + (do (swap! wstate assoc :stopping? true) + (clear-timer! wstate) + (when-let [worker (:worker @wstate)] + (.terminate ^js worker))))))) diff --git a/exporter/src/app/wasm/render.cljs b/exporter/src/app/wasm/render.cljs new file mode 100644 index 0000000000..7cd53b3417 --- /dev/null +++ b/exporter/src/app/wasm/render.cljs @@ -0,0 +1,447 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.wasm.render + "Headless render pipeline. Runs inside a `app.wasm.worker` thread, never on + the main thread: every step below is synchronous wasm work that would + otherwise block the node event loop for the whole export. + + Backs `app.renderer.wasm` (the Playwright backends' drop-in alternative), + which dispatches here through `app.wasm.pool`. Pipeline per request: + + init module (once) -> fetch shape bundle (backend RPC) + -> serialize scene + -> provision resources (team/google fonts + fallback fonts + images, + enumerated by the host-agnostic app.render-wasm.{resources,fallback-fonts}) + -> relayout text (real font metrics) -> render each object -> tempfile + + Handles png/jpeg/webp (Skia encodes all three natively) and pdf. `:svg` is + routed to the browser backend by `app.renderer` — it needs vector markup, not + a raster. + + Each worker owns one wasm module (one design state, one global mem buffer) + and handles one request at a time, so nothing here needs to guard against + concurrent access." + (:require + ["node:fs" :as fs] + ["undici" :as http] + [app.common.data :as d] + ;; Required for side effects: these register the transit read handlers and + ;; deftype impls the `get-page` response is decoded into. + [app.common.geom.matrix] + [app.common.geom.point] + [app.common.geom.rect] + [app.common.logging :as l] + [app.common.transit :as t] + [app.common.types.fills.impl] + [app.common.types.objects-map] + [app.common.types.path.impl] + [app.common.types.shape] + [app.common.uri :as u] + [app.common.uuid :as uuid] + [app.config :as cf] + [app.render-wasm.fallback-fonts :as fbf] + [app.render-wasm.resources :as resources] + [app.util.mime :as mime] + [app.util.shell :as sh] + [app.wasm :as wasm] + [app.wasm.gfonts :as gfonts] + [app.wasm.serialize :as serialize] + [cuerdas.core :as str] + [promesa.core :as p])) + +;; --- module lifecycle (one lazily-initialized instance per worker thread) + +(defonce ^:private module* (atom nil)) + +(defn- ensure-module! + [] + (or @module* + (reset! module* (wasm/init!)))) + +;; --- backend endpoints +;; +;; Every fetch below targets the *internal* endpoint (`internal-uri`, falling +;; back to `public-uri`), same as the Playwright backends and +;; `handlers.resources`: in a real deployment the exporter reaches the backend +;; over the container network, not through the public ingress. + +(defn- internal-uri + "Absolute URI for `path` on the internal (backend) endpoint." + [path] + (-> (cf/get-internal-uri) + (u/ensure-path-slash) + (u/join path) + (str))) + +(defn- rpc-headers + "Auth headers for backend RPC calls (management key + bearer)." + [token] + #js {"Content-Type" "application/transit+json" + "X-Shared-Key" (str "exporter " cf/management-key) + "Authorization" (str "Bearer " token)}) + +(defn- asset-headers + "Auth headers for `/assets/*` downloads. Cookie, not Bearer: those endpoints + redirect to a presigned S3/minio URL, and a Bearer Authorization header makes + S3 400 (\"multiple authentication types\")." + [token] + #js {"X-Shared-Key" (str "exporter " cf/management-key) + "Cookie" (str "auth-token=" token)}) + +;; --- shape bundle fetch (backend RPC) + +(defn- fetch-objects + "Fetches the page's `objects` map from the backend via the `get-page` RPC, + using the same auth the exporter uses elsewhere (management key + bearer)." + [{:keys [file-id page-id share-id token]}] + (let [headers (rpc-headers token) + ;; share-id is an OPTIONAL uuid on the backend; it must be omitted when + ;; absent, not sent as nil (nil fails the uuid schema). + body (t/encode-str (cond-> {:file-id file-id + :page-id page-id} + share-id (assoc :share-id share-id))) + uri (internal-uri "api/rpc/command/get-page")] + (l/dbg :hint "wasm render: get-page" + :uri uri + :file-id (str file-id) + :page-id (str page-id)) + (->> (http/fetch uri #js {:method "POST" :headers headers :body body}) + (p/mcat (fn [^js resp] + (if (= 200 (.-status resp)) + (.text resp) + ;; Surface the backend's actual error body so we can tell + ;; auth (401) from bad params (400/404). + (->> (.text resp) + (p/mcat (fn [resp-body] + (l/error :hint "wasm render: get-page failed" + :status (.-status resp) + :body resp-body) + (p/rejected (ex-info "get-page failed" + {:status (.-status resp) + :body resp-body})))))))) + (p/fmap t/decode-str) + (p/fmap :objects)))) + +;; --- font resolution +;; +;; Custom (team) fonts: the exporter's text serializer keeps their real font +;; uuid, so `wasm/fonts-for-shape` reports it. We fetch the file's team font +;; variants once (get-font-variants RPC), map uuid+weight+style -> ttf asset id, +;; and download the TTF from `assets/by-id/`. Google/builtin fonts still fall +;; back to the bundled default (they need the gfonts catalog — a later slice). + +(defn- fetch-font-variants + "Fetches the file's team (custom) font variants via the get-font-variants RPC. + Returns a promise of the variant vector (or nil on failure — fonts then just + fall back, they don't fail the export)." + [{:keys [file-id share-id token]}] + (let [headers (rpc-headers token) + body (t/encode-str (cond-> {:file-id file-id} + share-id (assoc :share-id share-id))) + uri (internal-uri "api/rpc/command/get-font-variants")] + (->> (http/fetch uri #js {:method "POST" :headers headers :body body}) + (p/mcat (fn [^js resp] + (if (= 200 (.-status resp)) + (.text resp) + (p/resolved nil)))) + (p/fmap (fn [s] (when s (t/decode-str s)))) + (p/merr (fn [cause] + (l/warn :hint "wasm render: get-font-variants failed" :cause cause) + (p/resolved nil)))))) + +(defn- fetch-asset-bytes + "Downloads a stored asset (font TTF) by id, returning a promise of an + ArrayBuffer (or nil)." + [asset-id {:keys [token]}] + (let [headers (asset-headers token) + uri (internal-uri (str "assets/by-id/" asset-id))] + (->> (http/fetch uri #js {:method "GET" :headers headers}) + (p/mcat (fn [^js resp] + (if (= 200 (.-status resp)) + (.arrayBuffer resp) + (p/resolved nil)))) + (p/merr (fn [cause] + (l/warn :hint "wasm render: font asset fetch failed" + :asset-id (str asset-id) :cause cause) + (p/resolved nil)))))) + +(defn- gfont-proxy-url + "Rewrites a gstatic ttf url to the local gfonts proxy (mirrors the browser's + `google-font-ttf-url`)." + [ttf-url] + (let [proxy (internal-uri "internal/gfonts/font/")] + (str/replace ttf-url "https://fonts.gstatic.com/s/" proxy))) + +(defn- fetch-gfont-bytes + "Downloads a google font TTF through the local gfonts proxy." + [ttf-url] + (let [uri (gfont-proxy-url ttf-url)] + (->> (http/fetch uri #js {:method "GET"}) + (p/mcat (fn [^js resp] + (if (= 200 (.-status resp)) + (.arrayBuffer resp) + (p/resolved nil)))) + (p/merr (fn [cause] + (l/warn :hint "wasm render: gfont fetch failed" :url uri :cause cause) + (p/resolved nil)))))) + +(defn- make-resolve-font + "Builds a `resolve-font` fn (family map -> promise of TTF bytes). Tries the + file's custom (team) font variants first — matching uuid + weight + style, + degrading to uuid+weight then uuid — then the shared google catalog. Returns + nil for fonts not found (builtin falls back to the bundled default)." + [variants params] + (fn [{:keys [id weight style]}] + (let [font-uuid (uuid/from-unsigned-parts (aget id 0) (aget id 1) (aget id 2) (aget id 3)) + style-str (if (zero? style) "normal" "italic") + variant (or (d/seek (fn [v] (and (= (:font-id v) font-uuid) + (= (:font-weight v) weight) + (= (name (:font-style v)) style-str))) + variants) + (d/seek (fn [v] (and (= (:font-id v) font-uuid) + (= (:font-weight v) weight))) + variants) + (d/seek (fn [v] (= (:font-id v) font-uuid)) variants))] + (if-let [ttf-id (:ttf-file-id variant)] + (fetch-asset-bytes ttf-id params) + (if-let [gurl (gfonts/resolve-ttf-url font-uuid weight style)] + (fetch-gfont-bytes gurl) + (p/resolved nil)))))) + +;; --- fallback fonts (emoji + per-script noto fonts) +;; +;; Emoji and non-latin scripts render through fallback families in the wasm +;; font store, not through any span's font family — so `wasm/fonts-for-shape` +;; never reports them and the per-object provisioning above never uploads +;; them. The browser workspace uploads them as a side effect of serializing +;; text (`add-emoji-font` / `add-noto-fonts`), which is why client-side single +;; exports show emoji/CJK while headless exports dropped them. Which fonts a +;; scene needs is computed by the host-agnostic +;; `app.render-wasm.fallback-fonts` (same data the browser uses). +;; `clear-fonts!` resets the store every request, so this must run per +;; request; the TTF bytes are cached per font for the process lifetime. + +(defn- scene-fallback-fonts + "Fallback font descriptors needed by the scene's text content. Deduped: + several languages share a noto family (`gfont-noto-sans` covers cyrillic, + greek, devanagari, latin-ext and vietnamese), and the descriptors are + provisioned concurrently — without this they'd all miss the byte cache at + once and download the same TTF several times." + [scene] + (let [texts (for [shape (vals scene) + :when (= :text (:type shape)) + node (or (some->> (:content shape) (tree-seq :children :children)) []) + :let [text (:text node)] + :when (string? text)] + text) + emoji? (boolean (some fbf/contains-emoji? texts)) + langs (reduce fbf/collect-used-languages #{} texts)] + (distinct + (cond-> (fbf/add-noto-fonts [] langs) + emoji? (fbf/add-emoji-font))))) + +(defonce ^:private fallback-font-bytes* (atom {})) + +(defn- fetch-fallback-font-bytes + "Downloads (and caches for the process lifetime) one fallback font's TTF." + [{:keys [font-id weight style]}] + (if-let [bytes (get @fallback-font-bytes* font-id)] + (p/resolved bytes) + (let [font-uuid (gfonts/gfont-id->uuid font-id) + ttf-url (some-> font-uuid (gfonts/resolve-ttf-url weight style))] + (if ttf-url + (->> (fetch-gfont-bytes ttf-url) + (p/fmap (fn [buf] + (when buf (swap! fallback-font-bytes* assoc font-id buf)) + buf))) + (p/resolved nil))))) + +(defn- provision-fallback-fonts! + [scene] + (->> (scene-fallback-fonts scene) + (map (fn [{:keys [font-id weight style is-emoji is-fallback] :as font}] + (if-let [font-uuid (gfonts/gfont-id->uuid font-id)] + (->> (fetch-fallback-font-bytes font) + (p/fmap (fn [buf] + (if buf + (wasm/store-font! {:id (uuid/get-u32 font-uuid) + :weight weight + :style style + :emoji? (boolean is-emoji) + :fallback? (boolean is-fallback)} + buf) + (l/warn :hint "wasm render: fallback font unavailable" + :font-id font-id))))) + (p/resolved nil)))) + (p/all))) + +;; --- image resolution +;; +;; Image fills reference file-media ids. We collect them from the scene, fetch +;; the encoded bytes from `assets/by-file-media-id/`, and hand them to +;; `_store_image` (Skia decodes them; no WebGL). Keyed by media uuid, so this is +;; done once per request rather than per rendered object. + +(defn- fetch-file-media-bytes + "Downloads an image fill's encoded bytes by file-media id." + [media-id {:keys [token]}] + (let [headers (asset-headers token) + uri (internal-uri (str "assets/by-file-media-id/" media-id))] + (->> (http/fetch uri #js {:method "GET" :headers headers}) + (p/mcat (fn [^js resp] + (if (= 200 (.-status resp)) + (.arrayBuffer resp) + (do + (l/warn :hint "wasm render: image fetch non-200" + :media-id (str media-id) + :uri uri + :status (.-status resp)) + (p/resolved nil))))) + (p/merr (fn [cause] + (l/warn :hint "wasm render: image fetch failed" + :media-id (str media-id) :uri uri :cause cause) + (p/resolved nil)))))) + +(defn- provision-images! + "Fetches and stores every image the scene references — shape fills, stroke + image fills, and text-span image fills (enumerated by the host-agnostic + `app.render-wasm.resources`, same source the workspace uses). Images the + module already holds are skipped: unlike fonts, the image store is not + reset per request, so repeated exports of the same file reuse them. + NOTE: that also means the store grows with every distinct image the + process ever exports; if exporter memory becomes a problem, add an + eviction policy on the Rust side rather than clearing per request." + [scene params] + (let [all-ids (resources/scene-image-ids scene) + new-ids (remove wasm/image-cached? all-ids)] + (l/dbg :hint "wasm render: provisioning images" + :total (count all-ids) + :cached (- (count all-ids) (count new-ids))) + (->> new-ids + (map (fn [image-id] + (->> (fetch-file-media-bytes image-id params) + (p/fmap (fn [buf] + (if buf + (do + (l/dbg :hint "wasm render: image stored" + :media-id (str image-id) + :bytes (.-byteLength ^js buf)) + (wasm/store-image! image-id buf)) + (l/warn :hint "wasm render: image unavailable" + :media-id (str image-id)))))))) + (p/all)))) + +(defn- relayout-text! + "Recomputes layout for every text shape in the scene. Called after fonts are + provisioned so metrics use the real fonts (serialize-time layout used the + fallback)." + [scene] + (doseq [shape (vals scene) + :when (= :text (:type shape))] + (wasm/update-text-layout! (:id shape)))) + +;; --- render + +(defn- render-object-bytes + [type id scale] + (if (= :pdf type) + (let [bytes (wasm/render-shape-pdf id scale)] + (l/dbg :hint "PDF generated via Skia (render-wasm headless)" + :object-id (str id) + :backend "skia-wasm" + :bytes (.-length bytes)) + bytes) + ;; The export type doubles as the encoder format: Skia encodes png/jpeg/webp + ;; natively, so — unlike the Playwright backend — webp needs no imagemagick + ;; conversion pass. `:svg` never reaches here; it needs vector markup, so + ;; `app.renderer` keeps it on the browser path. + (wasm/render-shape-raster id scale type))) + +;; NOTE: `handlers.export-shapes/prepare-exports` splits an export into +;; partitions of 50 objects, and each partition arrives here as its own +;; request. Since the unit of work is the partition, every one of them +;; re-fetches and re-serializes the *whole* page scene, and re-provisions its +;; fonts. Exporting 200 frames therefore pays that setup 4x — and with the pool +;; dispatching partitions round-robin, those 4 setups now land on 4 different +;; workers, so none of them share a warm image cache either. Worth revisiting +;; if setup cost shows up in practice. + +(defn render + "Runs one export request to completion. `emit-object!` is called once per + rendered object with the object map plus a `:path` to its tempfile; it may + return a promise, which is awaited before the next object renders." + [{:keys [scale type objects] :as params} emit-object!] + (l/dbg :hint "wasm render: start" + :type type + :scale scale + :objects (count objects) + :file-id (str (:file-id params)) + :page-id (str (:page-id params))) + (->> (ensure-module!) + (p/mcat (fn [_] (fetch-objects params))) + (p/mcat (fn [scene] + (l/dbg :hint "wasm render: scene fetched" :shapes (count scene)) + (serialize/serialize-scene! scene) + (l/dbg :hint "wasm render: scene serialized") + ;; Reset the shared module's font store so fonts from a previous + ;; request don't accumulate / leak into this one. + (wasm/clear-fonts!) + ;; Fetch the file's custom (team) font variants once, provision + ;; every referenced image once, then resolve/provision the + ;; deduped font set for all rendered objects. + (->> (p/all [(fetch-font-variants params) + (provision-images! scene params) + (provision-fallback-fonts! scene)]) + (p/mcat + (fn [[variants _]] + (let [resolve-font (make-resolve-font (or variants []) params)] + ;; Provision every object's fonts BEFORE rendering, so + ;; the text relayout below sees real font metrics. + ;; Deduped across objects: a 50-frame partition that + ;; shares one family downloads its TTF once, not 50 + ;; times. + (wasm/provision-fonts! (map :id objects) resolve-font)))) + (p/mcat + (fn [_] + ;; Serialize-time layout used the fallback font (fonts + ;; weren't uploaded yet); recompute now that they are, or + ;; text metrics/line breaks are wrong. + (relayout-text! scene) + (p/run + (fn [{:keys [id] :as object}] + (let [bytes (render-object-bytes type id scale) + path (sh/tempfile :prefix "penpot.tmp.wasm." + :suffix (mime/get-extension type))] + (l/dbg :hint "wasm render: object rendered" + :object-id (str id) :bytes (.-length bytes)) + (fs/writeFileSync path bytes) + ;; `emit-object!` may return a plain value or a + ;; promise; `p/do` normalizes both to a thenable so + ;; `p/mcat` doesn't throw "expected thenable". + (p/do (emit-object! (assoc object :path path))))) + objects)))))) + (p/fmap (fn [result] + ;; Trim the image store AFTER the request (never mid-render, so + ;; an image can't disappear under a running export). Images the + ;; next request needs again are simply re-provisioned. + (let [evicted (wasm/evict-images! (wasm/image-cache-mb))] + (when (pos? evicted) + (l/info :hint "wasm render: evicted cached images" :count evicted))) + result)) + (p/merr (fn [cause] + ;; undici hides the real network failure (ECONNREFUSED, a TLS + ;; rejection, ...) in `.cause`, and without it every failure + ;; reads as a bare "fetch failed". + (l/error :hint "wasm render: failed" + :reason (some-> (unchecked-get cause "cause") + (unchecked-get "code")) + :cause cause) + ;; A panic/abort can leave the module's buffer allocated or the + ;; wasm instance aborted; drop it so the next request rebuilds a + ;; fresh module instead of inheriting the bad state. + (reset! module* nil) + (p/rejected cause))))) diff --git a/exporter/src/app/wasm/worker.cljs b/exporter/src/app/wasm/worker.cljs new file mode 100644 index 0000000000..cabc199853 --- /dev/null +++ b/exporter/src/app/wasm/worker.cljs @@ -0,0 +1,61 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.wasm.worker + "Entry point of the headless render worker thread. + + Owns one wasm module and runs `app.wasm.render` on it. Kept deliberately + thin: everything here is message plumbing, so the pipeline itself stays + free of worker concerns and the module never touches the main thread. + + Protocol (see `app.wasm.pool` for the other end). Params and objects cross + the boundary transit-encoded, because they carry uuids and keywords that + structured clone would mangle: + + in {type: \"render\", id, params} + out {type: \"object\", id, object, path} one per rendered object + out {type: \"done\", id} + out {type: \"error\", id, message} + + The pool sends one request at a time per worker, so there is no need to + multiplex `id`s here — it is echoed back only so the pool can drop replies + from a request it already gave up on." + (:require + ["node:worker_threads" :as wt] + [app.common.logging :as l] + [app.common.transit :as t] + [app.wasm.render :as render] + [promesa.core :as p])) + +(enable-console-print!) +(l/setup! {:app :info}) + +(defn- post! + [msg] + (.postMessage ^js wt/parentPort msg)) + +(defn- handle-render! + [id params] + (->> (render/render (t/decode-str params) + (fn [object] + (post! #js {:type "object" + :id id + :object (t/encode-str (dissoc object :path)) + :path (:path object)}))) + (p/fnly (fn [_ cause] + (if cause + (post! #js {:type "error" + :id id + :message (or (ex-message cause) (str cause))}) + (post! #js {:type "done" :id id})))))) + +(defn main + [& _] + (.on ^js wt/parentPort "message" + (fn [^js msg] + (when (= "render" (unchecked-get msg "type")) + (handle-render! (unchecked-get msg "id") + (unchecked-get msg "params"))))))