mirror of
https://github.com/penpot/penpot.git
synced 2026-08-06 12:58:55 +00:00
✨ Run headless wasm exports in a worker thread pool
This commit is contained in:
Elena Torro
parent
b9b75805c9
commit
ed83e185be
@ -81,9 +81,15 @@ export PENPOT_INTERNAL_URI=http://localhost:3450
|
||||
# defaulting to ../frontend/resources/public/js (render-wasm.js/.wasm) — which
|
||||
# is where render-wasm/build leaves it in devenv. The docker image ships the
|
||||
# artifact inside the exporter bundle and points PENPOT_WASM_DIR at it.
|
||||
# export PENPOT_WASM_HEADLESS=true
|
||||
export PENPOT_WASM_HEADLESS=true
|
||||
# export PENPOT_WASM_DIR=../frontend/resources/public/js
|
||||
# Per worker, so the process can hold up to POOL_SIZE times this.
|
||||
# export PENPOT_WASM_IMAGE_CACHE_MB=256
|
||||
# Render worker threads. Each owns a wasm module, so this costs memory, not
|
||||
# just cpu. Renders are synchronous, so they run off the event loop.
|
||||
# export PENPOT_WASM_POOL_SIZE=2
|
||||
# Milliseconds without progress before a worker is treated as wedged.
|
||||
# export PENPOT_WASM_RENDER_TIMEOUT=120000
|
||||
|
||||
export JAVA_OPTS="\
|
||||
-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager \
|
||||
|
||||
@ -30,9 +30,9 @@
|
||||
},
|
||||
"scripts": {
|
||||
"clear:shadow-cache": "rm -rf .shadow-cljs && rm -rf target",
|
||||
"watch:app": "pnpm run clear:shadow-cache && clojure -M:dev:shadow-cljs watch main",
|
||||
"watch:app": "pnpm run clear:shadow-cache && clojure -M:dev:shadow-cljs watch main wasm-worker",
|
||||
"watch": "pnpm run watch:app",
|
||||
"build:app": "clojure -M:dev:shadow-cljs release main",
|
||||
"build:app": "clojure -M:dev:shadow-cljs release main wasm-worker",
|
||||
"build": "pnpm run clear:shadow-cache && pnpm run build:app",
|
||||
"fmt": "cljfmt fix --parallel=true src/",
|
||||
"check-fmt": "cljfmt check --parallel=true src/",
|
||||
|
||||
@ -40,4 +40,4 @@ EOF
|
||||
|
||||
chmod +x target/setup;
|
||||
|
||||
sed -i -re "s/\%version\%/$CURRENT_VERSION/g" ./target/app.js;
|
||||
sed -i -re "s/\%version\%/$CURRENT_VERSION/g" ./target/app.js ./target/worker/wasm-worker.js;
|
||||
|
||||
@ -23,6 +23,39 @@
|
||||
{:output-feature-set :es2020
|
||||
:output-wrapper false}
|
||||
|
||||
:release
|
||||
{:compiler-options
|
||||
{:fn-invoke-direct true
|
||||
:source-map true
|
||||
:optimizations #shadow/env ["PENPOT_BUILD_OPTIMIZATIONS" :as :keyword :default :simple]
|
||||
:pseudo-names true
|
||||
:pretty-print true
|
||||
:anon-fn-naming-policy :off
|
||||
:source-map-detail-level :all}}}
|
||||
|
||||
;; Headless render worker (see app.wasm.pool). A separate build rather than a
|
||||
;; second module of :main, so the bundle node loads through `new Worker(path)`
|
||||
;; is self-contained — code splitting would leave it importing shared chunks
|
||||
;; relative to a different base. Its own :output-dir for the same reason.
|
||||
:wasm-worker
|
||||
{:target :esm
|
||||
:runtime :node
|
||||
:output-dir "target/worker/"
|
||||
|
||||
:modules
|
||||
{:wasm-worker
|
||||
{:entries []
|
||||
:init-fn app.wasm.worker/main}}
|
||||
|
||||
:js-options
|
||||
{:entry-keys ["module" "browser" "main"]
|
||||
:export-conditions ["module" "import", "browser" "require" "default"]
|
||||
:js-provider :import}
|
||||
|
||||
:compiler-options
|
||||
{:output-feature-set :es2020
|
||||
:output-wrapper false}
|
||||
|
||||
:release
|
||||
{:compiler-options
|
||||
{:fn-invoke-direct true
|
||||
|
||||
@ -49,8 +49,18 @@
|
||||
[:wasm-headless {:optional true} :boolean]
|
||||
[:wasm-dir {:optional true} :string]
|
||||
;; Byte budget (in MB) for the WASM image cache; least-recently-used
|
||||
;; images are evicted between requests once the store exceeds it.
|
||||
[:wasm-image-cache-mb {:optional true} ::sm/int]])
|
||||
;; images are evicted between requests once the store exceeds it. Applies
|
||||
;; per worker, so the process can hold up to pool-size times this.
|
||||
[:wasm-image-cache-mb {:optional true} ::sm/int]
|
||||
;; Number of render worker threads. Each owns a full wasm module, so this
|
||||
;; trades memory for concurrency.
|
||||
[:wasm-pool-size {:optional true} ::sm/int]
|
||||
;; Milliseconds a worker may go without reporting progress before it is
|
||||
;; considered wedged, terminated and replaced.
|
||||
[:wasm-render-timeout {:optional true} ::sm/int]
|
||||
;; Escape hatch for the compiled worker bundle path; normally derived from
|
||||
;; the running script.
|
||||
[:wasm-worker-script {:optional true} :string]])
|
||||
|
||||
(def ^:private decode-config
|
||||
(sm/decoder schema:config sm/string-transformer))
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
[app.http :as http]
|
||||
[app.redis :as redis]
|
||||
[app.wasm :as wasm]
|
||||
[app.wasm.pool :as wasm.pool]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(enable-console-print!)
|
||||
@ -25,11 +26,10 @@
|
||||
:internal-uri (str (cf/get-internal-uri))
|
||||
:version (:full cf/version))
|
||||
(when (cf/get :wasm-headless)
|
||||
(l/warn :msg "headless wasm export enabled (experimental)"
|
||||
:hint (str "renders run in-process on a single shared wasm module "
|
||||
"and are serialized one at a time, so exports do not "
|
||||
"run concurrently; not recommended for busy instances")
|
||||
(l/info :msg "headless wasm export enabled"
|
||||
:hint "each worker holds its own wasm module and image cache"
|
||||
:wasm-dir (wasm/artifact-dir)
|
||||
:workers (wasm.pool/pool-size)
|
||||
:image-cache-mb (wasm/image-cache-mb)))
|
||||
(p/do!
|
||||
(bwr/init)
|
||||
@ -48,6 +48,7 @@
|
||||
(bwr/stop)
|
||||
(redis/stop)
|
||||
(http/stop)
|
||||
(wasm.pool/stop)
|
||||
(done)))
|
||||
|
||||
(.on proc/default "uncaughtException"
|
||||
|
||||
@ -85,12 +85,16 @@
|
||||
:cause (ex-message cause)}]
|
||||
(redis/pub! topic data)))
|
||||
|
||||
;; Keyed by `:index` (the frame's position in the request), NOT append
|
||||
;; order: partitions render concurrently, so objects arrive finished in
|
||||
;; whatever order they complete. Appending here would hand `pdfunite`
|
||||
;; its pages shuffled.
|
||||
result-cache
|
||||
(atom [])
|
||||
(atom {})
|
||||
|
||||
on-object
|
||||
(fn [{:keys [path] :as object}]
|
||||
(let [res (swap! result-cache conj path)]
|
||||
(fn [{:keys [path index]}]
|
||||
(let [res (swap! result-cache assoc index path)]
|
||||
(on-progress (count res))))
|
||||
|
||||
procs
|
||||
@ -98,7 +102,7 @@
|
||||
(map #(rd/render (assoc % :is-wasm is-wasm) on-object)))]
|
||||
|
||||
(->> (p/all procs)
|
||||
(p/fmap (fn [] @result-cache))
|
||||
(p/fmap (fn [] (->> @result-cache (sort-by key) (mapv val))))
|
||||
(p/mcat (partial join-pdf file-id))
|
||||
(p/mcat (partial move-file resource))
|
||||
(p/fmap (constantly resource))
|
||||
|
||||
@ -180,11 +180,17 @@
|
||||
{:id (:object-id entry)
|
||||
:filename (:filename entry)
|
||||
:name (:name entry)
|
||||
:suffix (:suffix entry)})]
|
||||
:suffix (:suffix entry)
|
||||
:index (:index entry)})]
|
||||
|
||||
;; `:index` is the object's position in the *requested* order. Renderers
|
||||
;; hand objects back as they finish, and partitions run concurrently, so
|
||||
;; completion order is not submission order — anything order-sensitive
|
||||
;; (`export-frames` joining a multi-page pdf) has to sort by this.
|
||||
(let [xform (comp
|
||||
(map #(assoc % :token token))
|
||||
(assoc-file-name))]
|
||||
(assoc-file-name)
|
||||
(map-indexed (fn [index params] (assoc params :index index))))]
|
||||
(->> (sequence xform exports)
|
||||
(d/group-by (juxt :scale :type))
|
||||
(map second)
|
||||
|
||||
@ -6,456 +6,20 @@
|
||||
|
||||
(ns app.renderer.wasm
|
||||
"Headless renderer backend: renders exports with the render-wasm Skia
|
||||
pipeline running in this Node process, with no browser and no WebGL.
|
||||
pipeline, with no browser and no WebGL.
|
||||
|
||||
Drop-in alternative to `app.renderer.bitmap`/`pdf` (the Playwright
|
||||
backends), selected from `app.renderer` when an export is flagged
|
||||
headless. Pipeline per request:
|
||||
backends), selected from `app.renderer` when an export is flagged headless.
|
||||
Handles png/jpeg/webp (Skia encodes all three natively) and pdf; `:svg` is
|
||||
routed to the browser backend by `app.renderer` — it needs vector markup,
|
||||
not a raster.
|
||||
|
||||
init module (once) -> fetch shape bundle (backend RPC)
|
||||
-> serialize scene
|
||||
-> provision resources (team/google fonts + fallback fonts + images,
|
||||
enumerated by the host-agnostic app.render-wasm.{resources,fallback-fonts})
|
||||
-> relayout text (real font metrics) -> render each object -> tempfile
|
||||
|
||||
Handles png/jpeg/webp (Skia encodes all three natively) and pdf. `:svg` is
|
||||
routed to the browser backend by `app.renderer` — it needs vector markup, not
|
||||
a raster.
|
||||
|
||||
NOTE (current slice): single shared WASM design state, so requests are
|
||||
serialized one at a time (no pooling yet)."
|
||||
Only the entry point lives here. The render itself is synchronous wasm work,
|
||||
so it runs in a worker thread rather than on the event loop: `app.wasm.pool`
|
||||
schedules it and `app.wasm.render` is the pipeline."
|
||||
(:require
|
||||
["node:fs" :as fs]
|
||||
["undici" :as http]
|
||||
[app.common.data :as d]
|
||||
;; Required for side effects: these register the transit read handlers and
|
||||
;; deftype impls the `get-page` response is decoded into.
|
||||
[app.common.geom.matrix]
|
||||
[app.common.geom.point]
|
||||
[app.common.geom.rect]
|
||||
[app.common.logging :as l]
|
||||
[app.common.transit :as t]
|
||||
[app.common.types.fills.impl]
|
||||
[app.common.types.objects-map]
|
||||
[app.common.types.path.impl]
|
||||
[app.common.types.shape]
|
||||
[app.common.uri :as u]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.render-wasm.fallback-fonts :as fbf]
|
||||
[app.render-wasm.resources :as resources]
|
||||
[app.util.mime :as mime]
|
||||
[app.util.shell :as sh]
|
||||
[app.wasm :as wasm]
|
||||
[app.wasm.gfonts :as gfonts]
|
||||
[app.wasm.serialize :as serialize]
|
||||
[cuerdas.core :as str]
|
||||
[promesa.core :as p]))
|
||||
|
||||
;; --- module lifecycle (one shared, lazily-initialized instance)
|
||||
|
||||
(defonce ^:private module* (atom nil))
|
||||
|
||||
(defn- ensure-module!
|
||||
[]
|
||||
(or @module*
|
||||
(reset! module* (wasm/init!))))
|
||||
|
||||
;; --- serialize access to the single shared WASM module ---
|
||||
;;
|
||||
;; There is one shared WASM instance (one design state, one global mem buffer),
|
||||
;; but `handle-multiple-export` fans out partitions concurrently (`p/all`). This
|
||||
;; promise queue runs module-touching work one task at a time so concurrent
|
||||
;; exports can't interleave their serialize/render/alloc on the shared state.
|
||||
|
||||
(defonce ^:private queue (atom (p/resolved nil)))
|
||||
|
||||
(defn- enqueue!
|
||||
"Runs `thunk` (0-arg, returns a promise) only after all previously enqueued
|
||||
work has settled. Returns `thunk`'s promise. A task's failure is isolated:
|
||||
it doesn't break the chain for the next task."
|
||||
[thunk]
|
||||
(let [result (p/handle @queue (fn [_ _] (thunk)))]
|
||||
(reset! queue (p/handle result (fn [_ _] nil)))
|
||||
result))
|
||||
|
||||
;; --- backend endpoints
|
||||
;;
|
||||
;; Every fetch below targets the *internal* endpoint (`internal-uri`, falling
|
||||
;; back to `public-uri`), same as the Playwright backends and
|
||||
;; `handlers.resources`: in a real deployment the exporter reaches the backend
|
||||
;; over the container network, not through the public ingress.
|
||||
|
||||
(defn- internal-uri
|
||||
"Absolute URI for `path` on the internal (backend) endpoint."
|
||||
[path]
|
||||
(-> (cf/get-internal-uri)
|
||||
(u/ensure-path-slash)
|
||||
(u/join path)
|
||||
(str)))
|
||||
|
||||
(defn- rpc-headers
|
||||
"Auth headers for backend RPC calls (management key + bearer)."
|
||||
[token]
|
||||
#js {"Content-Type" "application/transit+json"
|
||||
"X-Shared-Key" (str "exporter " cf/management-key)
|
||||
"Authorization" (str "Bearer " token)})
|
||||
|
||||
(defn- asset-headers
|
||||
"Auth headers for `/assets/*` downloads. Cookie, not Bearer: those endpoints
|
||||
redirect to a presigned S3/minio URL, and a Bearer Authorization header makes
|
||||
S3 400 (\"multiple authentication types\")."
|
||||
[token]
|
||||
#js {"X-Shared-Key" (str "exporter " cf/management-key)
|
||||
"Cookie" (str "auth-token=" token)})
|
||||
|
||||
;; --- shape bundle fetch (backend RPC)
|
||||
|
||||
(defn- fetch-objects
|
||||
"Fetches the page's `objects` map from the backend via the `get-page` RPC,
|
||||
using the same auth the exporter uses elsewhere (management key + bearer)."
|
||||
[{:keys [file-id page-id share-id token]}]
|
||||
(let [headers (rpc-headers token)
|
||||
;; share-id is an OPTIONAL uuid on the backend; it must be omitted when
|
||||
;; absent, not sent as nil (nil fails the uuid schema).
|
||||
body (t/encode-str (cond-> {:file-id file-id
|
||||
:page-id page-id}
|
||||
share-id (assoc :share-id share-id)))
|
||||
uri (internal-uri "api/rpc/command/get-page")]
|
||||
(l/dbg :hint "wasm render: get-page"
|
||||
:uri uri
|
||||
:file-id (str file-id)
|
||||
:page-id (str page-id))
|
||||
(->> (http/fetch uri #js {:method "POST" :headers headers :body body})
|
||||
(p/mcat (fn [^js resp]
|
||||
(if (= 200 (.-status resp))
|
||||
(.text resp)
|
||||
;; Surface the backend's actual error body so we can tell
|
||||
;; auth (401) from bad params (400/404).
|
||||
(->> (.text resp)
|
||||
(p/mcat (fn [resp-body]
|
||||
(l/error :hint "wasm render: get-page failed"
|
||||
:status (.-status resp)
|
||||
:body resp-body)
|
||||
(p/rejected (ex-info "get-page failed"
|
||||
{:status (.-status resp)
|
||||
:body resp-body}))))))))
|
||||
(p/fmap t/decode-str)
|
||||
(p/fmap :objects))))
|
||||
|
||||
;; --- font resolution
|
||||
;;
|
||||
;; Custom (team) fonts: the exporter's text serializer keeps their real font
|
||||
;; uuid, so `wasm/fonts-for-shape` reports it. We fetch the file's team font
|
||||
;; variants once (get-font-variants RPC), map uuid+weight+style -> ttf asset id,
|
||||
;; and download the TTF from `assets/by-id/<id>`. Google/builtin fonts still fall
|
||||
;; back to the bundled default (they need the gfonts catalog — a later slice).
|
||||
|
||||
(defn- fetch-font-variants
|
||||
"Fetches the file's team (custom) font variants via the get-font-variants RPC.
|
||||
Returns a promise of the variant vector (or nil on failure — fonts then just
|
||||
fall back, they don't fail the export)."
|
||||
[{:keys [file-id share-id token]}]
|
||||
(let [headers (rpc-headers token)
|
||||
body (t/encode-str (cond-> {:file-id file-id}
|
||||
share-id (assoc :share-id share-id)))
|
||||
uri (internal-uri "api/rpc/command/get-font-variants")]
|
||||
(->> (http/fetch uri #js {:method "POST" :headers headers :body body})
|
||||
(p/mcat (fn [^js resp]
|
||||
(if (= 200 (.-status resp))
|
||||
(.text resp)
|
||||
(p/resolved nil))))
|
||||
(p/fmap (fn [s] (when s (t/decode-str s))))
|
||||
(p/merr (fn [cause]
|
||||
(l/warn :hint "wasm render: get-font-variants failed" :cause cause)
|
||||
(p/resolved nil))))))
|
||||
|
||||
(defn- fetch-asset-bytes
|
||||
"Downloads a stored asset (font TTF) by id, returning a promise of an
|
||||
ArrayBuffer (or nil)."
|
||||
[asset-id {:keys [token]}]
|
||||
(let [headers (asset-headers token)
|
||||
uri (internal-uri (str "assets/by-id/" asset-id))]
|
||||
(->> (http/fetch uri #js {:method "GET" :headers headers})
|
||||
(p/mcat (fn [^js resp]
|
||||
(if (= 200 (.-status resp))
|
||||
(.arrayBuffer resp)
|
||||
(p/resolved nil))))
|
||||
(p/merr (fn [cause]
|
||||
(l/warn :hint "wasm render: font asset fetch failed"
|
||||
:asset-id (str asset-id) :cause cause)
|
||||
(p/resolved nil))))))
|
||||
|
||||
(defn- gfont-proxy-url
|
||||
"Rewrites a gstatic ttf url to the local gfonts proxy (mirrors the browser's
|
||||
`google-font-ttf-url`)."
|
||||
[ttf-url]
|
||||
(let [proxy (internal-uri "internal/gfonts/font/")]
|
||||
(str/replace ttf-url "https://fonts.gstatic.com/s/" proxy)))
|
||||
|
||||
(defn- fetch-gfont-bytes
|
||||
"Downloads a google font TTF through the local gfonts proxy."
|
||||
[ttf-url]
|
||||
(let [uri (gfont-proxy-url ttf-url)]
|
||||
(->> (http/fetch uri #js {:method "GET"})
|
||||
(p/mcat (fn [^js resp]
|
||||
(if (= 200 (.-status resp))
|
||||
(.arrayBuffer resp)
|
||||
(p/resolved nil))))
|
||||
(p/merr (fn [cause]
|
||||
(l/warn :hint "wasm render: gfont fetch failed" :url uri :cause cause)
|
||||
(p/resolved nil))))))
|
||||
|
||||
(defn- make-resolve-font
|
||||
"Builds a `resolve-font` fn (family map -> promise of TTF bytes). Tries the
|
||||
file's custom (team) font variants first — matching uuid + weight + style,
|
||||
degrading to uuid+weight then uuid — then the shared google catalog. Returns
|
||||
nil for fonts not found (builtin falls back to the bundled default)."
|
||||
[variants params]
|
||||
(fn [{:keys [id weight style]}]
|
||||
(let [font-uuid (uuid/from-unsigned-parts (aget id 0) (aget id 1) (aget id 2) (aget id 3))
|
||||
style-str (if (zero? style) "normal" "italic")
|
||||
variant (or (d/seek (fn [v] (and (= (:font-id v) font-uuid)
|
||||
(= (:font-weight v) weight)
|
||||
(= (name (:font-style v)) style-str)))
|
||||
variants)
|
||||
(d/seek (fn [v] (and (= (:font-id v) font-uuid)
|
||||
(= (:font-weight v) weight)))
|
||||
variants)
|
||||
(d/seek (fn [v] (= (:font-id v) font-uuid)) variants))]
|
||||
(if-let [ttf-id (:ttf-file-id variant)]
|
||||
(fetch-asset-bytes ttf-id params)
|
||||
(if-let [gurl (gfonts/resolve-ttf-url font-uuid weight style)]
|
||||
(fetch-gfont-bytes gurl)
|
||||
(p/resolved nil))))))
|
||||
|
||||
;; --- fallback fonts (emoji + per-script noto fonts)
|
||||
;;
|
||||
;; Emoji and non-latin scripts render through fallback families in the wasm
|
||||
;; font store, not through any span's font family — so `wasm/fonts-for-shape`
|
||||
;; never reports them and the per-object provisioning above never uploads
|
||||
;; them. The browser workspace uploads them as a side effect of serializing
|
||||
;; text (`add-emoji-font` / `add-noto-fonts`), which is why client-side single
|
||||
;; exports show emoji/CJK while headless exports dropped them. Which fonts a
|
||||
;; scene needs is computed by the host-agnostic
|
||||
;; `app.render-wasm.fallback-fonts` (same data the browser uses).
|
||||
;; `clear-fonts!` resets the store every request, so this must run per
|
||||
;; request; the TTF bytes are cached per font for the process lifetime.
|
||||
|
||||
(defn- scene-fallback-fonts
|
||||
"Fallback font descriptors needed by the scene's text content. Deduped:
|
||||
several languages share a noto family (`gfont-noto-sans` covers cyrillic,
|
||||
greek, devanagari, latin-ext and vietnamese), and the descriptors are
|
||||
provisioned concurrently — without this they'd all miss the byte cache at
|
||||
once and download the same TTF several times."
|
||||
[scene]
|
||||
(let [texts (for [shape (vals scene)
|
||||
:when (= :text (:type shape))
|
||||
node (or (some->> (:content shape) (tree-seq :children :children)) [])
|
||||
:let [text (:text node)]
|
||||
:when (string? text)]
|
||||
text)
|
||||
emoji? (boolean (some fbf/contains-emoji? texts))
|
||||
langs (reduce fbf/collect-used-languages #{} texts)]
|
||||
(distinct
|
||||
(cond-> (fbf/add-noto-fonts [] langs)
|
||||
emoji? (fbf/add-emoji-font)))))
|
||||
|
||||
(defonce ^:private fallback-font-bytes* (atom {}))
|
||||
|
||||
(defn- fetch-fallback-font-bytes
|
||||
"Downloads (and caches for the process lifetime) one fallback font's TTF."
|
||||
[{:keys [font-id weight style]}]
|
||||
(if-let [bytes (get @fallback-font-bytes* font-id)]
|
||||
(p/resolved bytes)
|
||||
(let [font-uuid (gfonts/gfont-id->uuid font-id)
|
||||
ttf-url (some-> font-uuid (gfonts/resolve-ttf-url weight style))]
|
||||
(if ttf-url
|
||||
(->> (fetch-gfont-bytes ttf-url)
|
||||
(p/fmap (fn [buf]
|
||||
(when buf (swap! fallback-font-bytes* assoc font-id buf))
|
||||
buf)))
|
||||
(p/resolved nil)))))
|
||||
|
||||
(defn- provision-fallback-fonts!
|
||||
[scene]
|
||||
(->> (scene-fallback-fonts scene)
|
||||
(map (fn [{:keys [font-id weight style is-emoji is-fallback] :as font}]
|
||||
(if-let [font-uuid (gfonts/gfont-id->uuid font-id)]
|
||||
(->> (fetch-fallback-font-bytes font)
|
||||
(p/fmap (fn [buf]
|
||||
(if buf
|
||||
(wasm/store-font! {:id (uuid/get-u32 font-uuid)
|
||||
:weight weight
|
||||
:style style
|
||||
:emoji? (boolean is-emoji)
|
||||
:fallback? (boolean is-fallback)}
|
||||
buf)
|
||||
(l/warn :hint "wasm render: fallback font unavailable"
|
||||
:font-id font-id)))))
|
||||
(p/resolved nil))))
|
||||
(p/all)))
|
||||
|
||||
;; --- image resolution
|
||||
;;
|
||||
;; Image fills reference file-media ids. We collect them from the scene, fetch
|
||||
;; the encoded bytes from `assets/by-file-media-id/<id>`, and hand them to
|
||||
;; `_store_image` (Skia decodes them; no WebGL). Keyed by media uuid, so this is
|
||||
;; done once per request rather than per rendered object.
|
||||
|
||||
(defn- fetch-file-media-bytes
|
||||
"Downloads an image fill's encoded bytes by file-media id."
|
||||
[media-id {:keys [token]}]
|
||||
(let [headers (asset-headers token)
|
||||
uri (internal-uri (str "assets/by-file-media-id/" media-id))]
|
||||
(->> (http/fetch uri #js {:method "GET" :headers headers})
|
||||
(p/mcat (fn [^js resp]
|
||||
(if (= 200 (.-status resp))
|
||||
(.arrayBuffer resp)
|
||||
(do
|
||||
(l/warn :hint "wasm render: image fetch non-200"
|
||||
:media-id (str media-id)
|
||||
:uri uri
|
||||
:status (.-status resp))
|
||||
(p/resolved nil)))))
|
||||
(p/merr (fn [cause]
|
||||
(l/warn :hint "wasm render: image fetch failed"
|
||||
:media-id (str media-id) :uri uri :cause cause)
|
||||
(p/resolved nil))))))
|
||||
|
||||
(defn- provision-images!
|
||||
"Fetches and stores every image the scene references — shape fills, stroke
|
||||
image fills, and text-span image fills (enumerated by the host-agnostic
|
||||
`app.render-wasm.resources`, same source the workspace uses). Images the
|
||||
module already holds are skipped: unlike fonts, the image store is not
|
||||
reset per request, so repeated exports of the same file reuse them.
|
||||
NOTE: that also means the store grows with every distinct image the
|
||||
process ever exports; if exporter memory becomes a problem, add an
|
||||
eviction policy on the Rust side rather than clearing per request."
|
||||
[scene params]
|
||||
(let [all-ids (resources/scene-image-ids scene)
|
||||
new-ids (remove wasm/image-cached? all-ids)]
|
||||
(l/dbg :hint "wasm render: provisioning images"
|
||||
:total (count all-ids)
|
||||
:cached (- (count all-ids) (count new-ids)))
|
||||
(->> new-ids
|
||||
(map (fn [image-id]
|
||||
(->> (fetch-file-media-bytes image-id params)
|
||||
(p/fmap (fn [buf]
|
||||
(if buf
|
||||
(do
|
||||
(l/dbg :hint "wasm render: image stored"
|
||||
:media-id (str image-id)
|
||||
:bytes (.-byteLength ^js buf))
|
||||
(wasm/store-image! image-id buf))
|
||||
(l/warn :hint "wasm render: image unavailable"
|
||||
:media-id (str image-id))))))))
|
||||
(p/all))))
|
||||
|
||||
(defn- relayout-text!
|
||||
"Recomputes layout for every text shape in the scene. Called after fonts are
|
||||
provisioned so metrics use the real fonts (serialize-time layout used the
|
||||
fallback)."
|
||||
[scene]
|
||||
(doseq [shape (vals scene)
|
||||
:when (= :text (:type shape))]
|
||||
(wasm/update-text-layout! (:id shape))))
|
||||
|
||||
;; --- render
|
||||
|
||||
(defn- render-object-bytes
|
||||
[type id scale]
|
||||
(if (= :pdf type)
|
||||
(let [bytes (wasm/render-shape-pdf id scale)]
|
||||
(l/dbg :hint "PDF generated via Skia (render-wasm headless)"
|
||||
:object-id (str id)
|
||||
:backend "skia-wasm"
|
||||
:bytes (.-length bytes))
|
||||
bytes)
|
||||
;; The export type doubles as the encoder format: Skia encodes png/jpeg/webp
|
||||
;; natively, so — unlike the Playwright backend — webp needs no imagemagick
|
||||
;; conversion pass. `:svg` never reaches here; it needs vector markup, so
|
||||
;; `app.renderer` keeps it on the browser path.
|
||||
(wasm/render-shape-raster id scale type)))
|
||||
|
||||
;; NOTE: `handlers.export-shapes/prepare-exports` splits an export into
|
||||
;; partitions of 50 objects, and each partition arrives here as its own
|
||||
;; request. Since the unit of work is the partition, every one of them
|
||||
;; re-fetches and re-serializes the *whole* page scene, and re-provisions its
|
||||
;; fonts. Exporting 200 frames therefore pays that setup 4x. Acceptable while
|
||||
;; the module is a single shared instance (`enqueue!` serializes requests
|
||||
;; anyway); revisit together with module pooling.
|
||||
|
||||
(defn- render*
|
||||
[{:keys [scale type objects] :as params} on-object]
|
||||
(l/dbg :hint "wasm render: start"
|
||||
:type type
|
||||
:scale scale
|
||||
:objects (count objects)
|
||||
:file-id (str (:file-id params))
|
||||
:page-id (str (:page-id params)))
|
||||
(->> (ensure-module!)
|
||||
(p/mcat (fn [_] (fetch-objects params)))
|
||||
(p/mcat (fn [scene]
|
||||
(l/dbg :hint "wasm render: scene fetched" :shapes (count scene))
|
||||
(serialize/serialize-scene! scene)
|
||||
(l/dbg :hint "wasm render: scene serialized")
|
||||
;; Reset the shared module's font store so fonts from a previous
|
||||
;; request don't accumulate / leak into this one.
|
||||
(wasm/clear-fonts!)
|
||||
;; Fetch the file's custom (team) font variants once, provision
|
||||
;; every referenced image once, then resolve/provision the
|
||||
;; deduped font set for all rendered objects.
|
||||
(->> (p/all [(fetch-font-variants params)
|
||||
(provision-images! scene params)
|
||||
(provision-fallback-fonts! scene)])
|
||||
(p/mcat
|
||||
(fn [[variants _]]
|
||||
(let [resolve-font (make-resolve-font (or variants []) params)]
|
||||
;; Provision every object's fonts BEFORE rendering, so
|
||||
;; the text relayout below sees real font metrics.
|
||||
;; Deduped across objects: a 50-frame partition that
|
||||
;; shares one family downloads its TTF once, not 50
|
||||
;; times.
|
||||
(wasm/provision-fonts! (map :id objects) resolve-font))))
|
||||
(p/mcat
|
||||
(fn [_]
|
||||
;; Serialize-time layout used the fallback font (fonts
|
||||
;; weren't uploaded yet); recompute now that they are, or
|
||||
;; text metrics/line breaks are wrong.
|
||||
(relayout-text! scene)
|
||||
(p/run
|
||||
(fn [{:keys [id] :as object}]
|
||||
(let [bytes (render-object-bytes type id scale)
|
||||
path (sh/tempfile :prefix "penpot.tmp.wasm."
|
||||
:suffix (mime/get-extension type))]
|
||||
(l/dbg :hint "wasm render: object rendered"
|
||||
:object-id (str id) :bytes (.-length bytes))
|
||||
(fs/writeFileSync path bytes)
|
||||
;; `on-object` may return a plain value (zip append
|
||||
;; returns the archiver instance) or a promise (single
|
||||
;; export's file move); `p/do` normalizes both to a
|
||||
;; thenable so `p/mcat` doesn't throw "expected thenable".
|
||||
(p/do (on-object (assoc object :path path)))))
|
||||
objects))))))
|
||||
(p/fmap (fn [result]
|
||||
;; Trim the image store AFTER the request (never mid-render, so
|
||||
;; an image can't disappear under a running export). Images the
|
||||
;; next request needs again are simply re-provisioned.
|
||||
(let [evicted (wasm/evict-images! (wasm/image-cache-mb))]
|
||||
(when (pos? evicted)
|
||||
(l/info :hint "wasm render: evicted cached images" :count evicted)))
|
||||
result))
|
||||
(p/merr (fn [cause]
|
||||
(l/error :hint "wasm render: failed" :cause cause)
|
||||
;; A panic/abort can leave the shared module's buffer allocated
|
||||
;; or the wasm instance aborted; drop it so the next request
|
||||
;; rebuilds a fresh module instead of inheriting the bad state.
|
||||
(reset! module* nil)
|
||||
(p/rejected cause)))))
|
||||
[app.wasm.pool :as pool]))
|
||||
|
||||
(defn render
|
||||
"Public entry. Serializes module access through `enqueue!` so concurrent
|
||||
exports (multi-partition zip) run one at a time on the shared WASM instance."
|
||||
[params on-object]
|
||||
(enqueue! (fn [] (render* params on-object))))
|
||||
(pool/render params on-object))
|
||||
|
||||
287
exporter/src/app/wasm/pool.cljs
Normal file
287
exporter/src/app/wasm/pool.cljs
Normal file
@ -0,0 +1,287 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.wasm.pool
|
||||
"Main-thread pool of headless render workers.
|
||||
|
||||
A render is synchronous wasm work; running it here would block the node
|
||||
event loop for the whole export, stalling the http server and every other
|
||||
request. So each render runs in a `node:worker_threads` worker owning its
|
||||
own wasm module, and this namespace does the scheduling.
|
||||
|
||||
Round-robin: any idle worker takes the next request, so one large export
|
||||
spreads across the pool. Two consequences worth knowing:
|
||||
|
||||
- Each worker holds its own image/font cache, so partitions of the same
|
||||
export do not share a warm cache and may refetch the same images.
|
||||
- Objects no longer complete in submission order across partitions. Callers
|
||||
that care about order must sort by `:index` (see
|
||||
`handlers.export-frames`); appending in completion order gives a PDF with
|
||||
shuffled pages.
|
||||
|
||||
Because the render is off-thread, a stuck request can finally be bounded:
|
||||
the watchdog below terminates a worker that stops reporting progress, and a
|
||||
fresh one replaces it."
|
||||
(:require
|
||||
["node:path" :as path]
|
||||
["node:process" :as proc]
|
||||
["node:worker_threads" :as wt]
|
||||
[app.common.logging :as l]
|
||||
[app.common.transit :as t]
|
||||
[app.config :as cf]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(def ^:private default-pool-size
|
||||
"Conservative: every worker holds a full wasm heap plus its own image cache,
|
||||
so memory grows with the pool, not just cpu usage."
|
||||
2)
|
||||
|
||||
(def ^:private default-timeout-ms
|
||||
"Watchdog budget between two progress reports, not for the whole request: a
|
||||
50-object partition is legitimately slow, a wedged one reports nothing."
|
||||
120000)
|
||||
|
||||
(def ^:private crash-window-ms
|
||||
"A worker dying sooner than this after spawning never got as far as doing
|
||||
work, so it is a startup failure (bad config, missing wasm artifact) that
|
||||
restarting cannot fix — not a crash on some particular file."
|
||||
5000)
|
||||
|
||||
(def ^:private max-restarts 5)
|
||||
(def ^:private restart-backoff-ms 1000)
|
||||
|
||||
(defn pool-size [] (max 1 (cf/get :wasm-pool-size default-pool-size)))
|
||||
(defn timeout-ms [] (cf/get :wasm-render-timeout default-timeout-ms))
|
||||
|
||||
(defn- worker-script
|
||||
"Absolute path of the compiled worker bundle. Derived from the running
|
||||
script (`app.js`) so it resolves both in devenv (`node target/app.js`) and
|
||||
in the docker image (`/opt/penpot/exporter/app.js`)."
|
||||
[]
|
||||
(or (cf/get :wasm-worker-script)
|
||||
(-> (aget (.-argv ^js proc/default) 1)
|
||||
(path/dirname)
|
||||
(path/resolve "worker" "wasm-worker.js"))))
|
||||
|
||||
;; --- pool state
|
||||
;;
|
||||
;; One atom per worker so a single worker's lifecycle can be swapped without
|
||||
;; touching the others. Each holds {:worker :job :timer}; `:job` is nil when
|
||||
;; idle, and a worker is never given a second job while one is in flight.
|
||||
|
||||
(defonce ^:private workers* (atom []))
|
||||
(defonce ^:private pending* (atom #queue []))
|
||||
(defonce ^:private req-id* (atom 0))
|
||||
|
||||
(declare pump! respawn!)
|
||||
|
||||
(defn- pool-dead?
|
||||
[]
|
||||
(let [workers @workers*]
|
||||
(and (seq workers) (every? (fn [w] (:dead? @w)) workers))))
|
||||
|
||||
(defn- drain-pending!
|
||||
"Fails everything still queued. Called when the last worker gives up: without
|
||||
it those requests would wait for a worker that is never coming back."
|
||||
[cause]
|
||||
(let [queued @pending*]
|
||||
(reset! pending* #queue [])
|
||||
(doseq [job queued]
|
||||
((:reject job) cause))))
|
||||
|
||||
;; --- watchdog
|
||||
|
||||
(defn- clear-timer!
|
||||
[wstate]
|
||||
(when-let [timer (:timer @wstate)]
|
||||
(js/clearTimeout timer))
|
||||
(swap! wstate dissoc :timer))
|
||||
|
||||
(defn- arm-timer!
|
||||
[wstate]
|
||||
(clear-timer! wstate)
|
||||
(let [timer (js/setTimeout
|
||||
(fn []
|
||||
(when-let [job (:job @wstate)]
|
||||
(l/error :hint "wasm pool: worker stopped reporting progress, terminating"
|
||||
:request-id (:id job)
|
||||
:timeout (timeout-ms))
|
||||
(swap! wstate assoc :job nil)
|
||||
((:reject job) (ex-info "headless render timed out"
|
||||
{:request-id (:id job)
|
||||
:timeout (timeout-ms)}))
|
||||
(respawn! wstate)))
|
||||
(timeout-ms))]
|
||||
(swap! wstate assoc :timer timer)))
|
||||
|
||||
;; --- job lifecycle
|
||||
|
||||
(defn- settle-ok!
|
||||
"Resolves the job, but only once every `on-object` call has settled: those
|
||||
run as a promise chain so the caller's zip append / file move keep the
|
||||
order the worker emitted them in."
|
||||
[wstate]
|
||||
(clear-timer! wstate)
|
||||
(when-let [job (:job @wstate)]
|
||||
(swap! wstate assoc :job nil)
|
||||
(->> @(:chain job)
|
||||
(p/fmap (fn [_] ((:resolve job) nil)))
|
||||
(p/merr (fn [cause] ((:reject job) cause) (p/resolved nil)))))
|
||||
(pump!))
|
||||
|
||||
(defn- settle-error!
|
||||
[wstate cause]
|
||||
(clear-timer! wstate)
|
||||
(when-let [job (:job @wstate)]
|
||||
(swap! wstate assoc :job nil)
|
||||
((:reject job) cause))
|
||||
(pump!))
|
||||
|
||||
(defn- append-object!
|
||||
"Chains one `on-object` call after the previous one. The worker may run ahead
|
||||
of the caller (it writes tempfiles as fast as it renders); chaining keeps the
|
||||
callbacks themselves strictly ordered."
|
||||
[job object path]
|
||||
(swap! (:chain job)
|
||||
(fn [prev]
|
||||
(->> prev
|
||||
(p/mcat (fn [_] (p/do ((:on-object job) (assoc object :path path)))))))))
|
||||
|
||||
(defn- on-message
|
||||
[wstate ^js msg]
|
||||
(let [type (unchecked-get msg "type")
|
||||
job (:job @wstate)]
|
||||
;; A reply can outlive the job it belongs to (watchdog fired, worker was
|
||||
;; already terminated); drop anything that no longer matches.
|
||||
(when (and job (= (:id job) (unchecked-get msg "id")))
|
||||
(case type
|
||||
"object" (do (arm-timer! wstate)
|
||||
(append-object! job
|
||||
(t/decode-str (unchecked-get msg "object"))
|
||||
(unchecked-get msg "path")))
|
||||
"done" (settle-ok! wstate)
|
||||
"error" (settle-error! wstate (ex-info (unchecked-get msg "message") {}))
|
||||
nil))))
|
||||
|
||||
;; --- worker lifecycle
|
||||
|
||||
(defn- note-death!
|
||||
"Counts consecutive startup crashes. A worker that survived past the crash
|
||||
window did real work before dying, so its counter resets."
|
||||
[wstate]
|
||||
(let [alive-ms (- (js/Date.now) (:spawned-at @wstate 0))]
|
||||
(swap! wstate update :failures
|
||||
(fn [n] (if (< alive-ms crash-window-ms) (inc (or n 0)) 0)))))
|
||||
|
||||
(defn- spawn!
|
||||
[wstate]
|
||||
(let [worker (new wt/Worker (worker-script))]
|
||||
(swap! wstate assoc :spawned-at (js/Date.now))
|
||||
(.on ^js worker "message" (fn [msg] (on-message wstate msg)))
|
||||
(.on ^js worker "error"
|
||||
(fn [cause]
|
||||
(l/error :hint "wasm pool: worker crashed" :cause cause)
|
||||
(settle-error! wstate cause)
|
||||
(respawn! wstate)))
|
||||
(.on ^js worker "exit"
|
||||
(fn [code]
|
||||
;; A clean exit only happens on `stop`; anything else means the
|
||||
;; worker died under a request, so fail it and rebuild.
|
||||
(when-not (:stopping? @wstate)
|
||||
(when (:job @wstate)
|
||||
(l/error :hint "wasm pool: worker exited mid-render" :code code)
|
||||
(settle-error! wstate (ex-info "headless render worker exited" {:code code})))
|
||||
(respawn! wstate))))
|
||||
(swap! wstate assoc :worker worker)
|
||||
wstate))
|
||||
|
||||
(defn- respawn!
|
||||
[wstate]
|
||||
(when-let [worker (:worker @wstate)]
|
||||
(swap! wstate dissoc :worker)
|
||||
(.terminate ^js worker))
|
||||
(note-death! wstate)
|
||||
(when-not (:stopping? @wstate)
|
||||
(if (>= (:failures @wstate 0) max-restarts)
|
||||
;; Restarting is not going to help: it died on startup this many times in
|
||||
;; a row, so the cause is the environment (config, missing artifact).
|
||||
;; Give up on this worker and, once none are left, stop accepting work
|
||||
;; instead of queueing requests nobody will ever pick up.
|
||||
(do
|
||||
(swap! wstate assoc :dead? true)
|
||||
(l/error :hint "wasm pool: worker keeps failing at startup, giving up"
|
||||
:restarts max-restarts
|
||||
:script (worker-script))
|
||||
(when (pool-dead?)
|
||||
(l/error :hint "wasm pool: no workers left, failing headless exports")
|
||||
(drain-pending! (ex-info "headless render pool is unavailable" {}))))
|
||||
(js/setTimeout (fn []
|
||||
(when-not (:stopping? @wstate)
|
||||
(spawn! wstate)
|
||||
(pump!)))
|
||||
(* restart-backoff-ms (inc (:failures @wstate 0)))))))
|
||||
|
||||
(defn- ensure-pool!
|
||||
[]
|
||||
(when (empty? @workers*)
|
||||
(let [size (pool-size)]
|
||||
(l/info :hint "wasm pool: starting" :workers size :script (worker-script))
|
||||
(reset! workers* (vec (for [_ (range size)]
|
||||
(spawn! (atom {}))))))))
|
||||
|
||||
;; --- dispatch
|
||||
|
||||
(defn- dispatch!
|
||||
[wstate job]
|
||||
(swap! wstate assoc :job job)
|
||||
(arm-timer! wstate)
|
||||
(.postMessage ^js (:worker @wstate)
|
||||
#js {:type "render"
|
||||
:id (:id job)
|
||||
:params (t/encode-str (:params job))}))
|
||||
|
||||
(defn- pump!
|
||||
"Hands queued requests to idle workers, oldest request first."
|
||||
[]
|
||||
(loop []
|
||||
(when-let [wstate (->> @workers*
|
||||
(filter (fn [w] (and (:worker @w) (nil? (:job @w)))))
|
||||
(first))]
|
||||
(when-let [job (peek @pending*)]
|
||||
(swap! pending* pop)
|
||||
(dispatch! wstate job)
|
||||
(recur)))))
|
||||
|
||||
(defn render
|
||||
"Runs one export request on a pool worker. `on-object` is called with each
|
||||
rendered object plus its tempfile `:path`, in the order the worker produced
|
||||
them. Resolves once the request is done and every `on-object` has settled."
|
||||
[params on-object]
|
||||
(ensure-pool!)
|
||||
(p/create
|
||||
(fn [resolve reject]
|
||||
(if (pool-dead?)
|
||||
(reject (ex-info "headless render pool is unavailable" {}))
|
||||
(do
|
||||
(swap! pending* conj {:id (swap! req-id* inc)
|
||||
:params params
|
||||
:on-object on-object
|
||||
:chain (atom (p/resolved nil))
|
||||
:resolve resolve
|
||||
:reject reject})
|
||||
(pump!))))))
|
||||
|
||||
(defn stop
|
||||
"Terminates every worker. Needed on devenv hot reload, where otherwise each
|
||||
reload would leak a full pool."
|
||||
[]
|
||||
(let [workers @workers*]
|
||||
(reset! workers* [])
|
||||
(p/all (for [wstate workers]
|
||||
(do (swap! wstate assoc :stopping? true)
|
||||
(clear-timer! wstate)
|
||||
(when-let [worker (:worker @wstate)]
|
||||
(.terminate ^js worker)))))))
|
||||
447
exporter/src/app/wasm/render.cljs
Normal file
447
exporter/src/app/wasm/render.cljs
Normal file
@ -0,0 +1,447 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.wasm.render
|
||||
"Headless render pipeline. Runs inside a `app.wasm.worker` thread, never on
|
||||
the main thread: every step below is synchronous wasm work that would
|
||||
otherwise block the node event loop for the whole export.
|
||||
|
||||
Backs `app.renderer.wasm` (the Playwright backends' drop-in alternative),
|
||||
which dispatches here through `app.wasm.pool`. Pipeline per request:
|
||||
|
||||
init module (once) -> fetch shape bundle (backend RPC)
|
||||
-> serialize scene
|
||||
-> provision resources (team/google fonts + fallback fonts + images,
|
||||
enumerated by the host-agnostic app.render-wasm.{resources,fallback-fonts})
|
||||
-> relayout text (real font metrics) -> render each object -> tempfile
|
||||
|
||||
Handles png/jpeg/webp (Skia encodes all three natively) and pdf. `:svg` is
|
||||
routed to the browser backend by `app.renderer` — it needs vector markup, not
|
||||
a raster.
|
||||
|
||||
Each worker owns one wasm module (one design state, one global mem buffer)
|
||||
and handles one request at a time, so nothing here needs to guard against
|
||||
concurrent access."
|
||||
(:require
|
||||
["node:fs" :as fs]
|
||||
["undici" :as http]
|
||||
[app.common.data :as d]
|
||||
;; Required for side effects: these register the transit read handlers and
|
||||
;; deftype impls the `get-page` response is decoded into.
|
||||
[app.common.geom.matrix]
|
||||
[app.common.geom.point]
|
||||
[app.common.geom.rect]
|
||||
[app.common.logging :as l]
|
||||
[app.common.transit :as t]
|
||||
[app.common.types.fills.impl]
|
||||
[app.common.types.objects-map]
|
||||
[app.common.types.path.impl]
|
||||
[app.common.types.shape]
|
||||
[app.common.uri :as u]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.render-wasm.fallback-fonts :as fbf]
|
||||
[app.render-wasm.resources :as resources]
|
||||
[app.util.mime :as mime]
|
||||
[app.util.shell :as sh]
|
||||
[app.wasm :as wasm]
|
||||
[app.wasm.gfonts :as gfonts]
|
||||
[app.wasm.serialize :as serialize]
|
||||
[cuerdas.core :as str]
|
||||
[promesa.core :as p]))
|
||||
|
||||
;; --- module lifecycle (one lazily-initialized instance per worker thread)
|
||||
|
||||
(defonce ^:private module* (atom nil))
|
||||
|
||||
(defn- ensure-module!
|
||||
[]
|
||||
(or @module*
|
||||
(reset! module* (wasm/init!))))
|
||||
|
||||
;; --- backend endpoints
|
||||
;;
|
||||
;; Every fetch below targets the *internal* endpoint (`internal-uri`, falling
|
||||
;; back to `public-uri`), same as the Playwright backends and
|
||||
;; `handlers.resources`: in a real deployment the exporter reaches the backend
|
||||
;; over the container network, not through the public ingress.
|
||||
|
||||
(defn- internal-uri
|
||||
"Absolute URI for `path` on the internal (backend) endpoint."
|
||||
[path]
|
||||
(-> (cf/get-internal-uri)
|
||||
(u/ensure-path-slash)
|
||||
(u/join path)
|
||||
(str)))
|
||||
|
||||
(defn- rpc-headers
|
||||
"Auth headers for backend RPC calls (management key + bearer)."
|
||||
[token]
|
||||
#js {"Content-Type" "application/transit+json"
|
||||
"X-Shared-Key" (str "exporter " cf/management-key)
|
||||
"Authorization" (str "Bearer " token)})
|
||||
|
||||
(defn- asset-headers
|
||||
"Auth headers for `/assets/*` downloads. Cookie, not Bearer: those endpoints
|
||||
redirect to a presigned S3/minio URL, and a Bearer Authorization header makes
|
||||
S3 400 (\"multiple authentication types\")."
|
||||
[token]
|
||||
#js {"X-Shared-Key" (str "exporter " cf/management-key)
|
||||
"Cookie" (str "auth-token=" token)})
|
||||
|
||||
;; --- shape bundle fetch (backend RPC)
|
||||
|
||||
(defn- fetch-objects
|
||||
"Fetches the page's `objects` map from the backend via the `get-page` RPC,
|
||||
using the same auth the exporter uses elsewhere (management key + bearer)."
|
||||
[{:keys [file-id page-id share-id token]}]
|
||||
(let [headers (rpc-headers token)
|
||||
;; share-id is an OPTIONAL uuid on the backend; it must be omitted when
|
||||
;; absent, not sent as nil (nil fails the uuid schema).
|
||||
body (t/encode-str (cond-> {:file-id file-id
|
||||
:page-id page-id}
|
||||
share-id (assoc :share-id share-id)))
|
||||
uri (internal-uri "api/rpc/command/get-page")]
|
||||
(l/dbg :hint "wasm render: get-page"
|
||||
:uri uri
|
||||
:file-id (str file-id)
|
||||
:page-id (str page-id))
|
||||
(->> (http/fetch uri #js {:method "POST" :headers headers :body body})
|
||||
(p/mcat (fn [^js resp]
|
||||
(if (= 200 (.-status resp))
|
||||
(.text resp)
|
||||
;; Surface the backend's actual error body so we can tell
|
||||
;; auth (401) from bad params (400/404).
|
||||
(->> (.text resp)
|
||||
(p/mcat (fn [resp-body]
|
||||
(l/error :hint "wasm render: get-page failed"
|
||||
:status (.-status resp)
|
||||
:body resp-body)
|
||||
(p/rejected (ex-info "get-page failed"
|
||||
{:status (.-status resp)
|
||||
:body resp-body}))))))))
|
||||
(p/fmap t/decode-str)
|
||||
(p/fmap :objects))))
|
||||
|
||||
;; --- font resolution
|
||||
;;
|
||||
;; Custom (team) fonts: the exporter's text serializer keeps their real font
|
||||
;; uuid, so `wasm/fonts-for-shape` reports it. We fetch the file's team font
|
||||
;; variants once (get-font-variants RPC), map uuid+weight+style -> ttf asset id,
|
||||
;; and download the TTF from `assets/by-id/<id>`. Google/builtin fonts still fall
|
||||
;; back to the bundled default (they need the gfonts catalog — a later slice).
|
||||
|
||||
(defn- fetch-font-variants
|
||||
"Fetches the file's team (custom) font variants via the get-font-variants RPC.
|
||||
Returns a promise of the variant vector (or nil on failure — fonts then just
|
||||
fall back, they don't fail the export)."
|
||||
[{:keys [file-id share-id token]}]
|
||||
(let [headers (rpc-headers token)
|
||||
body (t/encode-str (cond-> {:file-id file-id}
|
||||
share-id (assoc :share-id share-id)))
|
||||
uri (internal-uri "api/rpc/command/get-font-variants")]
|
||||
(->> (http/fetch uri #js {:method "POST" :headers headers :body body})
|
||||
(p/mcat (fn [^js resp]
|
||||
(if (= 200 (.-status resp))
|
||||
(.text resp)
|
||||
(p/resolved nil))))
|
||||
(p/fmap (fn [s] (when s (t/decode-str s))))
|
||||
(p/merr (fn [cause]
|
||||
(l/warn :hint "wasm render: get-font-variants failed" :cause cause)
|
||||
(p/resolved nil))))))
|
||||
|
||||
(defn- fetch-asset-bytes
|
||||
"Downloads a stored asset (font TTF) by id, returning a promise of an
|
||||
ArrayBuffer (or nil)."
|
||||
[asset-id {:keys [token]}]
|
||||
(let [headers (asset-headers token)
|
||||
uri (internal-uri (str "assets/by-id/" asset-id))]
|
||||
(->> (http/fetch uri #js {:method "GET" :headers headers})
|
||||
(p/mcat (fn [^js resp]
|
||||
(if (= 200 (.-status resp))
|
||||
(.arrayBuffer resp)
|
||||
(p/resolved nil))))
|
||||
(p/merr (fn [cause]
|
||||
(l/warn :hint "wasm render: font asset fetch failed"
|
||||
:asset-id (str asset-id) :cause cause)
|
||||
(p/resolved nil))))))
|
||||
|
||||
(defn- gfont-proxy-url
|
||||
"Rewrites a gstatic ttf url to the local gfonts proxy (mirrors the browser's
|
||||
`google-font-ttf-url`)."
|
||||
[ttf-url]
|
||||
(let [proxy (internal-uri "internal/gfonts/font/")]
|
||||
(str/replace ttf-url "https://fonts.gstatic.com/s/" proxy)))
|
||||
|
||||
(defn- fetch-gfont-bytes
|
||||
"Downloads a google font TTF through the local gfonts proxy."
|
||||
[ttf-url]
|
||||
(let [uri (gfont-proxy-url ttf-url)]
|
||||
(->> (http/fetch uri #js {:method "GET"})
|
||||
(p/mcat (fn [^js resp]
|
||||
(if (= 200 (.-status resp))
|
||||
(.arrayBuffer resp)
|
||||
(p/resolved nil))))
|
||||
(p/merr (fn [cause]
|
||||
(l/warn :hint "wasm render: gfont fetch failed" :url uri :cause cause)
|
||||
(p/resolved nil))))))
|
||||
|
||||
(defn- make-resolve-font
|
||||
"Builds a `resolve-font` fn (family map -> promise of TTF bytes). Tries the
|
||||
file's custom (team) font variants first — matching uuid + weight + style,
|
||||
degrading to uuid+weight then uuid — then the shared google catalog. Returns
|
||||
nil for fonts not found (builtin falls back to the bundled default)."
|
||||
[variants params]
|
||||
(fn [{:keys [id weight style]}]
|
||||
(let [font-uuid (uuid/from-unsigned-parts (aget id 0) (aget id 1) (aget id 2) (aget id 3))
|
||||
style-str (if (zero? style) "normal" "italic")
|
||||
variant (or (d/seek (fn [v] (and (= (:font-id v) font-uuid)
|
||||
(= (:font-weight v) weight)
|
||||
(= (name (:font-style v)) style-str)))
|
||||
variants)
|
||||
(d/seek (fn [v] (and (= (:font-id v) font-uuid)
|
||||
(= (:font-weight v) weight)))
|
||||
variants)
|
||||
(d/seek (fn [v] (= (:font-id v) font-uuid)) variants))]
|
||||
(if-let [ttf-id (:ttf-file-id variant)]
|
||||
(fetch-asset-bytes ttf-id params)
|
||||
(if-let [gurl (gfonts/resolve-ttf-url font-uuid weight style)]
|
||||
(fetch-gfont-bytes gurl)
|
||||
(p/resolved nil))))))
|
||||
|
||||
;; --- fallback fonts (emoji + per-script noto fonts)
|
||||
;;
|
||||
;; Emoji and non-latin scripts render through fallback families in the wasm
|
||||
;; font store, not through any span's font family — so `wasm/fonts-for-shape`
|
||||
;; never reports them and the per-object provisioning above never uploads
|
||||
;; them. The browser workspace uploads them as a side effect of serializing
|
||||
;; text (`add-emoji-font` / `add-noto-fonts`), which is why client-side single
|
||||
;; exports show emoji/CJK while headless exports dropped them. Which fonts a
|
||||
;; scene needs is computed by the host-agnostic
|
||||
;; `app.render-wasm.fallback-fonts` (same data the browser uses).
|
||||
;; `clear-fonts!` resets the store every request, so this must run per
|
||||
;; request; the TTF bytes are cached per font for the process lifetime.
|
||||
|
||||
(defn- scene-fallback-fonts
|
||||
"Fallback font descriptors needed by the scene's text content. Deduped:
|
||||
several languages share a noto family (`gfont-noto-sans` covers cyrillic,
|
||||
greek, devanagari, latin-ext and vietnamese), and the descriptors are
|
||||
provisioned concurrently — without this they'd all miss the byte cache at
|
||||
once and download the same TTF several times."
|
||||
[scene]
|
||||
(let [texts (for [shape (vals scene)
|
||||
:when (= :text (:type shape))
|
||||
node (or (some->> (:content shape) (tree-seq :children :children)) [])
|
||||
:let [text (:text node)]
|
||||
:when (string? text)]
|
||||
text)
|
||||
emoji? (boolean (some fbf/contains-emoji? texts))
|
||||
langs (reduce fbf/collect-used-languages #{} texts)]
|
||||
(distinct
|
||||
(cond-> (fbf/add-noto-fonts [] langs)
|
||||
emoji? (fbf/add-emoji-font)))))
|
||||
|
||||
(defonce ^:private fallback-font-bytes* (atom {}))
|
||||
|
||||
(defn- fetch-fallback-font-bytes
|
||||
"Downloads (and caches for the process lifetime) one fallback font's TTF."
|
||||
[{:keys [font-id weight style]}]
|
||||
(if-let [bytes (get @fallback-font-bytes* font-id)]
|
||||
(p/resolved bytes)
|
||||
(let [font-uuid (gfonts/gfont-id->uuid font-id)
|
||||
ttf-url (some-> font-uuid (gfonts/resolve-ttf-url weight style))]
|
||||
(if ttf-url
|
||||
(->> (fetch-gfont-bytes ttf-url)
|
||||
(p/fmap (fn [buf]
|
||||
(when buf (swap! fallback-font-bytes* assoc font-id buf))
|
||||
buf)))
|
||||
(p/resolved nil)))))
|
||||
|
||||
(defn- provision-fallback-fonts!
|
||||
[scene]
|
||||
(->> (scene-fallback-fonts scene)
|
||||
(map (fn [{:keys [font-id weight style is-emoji is-fallback] :as font}]
|
||||
(if-let [font-uuid (gfonts/gfont-id->uuid font-id)]
|
||||
(->> (fetch-fallback-font-bytes font)
|
||||
(p/fmap (fn [buf]
|
||||
(if buf
|
||||
(wasm/store-font! {:id (uuid/get-u32 font-uuid)
|
||||
:weight weight
|
||||
:style style
|
||||
:emoji? (boolean is-emoji)
|
||||
:fallback? (boolean is-fallback)}
|
||||
buf)
|
||||
(l/warn :hint "wasm render: fallback font unavailable"
|
||||
:font-id font-id)))))
|
||||
(p/resolved nil))))
|
||||
(p/all)))
|
||||
|
||||
;; --- image resolution
|
||||
;;
|
||||
;; Image fills reference file-media ids. We collect them from the scene, fetch
|
||||
;; the encoded bytes from `assets/by-file-media-id/<id>`, and hand them to
|
||||
;; `_store_image` (Skia decodes them; no WebGL). Keyed by media uuid, so this is
|
||||
;; done once per request rather than per rendered object.
|
||||
|
||||
(defn- fetch-file-media-bytes
|
||||
"Downloads an image fill's encoded bytes by file-media id."
|
||||
[media-id {:keys [token]}]
|
||||
(let [headers (asset-headers token)
|
||||
uri (internal-uri (str "assets/by-file-media-id/" media-id))]
|
||||
(->> (http/fetch uri #js {:method "GET" :headers headers})
|
||||
(p/mcat (fn [^js resp]
|
||||
(if (= 200 (.-status resp))
|
||||
(.arrayBuffer resp)
|
||||
(do
|
||||
(l/warn :hint "wasm render: image fetch non-200"
|
||||
:media-id (str media-id)
|
||||
:uri uri
|
||||
:status (.-status resp))
|
||||
(p/resolved nil)))))
|
||||
(p/merr (fn [cause]
|
||||
(l/warn :hint "wasm render: image fetch failed"
|
||||
:media-id (str media-id) :uri uri :cause cause)
|
||||
(p/resolved nil))))))
|
||||
|
||||
(defn- provision-images!
|
||||
"Fetches and stores every image the scene references — shape fills, stroke
|
||||
image fills, and text-span image fills (enumerated by the host-agnostic
|
||||
`app.render-wasm.resources`, same source the workspace uses). Images the
|
||||
module already holds are skipped: unlike fonts, the image store is not
|
||||
reset per request, so repeated exports of the same file reuse them.
|
||||
NOTE: that also means the store grows with every distinct image the
|
||||
process ever exports; if exporter memory becomes a problem, add an
|
||||
eviction policy on the Rust side rather than clearing per request."
|
||||
[scene params]
|
||||
(let [all-ids (resources/scene-image-ids scene)
|
||||
new-ids (remove wasm/image-cached? all-ids)]
|
||||
(l/dbg :hint "wasm render: provisioning images"
|
||||
:total (count all-ids)
|
||||
:cached (- (count all-ids) (count new-ids)))
|
||||
(->> new-ids
|
||||
(map (fn [image-id]
|
||||
(->> (fetch-file-media-bytes image-id params)
|
||||
(p/fmap (fn [buf]
|
||||
(if buf
|
||||
(do
|
||||
(l/dbg :hint "wasm render: image stored"
|
||||
:media-id (str image-id)
|
||||
:bytes (.-byteLength ^js buf))
|
||||
(wasm/store-image! image-id buf))
|
||||
(l/warn :hint "wasm render: image unavailable"
|
||||
:media-id (str image-id))))))))
|
||||
(p/all))))
|
||||
|
||||
(defn- relayout-text!
|
||||
"Recomputes layout for every text shape in the scene. Called after fonts are
|
||||
provisioned so metrics use the real fonts (serialize-time layout used the
|
||||
fallback)."
|
||||
[scene]
|
||||
(doseq [shape (vals scene)
|
||||
:when (= :text (:type shape))]
|
||||
(wasm/update-text-layout! (:id shape))))
|
||||
|
||||
;; --- render
|
||||
|
||||
(defn- render-object-bytes
|
||||
[type id scale]
|
||||
(if (= :pdf type)
|
||||
(let [bytes (wasm/render-shape-pdf id scale)]
|
||||
(l/dbg :hint "PDF generated via Skia (render-wasm headless)"
|
||||
:object-id (str id)
|
||||
:backend "skia-wasm"
|
||||
:bytes (.-length bytes))
|
||||
bytes)
|
||||
;; The export type doubles as the encoder format: Skia encodes png/jpeg/webp
|
||||
;; natively, so — unlike the Playwright backend — webp needs no imagemagick
|
||||
;; conversion pass. `:svg` never reaches here; it needs vector markup, so
|
||||
;; `app.renderer` keeps it on the browser path.
|
||||
(wasm/render-shape-raster id scale type)))
|
||||
|
||||
;; NOTE: `handlers.export-shapes/prepare-exports` splits an export into
|
||||
;; partitions of 50 objects, and each partition arrives here as its own
|
||||
;; request. Since the unit of work is the partition, every one of them
|
||||
;; re-fetches and re-serializes the *whole* page scene, and re-provisions its
|
||||
;; fonts. Exporting 200 frames therefore pays that setup 4x — and with the pool
|
||||
;; dispatching partitions round-robin, those 4 setups now land on 4 different
|
||||
;; workers, so none of them share a warm image cache either. Worth revisiting
|
||||
;; if setup cost shows up in practice.
|
||||
|
||||
(defn render
|
||||
"Runs one export request to completion. `emit-object!` is called once per
|
||||
rendered object with the object map plus a `:path` to its tempfile; it may
|
||||
return a promise, which is awaited before the next object renders."
|
||||
[{:keys [scale type objects] :as params} emit-object!]
|
||||
(l/dbg :hint "wasm render: start"
|
||||
:type type
|
||||
:scale scale
|
||||
:objects (count objects)
|
||||
:file-id (str (:file-id params))
|
||||
:page-id (str (:page-id params)))
|
||||
(->> (ensure-module!)
|
||||
(p/mcat (fn [_] (fetch-objects params)))
|
||||
(p/mcat (fn [scene]
|
||||
(l/dbg :hint "wasm render: scene fetched" :shapes (count scene))
|
||||
(serialize/serialize-scene! scene)
|
||||
(l/dbg :hint "wasm render: scene serialized")
|
||||
;; Reset the shared module's font store so fonts from a previous
|
||||
;; request don't accumulate / leak into this one.
|
||||
(wasm/clear-fonts!)
|
||||
;; Fetch the file's custom (team) font variants once, provision
|
||||
;; every referenced image once, then resolve/provision the
|
||||
;; deduped font set for all rendered objects.
|
||||
(->> (p/all [(fetch-font-variants params)
|
||||
(provision-images! scene params)
|
||||
(provision-fallback-fonts! scene)])
|
||||
(p/mcat
|
||||
(fn [[variants _]]
|
||||
(let [resolve-font (make-resolve-font (or variants []) params)]
|
||||
;; Provision every object's fonts BEFORE rendering, so
|
||||
;; the text relayout below sees real font metrics.
|
||||
;; Deduped across objects: a 50-frame partition that
|
||||
;; shares one family downloads its TTF once, not 50
|
||||
;; times.
|
||||
(wasm/provision-fonts! (map :id objects) resolve-font))))
|
||||
(p/mcat
|
||||
(fn [_]
|
||||
;; Serialize-time layout used the fallback font (fonts
|
||||
;; weren't uploaded yet); recompute now that they are, or
|
||||
;; text metrics/line breaks are wrong.
|
||||
(relayout-text! scene)
|
||||
(p/run
|
||||
(fn [{:keys [id] :as object}]
|
||||
(let [bytes (render-object-bytes type id scale)
|
||||
path (sh/tempfile :prefix "penpot.tmp.wasm."
|
||||
:suffix (mime/get-extension type))]
|
||||
(l/dbg :hint "wasm render: object rendered"
|
||||
:object-id (str id) :bytes (.-length bytes))
|
||||
(fs/writeFileSync path bytes)
|
||||
;; `emit-object!` may return a plain value or a
|
||||
;; promise; `p/do` normalizes both to a thenable so
|
||||
;; `p/mcat` doesn't throw "expected thenable".
|
||||
(p/do (emit-object! (assoc object :path path)))))
|
||||
objects))))))
|
||||
(p/fmap (fn [result]
|
||||
;; Trim the image store AFTER the request (never mid-render, so
|
||||
;; an image can't disappear under a running export). Images the
|
||||
;; next request needs again are simply re-provisioned.
|
||||
(let [evicted (wasm/evict-images! (wasm/image-cache-mb))]
|
||||
(when (pos? evicted)
|
||||
(l/info :hint "wasm render: evicted cached images" :count evicted)))
|
||||
result))
|
||||
(p/merr (fn [cause]
|
||||
;; undici hides the real network failure (ECONNREFUSED, a TLS
|
||||
;; rejection, ...) in `.cause`, and without it every failure
|
||||
;; reads as a bare "fetch failed".
|
||||
(l/error :hint "wasm render: failed"
|
||||
:reason (some-> (unchecked-get cause "cause")
|
||||
(unchecked-get "code"))
|
||||
:cause cause)
|
||||
;; A panic/abort can leave the module's buffer allocated or the
|
||||
;; wasm instance aborted; drop it so the next request rebuilds a
|
||||
;; fresh module instead of inheriting the bad state.
|
||||
(reset! module* nil)
|
||||
(p/rejected cause)))))
|
||||
61
exporter/src/app/wasm/worker.cljs
Normal file
61
exporter/src/app/wasm/worker.cljs
Normal file
@ -0,0 +1,61 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.wasm.worker
|
||||
"Entry point of the headless render worker thread.
|
||||
|
||||
Owns one wasm module and runs `app.wasm.render` on it. Kept deliberately
|
||||
thin: everything here is message plumbing, so the pipeline itself stays
|
||||
free of worker concerns and the module never touches the main thread.
|
||||
|
||||
Protocol (see `app.wasm.pool` for the other end). Params and objects cross
|
||||
the boundary transit-encoded, because they carry uuids and keywords that
|
||||
structured clone would mangle:
|
||||
|
||||
in {type: \"render\", id, params}
|
||||
out {type: \"object\", id, object, path} one per rendered object
|
||||
out {type: \"done\", id}
|
||||
out {type: \"error\", id, message}
|
||||
|
||||
The pool sends one request at a time per worker, so there is no need to
|
||||
multiplex `id`s here — it is echoed back only so the pool can drop replies
|
||||
from a request it already gave up on."
|
||||
(:require
|
||||
["node:worker_threads" :as wt]
|
||||
[app.common.logging :as l]
|
||||
[app.common.transit :as t]
|
||||
[app.wasm.render :as render]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(enable-console-print!)
|
||||
(l/setup! {:app :info})
|
||||
|
||||
(defn- post!
|
||||
[msg]
|
||||
(.postMessage ^js wt/parentPort msg))
|
||||
|
||||
(defn- handle-render!
|
||||
[id params]
|
||||
(->> (render/render (t/decode-str params)
|
||||
(fn [object]
|
||||
(post! #js {:type "object"
|
||||
:id id
|
||||
:object (t/encode-str (dissoc object :path))
|
||||
:path (:path object)})))
|
||||
(p/fnly (fn [_ cause]
|
||||
(if cause
|
||||
(post! #js {:type "error"
|
||||
:id id
|
||||
:message (or (ex-message cause) (str cause))})
|
||||
(post! #js {:type "done" :id id}))))))
|
||||
|
||||
(defn main
|
||||
[& _]
|
||||
(.on ^js wt/parentPort "message"
|
||||
(fn [^js msg]
|
||||
(when (= "render" (unchecked-get msg "type"))
|
||||
(handle-render! (unchecked-get msg "id")
|
||||
(unchecked-get msg "params"))))))
|
||||
Loading…
x
Reference in New Issue
Block a user