github/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2026-09-22 03:46:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Elena Torro 2026-09-16 12:29:10 +02:00
parent 286ccb03fa
commit 8455fcf170
19 changed files with 760 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/app/media/validation.clj
View File

@ -33,7 +33,9 @@
(sm/check-fn schema:input))
(defn validate-media-type!
([upload] (validate-media-type! upload cm/image-types))
;; `upload-types` rather than `image-types`: a file can also hold video.
;; Callers that must stay image-only pass `cm/image-types` explicitly.
([upload] (validate-media-type! upload cm/upload-types))
([upload allowed]
(when-not (contains? allowed (:mtype upload))
(ex/raise :type :validation

37
backend/src/app/rpc/commands/media.clj
View File

@ -9,6 +9,7 @@
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.logging :as l]
[app.common.media :as cm]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.uuid :as uuid]
@ -52,7 +53,11 @@
[:file-id ::sm/uuid]
[:is-local ::sm/boolean]
[:name [:string {:max 250}]]
[:content media.v/schema:upload]])
[:content media.v/schema:upload]
;; Only video sends these: nothing on the backend can read them from the
;; file, so the client reports what it decoded.
[:width {:optional true} ::sm/int]
[:height {:optional true} ::sm/int]])
(sv/defmethod ::upload-file-media-object
{::doc/added "1.17"
@ -170,9 +175,31 @@
:always
(assoc ::image (process-main-image info)))))
(defn- process-video
"Video is stored and served, never decoded: nothing here reads its
dimensions or renders a frame to thumbnail. The client sends the dimensions
it read from the file. Shaped like `process-image` so the caller does not
care which one ran."
[content width height]
(when-not (and width height)
(ex/raise :type :validation
:code :missing-video-dimensions
:hint "video uploads must report the dimensions the client decoded"))
(let [info (assoc content
:ts (ct/now)
:width width
:height height)]
(assoc info ::image (process-main-image info))))
(defn- process-content
[cfg {:keys [mtype] :as content} width height]
(if (cm/video-type? mtype)
(process-video content width height)
(process-image cfg content)))
(defn- create-file-media-object
[{:keys [::sto/storage ::db/conn] :as cfg}
{:keys [id file-id is-local name content from-url? from-chunks?]}]
{:keys [id file-id is-local name content width height from-url? from-chunks?]}]
(let [tpoint (ct/tpoint)
id (or id (uuid/next))
@ -192,7 +219,7 @@
:path (str (:path content))
:origin origin)
(let [result (process-image cfg content)
(let [result (process-content cfg content width height)
image (sto/put-object! storage (::image result))
thumb (when-let [params (::thumb result)]
(sto/put-object! storage params))
@ -453,7 +480,9 @@
[:is-local ::sm/boolean]
[:name [:string {:max 250}]]
[:mtype :string]
[:id {:optional true} ::sm/uuid]])
[:id {:optional true} ::sm/uuid]
[:width {:optional true} ::sm/int]
[:height {:optional true} ::sm/int]])
(sv/defmethod ::assemble-file-media-object
{::doc/added "2.17"

26
common/src/app/common/media.cljc
View File

@ -22,6 +22,20 @@
"image/gif"
"image/svg+xml"})
(def video-types
#{"video/mp4"
"video/webm"})
(defn video-type?
[mtype]
(contains? video-types mtype))
;; Media a file can hold. Video is a proof of concept: the backend stores and
;; serves it but decodes nothing, so it carries the dimensions the client read
;; from the file.
(def upload-types
(into image-types video-types))
(def tempfile-types
(conj image-types "application/pdf" "application/zip"))
@ -32,7 +46,9 @@
:jpeg ".jpg"
:webp ".webp"
:gif ".gif"
:svg ".svg"))
:svg ".svg"
:mp4 ".mp4"
:webm ".webm"))
(defn format->mtype
[format]
@ -43,6 +59,8 @@
:webp "image/webp"
:gif "image/gif"
:svg "image/svg+xml"
:mp4 "video/mp4"
:webm "video/webm"
"application/octet-stream"))
(defn mtype->format
@ -53,6 +71,8 @@
"image/webp" :webp
"image/gif" :gif
"image/svg+xml" :svg
"video/mp4" :mp4
"video/webm" :webm
nil))
(defn mtype->extension [mtype]
@ -65,6 +85,8 @@
"image/png" ".png"
"image/svg+xml" ".svg"
"image/webp" ".webp"
"video/mp4" ".mp4"
"video/webm" ".webm"
"application/zip" ".zip"
"application/penpot" ".penpot"
"application/pdf" ".pdf"
@ -78,7 +100,7 @@
(defn strip-image-extension
[filename]
(let [image-extensions-re #"(\.png)|(\.jpg)|(\.jpeg)|(\.webp)|(\.gif)|(\.svg)$"]
(let [image-extensions-re #"(\.png)|(\.jpg)|(\.jpeg)|(\.webp)|(\.gif)|(\.svg)|(\.mp4)|(\.webm)$"]
(str/replace filename image-extensions-re "")))
(defn parse-font-weight

9
common/src/app/common/types/fills/impl.cljc
View File

@ -160,7 +160,9 @@
"image/png" 0x02
"image/gif" 0x03
"image/webp" 0x04
"image/svg+xml" 0x05)]
"image/svg+xml" 0x05
"video/mp4" 0x06
"video/webm" 0x07)]
(buf/write-short buffer (+ offset 2) val)))
(if (and (some? ref-file)
@ -241,7 +243,10 @@
0x02 "image/png"
0x03 "image/gif"
0x04 "image/webp"
0x05 "image/svg+xml")]
0x05 "image/svg+xml"
0x06 "video/mp4"
0x07 "video/webm"
nil)]
{:fill-opacity opacity
:fill-image (cond-> {:id id
:width width

10
common/src/app/common/types/shape.cljc
View File

@ -276,7 +276,13 @@
[:svg-attrs {:optional true} :map]
[:svg-defs {:optional true} :map]
[:svg-transform {:optional true} :map]
[:svg-viewbox {:optional true} :map]])
[:svg-viewbox {:optional true} :map]
;; Video proof of concept: the source played into the shape's image fill,
;; either a name under the static asset folder or a full URL. Only the
;; render-wasm renderer paints it; every other consumer keeps showing the
;; image fill, which doubles as the poster frame.
[:video {:optional true} [:maybe :string]]])
(def schema:group-attrs
[:map {:title "GroupAttrs"}
@ -516,7 +522,7 @@
:hidden :masked-group :fills :proportion :proportion-lock :constraints-h
:constraints-v :fixed-scroll :r1 :r2 :r3 :r4 :rotation :opacity :grids :exports
:strokes :blend-mode :interactions :shadow :blur :background-blur :grow-type :applied-tokens
:plugin-data})
:plugin-data :video})
(def ^:private allowed-shape-geom-attrs #{:x :y :width :height})
(def ^:private allowed-shape-base-attrs #{:id :name :type :selrect :points :transform

2
docker/devenv/files/nginx.conf
View File

@ -278,7 +278,7 @@ http {
add_header X-Cache-Status $upstream_cache_status;
}
location ~* \.(jpg|png|svg|ttf|woff|woff2|gif)$ {
location ~* \.(jpg|png|svg|ttf|woff|woff2|gif|mp4|webm|mov)$ {
include /home/penpot/penpot/docker/devenv/files/nginx-security-headers.conf;
add_header Cache-Control "public, max-age=604800" always; # 7 days
}

2
frontend/src/app/main/data/media.cljs
View File

@ -40,7 +40,7 @@
(defn validate-file
"Check that a file obtained with the file javascript API is valid."
[file]
(when-not (contains? cm/image-types (.-type file))
(when-not (contains? cm/upload-types (.-type file))
(ex/raise :type :validation
:code :media-type-not-allowed
:hint (str/ffmt "media type % is not supported" (.-type file))))

67
frontend/src/app/main/data/workspace/media.cljs
View File

@ -38,7 +38,32 @@
[tubax.core :as tubax]))
(def accept-image-types
(str/join "," media/image-types))
(str/join "," media/upload-types))
(defn- read-video-dimensions
"Resolves the intrinsic size of a video blob. Nothing on the backend decodes
video, so the client is what reports the dimensions an upload is stored
with."
[blob]
(p/create
(fn [resolve reject]
(let [url (js/URL.createObjectURL blob)
element (js/document.createElement "video")
done (fn [f value]
(js/URL.revokeObjectURL url)
(f value))]
(set! (.-preload element) "metadata")
(set! (.-muted element) true)
(set! (.-onloadedmetadata element)
(fn [_]
(done resolve {:width (.-videoWidth element)
:height (.-videoHeight element)})))
(set! (.-onerror element)
(fn [_]
(done reject (ex/error :type :validation
:code :invalid-video
:hint "could not read the video dimensions"))))
(set! (.-src element) url)))))
(defn- optimize
[input]
@ -123,17 +148,19 @@
"Uploads `blob` to `file-id` as a chunked media object using the
three-step session API. Returns an observable that emits the
assembled file-media-object map."
[{:keys [file-id name is-local blob]}]
[{:keys [file-id name is-local blob width height]}]
(let [mtype (.-type blob)]
(->> (uploads/upload-blob-chunked blob)
(rx/mapcat
(fn [{:keys [session-id]}]
(rp/cmd! :assemble-file-media-object
{:session-id session-id
:file-id file-id
:is-local is-local
:name name
:mtype mtype}))))))
(cond-> {:session-id session-id
:file-id file-id
:is-local is-local
:name name
:mtype mtype}
(some? width) (assoc :width width)
(some? height) (assoc :height height))))))))
(defn process-uris
[{:keys [file-id local? name uris mtype on-image on-svg]}]
@ -175,18 +202,26 @@
(and (not force-media)
(= (.-type blob) "image/svg+xml")))
(upload-blob [blob]
(let [params {:file-id file-id
:name (or name (if (dmm/file? blob) (media/strip-image-extension (.-name blob)) "blob"))
:is-local local?
:blob blob}]
(upload-blob* [blob dimensions]
(let [params (merge {:file-id file-id
:name (or name (if (dmm/file? blob) (media/strip-image-extension (.-name blob)) "blob"))
:is-local local?
:blob blob}
dimensions)]
(if (>= (.-size blob) chunk-size)
(upload-blob-chunked params)
(rp/cmd! :upload-file-media-object
{:file-id file-id
:name (:name params)
:is-local local?
:content blob}))))
(merge {:file-id file-id
:name (:name params)
:is-local local?
:content blob}
dimensions)))))
(upload-blob [blob]
(if (media/video-type? (.-type blob))
(->> (rx/from (read-video-dimensions blob))
(rx/mapcat #(upload-blob* blob %)))
(upload-blob* blob nil)))
(extract-content [blob]
(let [name (or name (.-name blob))]

92
frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs Normal file
View File

@ -0,0 +1,92 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns app.main.ui.workspace.sidebar.options.menus.video
"Proof of concept: plays a video into a shape's image fill.
Nothing uploads video, so the source is a path the browser can already
reach — a file under `frontend/resources/public/images/` or a full URL. Only
the render-wasm renderer paints it; elsewhere the image fill still shows."
(:require-macros [app.main.style :as stl])
(:require
[app.main.data.workspace.shapes :as dwsh]
[app.main.store :as st]
[app.main.ui.components.title-bar :refer [title-bar*]]
[app.main.ui.ds.buttons.icon-button :refer [icon-button*]]
[app.main.ui.ds.controls.input :refer [input*]]
[app.main.ui.ds.foundations.assets.icon :as i]
[app.render-wasm.api.video :as video]
[app.util.dom :as dom]
[app.util.i18n :refer [tr]]
[clojure.string :as str]
[rumext.v2 :as mf]))
(defn set-video
"Stores the source on the shape. `app.render-wasm.shape` picks the change up
and attaches or detaches the video, so undo and reload both behave."
[ids source]
(dwsh/update-shapes ids (fn [shape]
(if (str/blank? source)
(dissoc shape :video)
(assoc shape :video source)))))
(mf/defc video-menu*
[{:keys [ids image-id source is-asset]}]
(let [;; `video/playing?` reads the element, which is outside app state, so
;; the button tracks it locally.
playing* (mf/use-state #(video/playing? image-id))
playing (deref playing*)
has-source (not (str/blank? source))
on-change
(mf/use-fn
(mf/deps ids)
(fn [event]
(let [value (-> event dom/get-target dom/get-value str/trim)]
(st/emit! (set-video ids value)))))
on-toggle-play
(mf/use-fn
(mf/deps image-id)
(fn []
(reset! playing* (video/toggle-play! image-id))))
on-remove
(mf/use-fn
(mf/deps ids)
(fn []
(st/emit! (set-video ids nil))))]
[:section {:class (stl/css :element-set)
:aria-label (tr "workspace.options.video")}
[:div {:class (stl/css :element-title)}
[:> title-bar* {:collapsable false
:title (tr "workspace.options.video")}
(when has-source
[:> icon-button* {:variant "ghost"
:aria-label (if playing
(tr "workspace.options.video.pause")
(tr "workspace.options.video.play"))
:on-click on-toggle-play
:selected playing
:tooltip-placement "top-left"
:icon i/play}])]]
;; An uploaded video has nothing to type: its source is the asset itself,
;; and removing it means deleting the shape.
(when-not is-asset
[:div {:class (stl/css :row)}
[:> input* {:class (stl/css :source-input)
:placeholder (tr "workspace.options.video.placeholder")
:default-value (or source "")
:aria-label (tr "workspace.options.video.source")
:on-blur on-change}]
[:> icon-button* {:variant "ghost"
:aria-label (tr "workspace.options.video.remove")
:on-click on-remove
:disabled (not has-source)
:tooltip-placement "top-left"
:icon i/remove}]])]))

23
frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss Normal file
View File

@ -0,0 +1,23 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Copyright (c) KALEIDOS SUBSIDIARY SL
@use "../../../sidebar/common/sidebar.scss" as sidebar;
.element-set {
max-width: var(--options-width);
}
.row {
display: flex;
align-items: center;
gap: var(--sp-xs);
margin-bottom: var(--sp-s);
}
.source-input {
flex-grow: 1;
min-width: 0;
}

23
frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs
View File

@ -8,6 +8,7 @@
(:require
[app.common.data.macros :as dm]
[app.common.types.shape.layout :as ctl]
[app.main.features :as features]
[app.main.refs :as refs]
[app.main.ui.workspace.sidebar.options.menus.blur :refer [blur-menu*]]
[app.main.ui.workspace.sidebar.options.menus.constraints :refer [constraint-attrs constraints-menu*]]
@ -21,6 +22,8 @@
[app.main.ui.workspace.sidebar.options.menus.shadow :refer [shadow-menu*]]
[app.main.ui.workspace.sidebar.options.menus.stroke :refer [stroke-attrs stroke-menu*]]
[app.main.ui.workspace.sidebar.options.menus.svg-attrs :refer [svg-attrs-menu*]]
[app.main.ui.workspace.sidebar.options.menus.video :refer [video-menu*]]
[app.render-wasm.api.video :as video]
[rumext.v2 :as mf]))
(mf/defc options*
@ -80,7 +83,19 @@
(refs/parents-by-ids ids))
parents
(mf/deref parents-by-ids-ref)]
(mf/deref parents-by-ids-ref)
;; Video is painted by re-uploading frames into the shape's image fill,
;; and only the wasm renderer does that.
render-wasm?
(features/use-feature "render-wasm/v1")
video-image-id
(when render-wasm? (video/image-fill-id shape))
;; An uploaded video asset needs no source field, only playback.
video-asset?
(and render-wasm? (some? (video/video-fill shape)))]
[:*
[:> layer-menu* {:ids ids
@ -127,6 +142,12 @@
:values shape
:applied-tokens applied-tokens}]
(when (some? video-image-id)
[:> video-menu* {:ids ids
:image-id video-image-id
:source (video/shape-source shape)
:is-asset video-asset?}])
[:> stroke-menu* {:ids ids
:type type
:values stroke-values

47
frontend/src/app/render_wasm/api.cljs
View File

@ -16,6 +16,7 @@
[app.common.fonts :as cfnt]
[app.common.logging :as log]
[app.common.math :as mth]
[app.common.media :as cm]
[app.common.render-wasm.api.props :as props]
[app.common.render-wasm.api.upload :as upload]
[app.common.render-wasm.helpers :as h]
@ -45,6 +46,7 @@
[app.render-wasm.api.enums]
[app.render-wasm.api.fonts :as f]
[app.render-wasm.api.texts :as t]
[app.render-wasm.api.video :as video]
[app.render-wasm.api.webgl :as webgl]
[app.render-wasm.deserializers :as dr]
[app.render-wasm.gesture :as wasm-gesture]
@ -490,6 +492,10 @@
(defn- render
[timestamp]
(when (wasm/live?)
;; Upload pending video frames before drawing, so this frame paints the
;; current one instead of the previous.
(video/tick!)
;; SYNC-TILES makes WASM keep the last presented frame while the new tiles
;; are rasterized, rather than clearing to the background first. The flag is
;; one-shot on both sides: WASM clears it when the render loop starts, so
@ -510,6 +516,12 @@
(js/console.error "text-editor overlay/update failed:" e)))
(set! wasm/internal-frame-id nil)
;; Playing video has no other trigger to keep the loop alive. Requested
;; after the frame id is cleared so the scheduled frame is the one kept.
(when (video/active?)
(request-render "video-frame"))
(ug/dispatch! (ug/event "penpot:wasm:render"))))
(defn render-ui-only
@ -663,6 +675,8 @@
(throw e))))))]
(set! wasm/internal-frame-id frame-id))))))
(video/set-render-requester! request-render)
(defn request-render-preserving-target
"Like `request-render`, but keeps the previously presented frame on screen
while the new tiles are rasterized instead of blanking the canvas first.
@ -845,15 +859,6 @@
nil)
(defn- get-texture-id-for-gl-object
"Registers a WebGL texture with Emscripten's GL object system and returns its ID"
[texture]
(let [gl-obj (unchecked-get wasm/internal-module "GL")
textures (.-textures ^js gl-obj)
new-id (.getNewId ^js gl-obj textures)]
(aset textures new-id texture)
new-id))
(defn- svg-blob?
[^js blob]
(str/starts-with? (.-type blob) "image/svg"))
@ -905,7 +910,7 @@
[shape-id image-id thumbnail? img]
(when-let [gl (webgl/get-webgl-context)]
(let [texture (webgl/create-webgl-texture-from-image gl img)
texture-id (get-texture-id-for-gl-object texture)
texture-id (webgl/register-texture! texture)
width (.-width ^js img)
height (.-height ^js img)
;; Header: 32 bytes (2 UUIDs) + 4 bytes (thumbnail)
@ -964,6 +969,17 @@
:cause cause)
(rx/empty)))))}))
(defn- still-image-ids
"Image ids the renderer has to fetch. Video fills are skipped: their frames
are uploaded by `app.render-wasm.api.video`, so downloading the container as
an image would burn the bandwidth and then fail to decode."
[fills]
(into #{}
(comp (keep :fill-image)
(remove #(cm/video-type? (:mtype %)))
(map :id))
(seq fills)))
(defn- get-fill-images
[leaf]
(filter :fill-image (:fills leaf)))
@ -1018,7 +1034,7 @@
(store-image-url! id (cf/resolve-file-media {:id id} thumbnail?))
(when (zero? cached-image?)
(fetch-image shape-id id thumbnail?))))
(types.fills/get-image-ids fills))))))
(still-image-ids fills))))))
(defn- stroke-image-ids
[strokes]
@ -1743,6 +1759,9 @@
update-text-layouts fires for all text shapes after fonts load — not
just the first shape that triggered the fetch."
[shapes]
;; Newly added shapes never reach `set-wasm-attr!`, which is what starts a
;; video on an edit, so a shape created with one is started here instead.
(run! video/sync-shape! shapes)
(let [total-shapes (count shapes)
{:keys [thumbnails full text-font-state]}
(loop [index 0
@ -2035,6 +2054,9 @@
(set-objects objects render-callback nil false))
([objects render-callback on-shapes-ready force-sync]
(when (wasm/live?)
;; Start (and stop) the videos this page asks for. Cheap: one pass over
;; the objects, and attachments that did not change keep playing.
(video/sync-shapes! objects)
(perf/begin-measure "set-objects")
(let [shapes (shapes-in-tree-order objects)
total-shapes (count shapes)]
@ -2611,6 +2633,9 @@
:as payload}]
(ug/dispatch! (ug/event "penpot:wasm:reload-start"))
(reset! wasm/reloading? true)
;; Attached videos hold texture ids from the context that is about to be
;; destroyed; a frame uploaded after the reload would wrap a stale texture.
(video/detach-all!)
(let [fonts (derive-font-resources base-objects fonts)]
(-> (p/resolved nil)
;; Keep teardown strict (`_clean_up` + deleteContext) but do not

282
frontend/src/app/render_wasm/api/video.cljs Normal file
View File

@ -0,0 +1,282 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns app.render-wasm.api.video
"Proof of concept: video frames painted as image fills.
Skia has no video decoder, so the browser decodes and we only hand Skia a
texture. Each attached video owns one WebGL texture; on every rendered frame
the current frame is uploaded into that texture and WASM re-wraps it, so
every shape whose image fill points at `image-id` paints it.
The source is stored on the shape as `:video`, so it persists with the file
and comes back on load. Drop a file into `frontend/resources/public/images/`,
where it is served like any other static asset, and name it from the Video
section of the design sidebar. `penpotAttachVideo(\"clip.mp4\")` does the same
from the console for the selected shape, without persisting it.
There is no backend support: nothing uploads video, so the source is a path
the browser can reach on its own."
(:require
[app.common.logging :as log]
[app.common.media :as cm]
[app.common.render-wasm.helpers :as h]
[app.common.render-wasm.mem :as mem]
[app.common.render-wasm.mem.heap32 :as mem.h32]
[app.common.render-wasm.wasm :as wasm]
[app.common.types.fills :as types.fills]
[app.config :as cf]
[app.main.data.helpers :as dsh]
[app.main.store :as st]
[app.render-wasm.api.webgl :as webgl]
[clojure.string :as str]))
;; HTMLMediaElement.readyState: the element has data for the current position.
(def ^:private HAVE-CURRENT-DATA 2)
;; Videos live next to the other static assets, so a bare file name is enough.
(def ^:private assets-path "/images/")
;; image-id -> {:element :source :texture :texture-id :shape-id :image-id :last-time}
;; The texture keys arrive later, once the video has decoded its first frame.
(defonce ^:private videos (atom {}))
;; `api.cljs` owns the render loop and requires this namespace, so it installs
;; its requester here rather than being required back.
(defonce ^:private render-requester (atom nil))
(defn set-render-requester!
[f]
(reset! render-requester f))
(defn- request-render!
[]
(when-let [request @render-requester]
(request "video-frame")))
(defn- playable?
"False in the render worker, which runs the same renderer with no DOM to
build a video element in."
[]
(exists? js/document))
(defn- send-frame!
"Hands the texture to WASM, which re-wraps it as a Skia image and invalidates
the tiles of every shape painting it. Same layout as `_store_image_from_texture`."
[{:keys [shape-id image-id texture-id ^js element]}]
(let [offset (mem/alloc->offset-32 48)
heap32 (mem/get-heap-u32)]
(mem.h32/write-uuid offset heap32 shape-id)
(mem.h32/write-uuid (+ offset 4) heap32 image-id)
(aset heap32 (+ offset 8) 0) ;; thumbnail flag
(aset heap32 (+ offset 9) texture-id)
(aset heap32 (+ offset 10) (.-videoWidth element))
(aset heap32 (+ offset 11) (.-videoHeight element))
(h/call wasm/internal-module "_update_image_from_texture")))
(defn- decoded-frame?
[{:keys [^js element image-id texture-id]}]
(and (some? texture-id)
(>= (.-readyState element) HAVE-CURRENT-DATA)
(pos? (.-videoWidth element))
;; rAF runs faster than most videos decode; skip the upload when
;; playback has not advanced since the last one.
(not= (.-currentTime element) (get-in @videos [image-id :last-time]))))
(defn- upload-frame!
[gl {:keys [^js element texture image-id] :as entry}]
(when (decoded-frame? entry)
(swap! videos assoc-in [image-id :last-time] (.-currentTime element))
(webgl/upload-texture-source! gl texture element)
(send-frame! entry)
true))
(defn tick!
"Uploads a frame for every attached video that advanced. Called from the
renderer's rAF, before the frame is drawn."
[]
(when (seq @videos)
(when-let [gl (webgl/get-webgl-context)]
(reduce (fn [uploaded entry]
(or (upload-frame! gl entry) uploaded))
false
(vals @videos)))))
(defn active?
"True while some attached video is playing, so the renderer keeps scheduling
frames instead of settling."
[]
(boolean (some (fn [{:keys [^js element]}] (not (.-paused element)))
(vals @videos))))
(defn detach!
"Stops a video. The last frame stays on screen: the texture is deliberately
kept alive because WASM holds a Skia image borrowing it."
[image-id]
(when-let [{:keys [^js element]} (get @videos image-id)]
(.pause element)
(.removeAttribute element "src")
(.load element)
(swap! videos dissoc image-id)))
(defn detach-all!
[]
(run! detach! (keys @videos)))
(defn- register!
"Gives the video a texture once it has decoded something to fill it with."
[image-id ^js element]
(if-let [gl (webgl/get-webgl-context)]
(let [texture (webgl/create-webgl-texture-from-image gl element)]
(swap! videos update image-id merge
{:texture texture
:texture-id (webgl/register-texture! texture)})
(-> (.play element)
;; Nothing else would ask for a frame: `tick!` only runs inside a
;; render, and the loop only keeps itself alive once it has started.
(.then (fn [_] (request-render!)))
(.catch (fn [cause]
(log/error :hint "Could not play video" :cause cause)))))
(log/error :hint "No WebGL context available for video")))
(defn resolve-url
"Bare names resolve against the static asset folder; absolute paths and full
URLs are used as given."
[source]
(if (or (str/starts-with? source "/")
(str/includes? source "://"))
source
(str assets-path source)))
(defn- attach-element!
[shape-id image-id source]
(let [url (resolve-url source)
element (js/document.createElement "video")]
;; Registered before it loads, so a second `sync-shapes!` does not start a
;; competing element for the same fill.
(swap! videos assoc image-id
{:element element
:source source
:shape-id shape-id
:image-id image-id
:last-time nil})
;; Same-origin assets need no CORS, but a remote video served without the
;; headers would taint the canvas and make `texImage2D` throw.
(set! (.-crossOrigin element) "anonymous")
(set! (.-muted element) true)
(set! (.-loop element) true)
(set! (.-playsInline element) true)
(set! (.-onerror element)
(fn [_]
(log/error :hint "Could not load video" :url url)
(detach! image-id)))
(.addEventListener element "loadeddata"
(fn [] (register! image-id element))
#js {:once true})
(set! (.-src element) url)
nil))
(defn attach!
"Plays `source` into the image fill `image-id` of `shape-id`.
The video is muted and looped: browsers refuse to autoplay audible video, and
there is no playback UI to unmute it with."
[shape-id image-id source]
(detach! image-id)
(when (playable?)
(attach-element! shape-id image-id source)))
(defn image-fill-id
"Image painted by `shape`, from its fills or, for image shapes, its metadata."
[shape]
(or (first (types.fills/get-image-ids (types.fills/coerce (:fills shape))))
(get-in shape [:metadata :id])))
(defn video-fill
"The shape's fill image, when the uploaded media is video rather than a still."
[shape]
(->> (types.fills/coerce (:fills shape))
(seq)
(keep :fill-image)
(filter #(cm/video-type? (:mtype %)))
(first)))
(defn shape-source
"Where `shape` plays its video from: an uploaded video asset if it has one,
otherwise the `:video` attribute naming a file the browser can reach."
[shape]
(if-let [image (video-fill shape)]
(cf/resolve-file-media image)
(:video shape)))
(defn playing?
[image-id]
(boolean (when-let [^js element (get-in @videos [image-id :element])]
(not (.-paused element)))))
(defn toggle-play!
[image-id]
(when-let [^js element (get-in @videos [image-id :element])]
(if (.-paused element)
(-> (.play element)
(.then (fn [_] (request-render!)))
(.catch (fn [cause]
(log/error :hint "Could not play video" :cause cause))))
(.pause element))
(not (.-paused element))))
(defn sync-shape!
"Reconciles one shape with its `:video` attribute. Called whenever the
attribute changes, so attaching is a plain shape edit and undo comes free.
Returns nil: `set-wasm-attr!` treats what it gets back as pending image
loads, and `detach!` would otherwise hand it the video registry."
[shape]
(when-let [image-id (image-fill-id shape)]
(let [source (shape-source shape)]
(cond
(str/blank? source)
(detach! image-id)
(not= source (get-in @videos [image-id :source]))
(attach! (:id shape) image-id source))))
nil)
(defn sync-shapes!
"Reconciles every attachment with `objects`: starts the videos the shapes ask
for and stops the ones no shape wants any more. Run on page load, so a video
survives a reload; an unchanged attachment keeps playing."
[objects]
(let [wanted (into {}
(keep (fn [[shape-id shape]]
(let [source (shape-source shape)]
(when-not (str/blank? source)
(when-let [image-id (image-fill-id shape)]
[image-id {:shape-id shape-id :source source}])))))
objects)]
(doseq [[image-id {:keys [source]}] @videos]
(when (not= source (get-in wanted [image-id :source]))
(detach! image-id)))
(doseq [[image-id {:keys [shape-id source]}] wanted]
(when-not (contains? @videos image-id)
(attach! shape-id image-id source)))))
(defn attach-to-selected!
"Console entry point: attaches `source` to the selected image shape."
[source]
(let [state (deref st/state)
shape (some->> (first (dsh/lookup-selected state))
(dsh/lookup-shape state))]
(if-let [image-id (some-> shape image-fill-id)]
(do (attach! (:id shape) image-id source) true)
(do (log/error :hint "Select a shape with an image fill first"
:shape-id (:id shape)
:shape-type (:type shape))
false))))
;; Reachable from the browser console while there is no UI for this.
(unchecked-set js/globalThis "penpotAttachVideo" attach-to-selected!)
(unchecked-set js/globalThis "penpotDetachVideos" detach-all!)

20
frontend/src/app/render_wasm/api/webgl.cljs
View File

@ -23,6 +23,24 @@
(when current-ctx
(.-GLctx ^js current-ctx)))))))
(defn register-texture!
"Registers a WebGL texture with Emscripten's GL object system and returns its ID,
which is what WASM needs to wrap the texture as a Skia image."
[texture]
(let [gl-obj (unchecked-get wasm/internal-module "GL")
textures (.-textures ^js gl-obj)
new-id (.getNewId ^js gl-obj textures)]
(aset textures new-id texture)
new-id))
(defn upload-texture-source!
"Uploads `source` — an HTMLImageElement, ImageBitmap or HTMLVideoElement — into
an existing texture."
[gl texture source]
(.bindTexture ^js gl (.-TEXTURE_2D ^js gl) texture)
(.texImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 (.-RGBA ^js gl) (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) source)
(.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil))
(defn create-webgl-texture-from-image
"Creates a WebGL texture from an HTMLImageElement or ImageBitmap and returns the texture object"
[gl image-element]
@ -32,8 +50,8 @@
(.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_WRAP_T ^js gl) (.-CLAMP_TO_EDGE ^js gl))
(.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_MIN_FILTER ^js gl) (.-LINEAR ^js gl))
(.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_MAG_FILTER ^js gl) (.-LINEAR ^js gl))
(.texImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 (.-RGBA ^js gl) (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) image-element)
(.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil)
(upload-texture-source! gl texture image-element)
texture))
;; FIXME: temporary function until we are able to keep the same <canvas> across pages.

4
frontend/src/app/render_wasm/shape.cljs
View File

@ -14,6 +14,7 @@
[app.common.types.shape.layout :as ctl]
[app.main.refs :as refs]
[app.render-wasm.api :as api]
[app.render-wasm.api.video :as video]
[app.render-wasm.svg-filters :as svg-filters]
[beicon.v2.core :as rx]
[cljs.core :as c]
@ -313,6 +314,9 @@
(ctl/flex-layout? shape)
(api/set-flex-layout shape))
:video
(video/sync-shape! shape)
;; Property not in WASM
nil))))

24
frontend/translations/en.po
View File

@ -8895,6 +8895,30 @@ msgstr "Uppercase"
msgid "workspace.options.use-play-button"
msgstr "Use the play button at the header to run the prototype view."
#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:66
msgid "workspace.options.video"
msgstr "Video"
#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:73
msgid "workspace.options.video.pause"
msgstr "Pause video"
#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:80
msgid "workspace.options.video.placeholder"
msgstr "clip.mp4"
#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:74
msgid "workspace.options.video.play"
msgstr "Play video"
#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:86
msgid "workspace.options.video.remove"
msgstr "Remove video"
#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:82
msgid "workspace.options.video.source"
msgstr "Video source"
#: src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:538, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:560
msgid "workspace.options.width"
msgstr "Width"

8
render-wasm/src/render/gpu_state.rs
View File

@ -55,6 +55,14 @@ impl GpuState {
})
}
/// Drops the texture bindings Skia believes are current. Required whenever
/// JS writes a texture behind Ganesh's back (`texImage2D` from the CLJS
/// side): Skia caches which texture is bound to each unit, so after an
/// external bind it would keep drawing with a stale one.
pub fn reset_texture_bindings(&mut self) {
self.context.reset_gl_texture_bindings();
}
pub fn max_texture_size(&self) -> i32 {
self.context
.max_texture_size()

32
render-wasm/src/render/images.rs
View File

@ -398,6 +398,38 @@ impl ImageStore {
Ok(())
}
/// Rebinds an already-registered image to a different GL texture, or to the
/// same texture after its contents changed. Video frames go through here:
/// the CLJS side uploads each decoded frame into one texture it owns and
/// calls this, which re-wraps it as a fresh `skia::Image`. Mutating the
/// texture under a live `Image` is not enough — Skia keys its caches
/// (filters, mipmaps, tile contents) on the image's unique id, so it would
/// keep serving the frame it first saw.
pub fn rebind_gl_texture(
&mut self,
id: Uuid,
is_thumbnail: bool,
texture_id: u32,
width: i32,
height: i32,
) -> Result<()> {
let Some(context) = self.context.as_mut() else {
return Err(crate::error::Error::CriticalError(
"Cannot rebind a GL texture without a GPU context".to_string(),
));
};
let image = create_image_from_gl_texture(context, texture_id, width, height)?;
let key = (id, is_thumbnail);
if let Some(previous) = self.images.remove(&key) {
self.total_bytes -= previous.bytes;
}
let bytes = (width as usize) * (height as usize) * 4;
self.insert_entry(key, StoredImage::Gpu(image), bytes);
Ok(())
}
pub fn contains(&self, id: &Uuid, is_thumbnail: bool) -> bool {
self.images.contains_key(&(*id, is_thumbnail))
}

143
render-wasm/src/wasm/fills/image.rs
View File

@ -1,4 +1,5 @@
use crate::error::{Error, Result};
use crate::get_gpu_state;
use crate::get_resources;
use crate::mem;
use crate::shapes::Fill;
@ -183,6 +184,56 @@ pub extern "C" fn store_image_url(a: u32, b: u32, c: u32, d: u32) -> Result<()>
Ok(())
}
/// A GL texture handed over from JS, as written by `store-image-texture` /
/// `upload-video-frame!` on the CLJS side.
struct TextureUpload {
ids: ShapeImageIds,
is_thumbnail: bool,
texture_id: u32,
width: i32,
height: i32,
}
const TEXTURE_UPLOAD_SIZE: usize = 48; // header + texture id + width + height
/// Reads a `TextureUpload` out of the shared buffer. The buffer is freed by the
/// caller on success and here on failure, so a malformed payload cannot leak it.
fn read_texture_upload() -> Result<TextureUpload> {
let bytes = mem::bytes();
if bytes.len() < TEXTURE_UPLOAD_SIZE {
// FIXME: Review if this should be an critical or a recoverable error.
eprintln!("read_texture_upload: insufficient data");
mem::free_bytes()?;
return Err(Error::RecoverableError(
"read_texture_upload: insufficient data".to_string(),
));
}
let ids = ShapeImageIds::try_from(&bytes[0..IMAGE_IDS_SIZE])
.map_err(|_| Error::CriticalError("Invalid image ids".to_string()))?;
// FIXME: read bytes in a safe way
let read_u32 = |range: std::ops::Range<usize>, what: &str| -> Result<u32> {
Ok(u32::from_le_bytes((&bytes[range]).try_into().map_err(
|_| Error::CriticalError(format!("Invalid bytes for {}", what)),
)?))
};
let is_thumbnail = read_u32(IMAGE_IDS_SIZE..IMAGE_HEADER_SIZE, "is_thumbnail flag")? != 0;
let texture_id = read_u32(36..40, "texture id")?;
let width = read_u32(40..44, "width")? as i32;
let height = read_u32(44..48, "height")? as i32;
Ok(TextureUpload {
ids,
is_thumbnail,
texture_id,
width,
height,
})
}
/// Stores an image from an existing WebGL texture, avoiding re-decoding
/// Expected memory layout:
/// - bytes 0-15: shape UUID
@ -194,66 +245,52 @@ pub extern "C" fn store_image_url(a: u32, b: u32, c: u32, d: u32) -> Result<()>
#[no_mangle]
#[wasm_error]
pub extern "C" fn store_image_from_texture() -> Result<()> {
let bytes = mem::bytes();
// FIXME: where does this 48 come from?
if bytes.len() < 48 {
// FIXME: Review if this should be an critical or a recoverable error.
eprintln!("store_image_from_texture: insufficient data");
mem::free_bytes()?;
return Err(Error::RecoverableError(
"store_image_from_texture: insufficient data".to_string(),
));
}
let ids = ShapeImageIds::try_from(&bytes[0..IMAGE_IDS_SIZE])
.map_err(|_| Error::CriticalError("Invalid image ids".to_string()))?;
// FIXME: read bytes in a safe way
// Read is_thumbnail flag (4 bytes as u32)
let is_thumbnail_bytes = &bytes[IMAGE_IDS_SIZE..IMAGE_HEADER_SIZE];
let is_thumbnail_value =
u32::from_le_bytes(is_thumbnail_bytes.try_into().map_err(|_| {
Error::CriticalError("Invalid bytes for is_thumbnail flag".to_string())
})?);
let is_thumbnail = is_thumbnail_value != 0;
// Read GL texture ID (4 bytes as u32)
let texture_id_bytes = &bytes[36..40];
let texture_id = u32::from_le_bytes(
texture_id_bytes
.try_into()
.map_err(|_| Error::CriticalError("Invalid bytes for texture id".to_string()))?,
);
// Read width and height (8 bytes as two i32s)
let width_bytes = &bytes[40..44];
let width = i32::from_le_bytes(
width_bytes
.try_into()
.map_err(|_| Error::CriticalError("Invalid bytes for width".to_string()))?,
);
let height_bytes = &bytes[44..48];
let height = i32::from_le_bytes(
height_bytes
.try_into()
.map_err(|_| Error::CriticalError("Invalid bytes for height".to_string()))?,
);
let upload = read_texture_upload()?;
with_state!(state, {
if let Err(msg) = get_resources().images.add_image_from_gl_texture(
ids.image_id,
is_thumbnail,
texture_id,
width,
height,
upload.ids.image_id,
upload.is_thumbnail,
upload.texture_id,
upload.width,
upload.height,
) {
// FIXME: Review if we should return a RecoverableError
eprintln!("store_image_from_texture error: {}", msg);
}
touch_shapes_with_image(state, ids.image_id);
touch_shapes_with_image(state, upload.ids.image_id);
});
mem::free_bytes()?;
Ok(())
}
/// Rebinds an already-stored image to the current contents of a GL texture and
/// invalidates the tiles of every shape that paints it. Same memory layout as
/// `store_image_from_texture`.
///
/// This is the per-frame entry point for video: the CLJS side owns one texture
/// per playing video, uploads the decoded frame into it and calls this.
#[no_mangle]
#[wasm_error]
pub extern "C" fn update_image_from_texture() -> Result<()> {
let upload = read_texture_upload()?;
// The caller just bound and wrote the texture from JS, which Skia has no
// way to observe. Drop its cached bindings before it samples the texture.
get_gpu_state().reset_texture_bindings();
with_state!(state, {
if let Err(msg) = get_resources().images.rebind_gl_texture(
upload.ids.image_id,
upload.is_thumbnail,
upload.texture_id,
upload.width,
upload.height,
) {
eprintln!("update_image_from_texture error: {}", msg);
}
touch_shapes_with_image(state, upload.ids.image_id);
});
mem::free_bytes()?;