From 8455fcf170f86fb38bbb92a65559e38b70d7c78d Mon Sep 17 00:00:00 2001 From: Elena Torro Date: Wed, 16 Sep 2026 12:29:10 +0200 Subject: [PATCH] wip --- backend/src/app/media/validation.clj | 4 +- backend/src/app/rpc/commands/media.clj | 37 ++- common/src/app/common/media.cljc | 26 +- common/src/app/common/types/fills/impl.cljc | 9 +- common/src/app/common/types/shape.cljc | 10 +- docker/devenv/files/nginx.conf | 2 +- frontend/src/app/main/data/media.cljs | 2 +- .../src/app/main/data/workspace/media.cljs | 67 ++++- .../sidebar/options/menus/video.cljs | 92 ++++++ .../sidebar/options/menus/video.scss | 23 ++ .../sidebar/options/shapes/rect.cljs | 23 +- frontend/src/app/render_wasm/api.cljs | 47 ++- frontend/src/app/render_wasm/api/video.cljs | 282 ++++++++++++++++++ frontend/src/app/render_wasm/api/webgl.cljs | 20 +- frontend/src/app/render_wasm/shape.cljs | 4 + frontend/translations/en.po | 24 ++ render-wasm/src/render/gpu_state.rs | 8 + render-wasm/src/render/images.rs | 32 ++ render-wasm/src/wasm/fills/image.rs | 143 +++++---- 19 files changed, 760 insertions(+), 95 deletions(-) create mode 100644 frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs create mode 100644 frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss create mode 100644 frontend/src/app/render_wasm/api/video.cljs diff --git a/backend/src/app/media/validation.clj b/backend/src/app/media/validation.clj index 5da86fb99f..61cbbefdd8 100644 --- a/backend/src/app/media/validation.clj +++ b/backend/src/app/media/validation.clj @@ -33,7 +33,9 @@ (sm/check-fn schema:input)) (defn validate-media-type! - ([upload] (validate-media-type! upload cm/image-types)) + ;; `upload-types` rather than `image-types`: a file can also hold video. + ;; Callers that must stay image-only pass `cm/image-types` explicitly. + ([upload] (validate-media-type! upload cm/upload-types)) ([upload allowed] (when-not (contains? allowed (:mtype upload)) (ex/raise :type :validation diff --git a/backend/src/app/rpc/commands/media.clj b/backend/src/app/rpc/commands/media.clj index 7bf42c57a9..8b81f07168 100644 --- a/backend/src/app/rpc/commands/media.clj +++ b/backend/src/app/rpc/commands/media.clj @@ -9,6 +9,7 @@ [app.common.data :as d] [app.common.exceptions :as ex] [app.common.logging :as l] + [app.common.media :as cm] [app.common.schema :as sm] [app.common.time :as ct] [app.common.uuid :as uuid] @@ -52,7 +53,11 @@ [:file-id ::sm/uuid] [:is-local ::sm/boolean] [:name [:string {:max 250}]] - [:content media.v/schema:upload]]) + [:content media.v/schema:upload] + ;; Only video sends these: nothing on the backend can read them from the + ;; file, so the client reports what it decoded. + [:width {:optional true} ::sm/int] + [:height {:optional true} ::sm/int]]) (sv/defmethod ::upload-file-media-object {::doc/added "1.17" @@ -170,9 +175,31 @@ :always (assoc ::image (process-main-image info))))) +(defn- process-video + "Video is stored and served, never decoded: nothing here reads its + dimensions or renders a frame to thumbnail. The client sends the dimensions + it read from the file. Shaped like `process-image` so the caller does not + care which one ran." + [content width height] + (when-not (and width height) + (ex/raise :type :validation + :code :missing-video-dimensions + :hint "video uploads must report the dimensions the client decoded")) + (let [info (assoc content + :ts (ct/now) + :width width + :height height)] + (assoc info ::image (process-main-image info)))) + +(defn- process-content + [cfg {:keys [mtype] :as content} width height] + (if (cm/video-type? mtype) + (process-video content width height) + (process-image cfg content))) + (defn- create-file-media-object [{:keys [::sto/storage ::db/conn] :as cfg} - {:keys [id file-id is-local name content from-url? from-chunks?]}] + {:keys [id file-id is-local name content width height from-url? from-chunks?]}] (let [tpoint (ct/tpoint) id (or id (uuid/next)) @@ -192,7 +219,7 @@ :path (str (:path content)) :origin origin) - (let [result (process-image cfg content) + (let [result (process-content cfg content width height) image (sto/put-object! storage (::image result)) thumb (when-let [params (::thumb result)] (sto/put-object! storage params)) @@ -453,7 +480,9 @@ [:is-local ::sm/boolean] [:name [:string {:max 250}]] [:mtype :string] - [:id {:optional true} ::sm/uuid]]) + [:id {:optional true} ::sm/uuid] + [:width {:optional true} ::sm/int] + [:height {:optional true} ::sm/int]]) (sv/defmethod ::assemble-file-media-object {::doc/added "2.17" diff --git a/common/src/app/common/media.cljc b/common/src/app/common/media.cljc index 8e6038ea11..ada45243aa 100644 --- a/common/src/app/common/media.cljc +++ b/common/src/app/common/media.cljc @@ -22,6 +22,20 @@ "image/gif" "image/svg+xml"}) +(def video-types + #{"video/mp4" + "video/webm"}) + +(defn video-type? + [mtype] + (contains? video-types mtype)) + +;; Media a file can hold. Video is a proof of concept: the backend stores and +;; serves it but decodes nothing, so it carries the dimensions the client read +;; from the file. +(def upload-types + (into image-types video-types)) + (def tempfile-types (conj image-types "application/pdf" "application/zip")) @@ -32,7 +46,9 @@ :jpeg ".jpg" :webp ".webp" :gif ".gif" - :svg ".svg")) + :svg ".svg" + :mp4 ".mp4" + :webm ".webm")) (defn format->mtype [format] @@ -43,6 +59,8 @@ :webp "image/webp" :gif "image/gif" :svg "image/svg+xml" + :mp4 "video/mp4" + :webm "video/webm" "application/octet-stream")) (defn mtype->format @@ -53,6 +71,8 @@ "image/webp" :webp "image/gif" :gif "image/svg+xml" :svg + "video/mp4" :mp4 + "video/webm" :webm nil)) (defn mtype->extension [mtype] @@ -65,6 +85,8 @@ "image/png" ".png" "image/svg+xml" ".svg" "image/webp" ".webp" + "video/mp4" ".mp4" + "video/webm" ".webm" "application/zip" ".zip" "application/penpot" ".penpot" "application/pdf" ".pdf" @@ -78,7 +100,7 @@ (defn strip-image-extension [filename] - (let [image-extensions-re #"(\.png)|(\.jpg)|(\.jpeg)|(\.webp)|(\.gif)|(\.svg)$"] + (let [image-extensions-re #"(\.png)|(\.jpg)|(\.jpeg)|(\.webp)|(\.gif)|(\.svg)|(\.mp4)|(\.webm)$"] (str/replace filename image-extensions-re ""))) (defn parse-font-weight diff --git a/common/src/app/common/types/fills/impl.cljc b/common/src/app/common/types/fills/impl.cljc index 2f8ca65a6f..82a93adb5d 100644 --- a/common/src/app/common/types/fills/impl.cljc +++ b/common/src/app/common/types/fills/impl.cljc @@ -160,7 +160,9 @@ "image/png" 0x02 "image/gif" 0x03 "image/webp" 0x04 - "image/svg+xml" 0x05)] + "image/svg+xml" 0x05 + "video/mp4" 0x06 + "video/webm" 0x07)] (buf/write-short buffer (+ offset 2) val))) (if (and (some? ref-file) @@ -241,7 +243,10 @@ 0x02 "image/png" 0x03 "image/gif" 0x04 "image/webp" - 0x05 "image/svg+xml")] + 0x05 "image/svg+xml" + 0x06 "video/mp4" + 0x07 "video/webm" + nil)] {:fill-opacity opacity :fill-image (cond-> {:id id :width width diff --git a/common/src/app/common/types/shape.cljc b/common/src/app/common/types/shape.cljc index 6a617c21cc..4ee96be17e 100644 --- a/common/src/app/common/types/shape.cljc +++ b/common/src/app/common/types/shape.cljc @@ -276,7 +276,13 @@ [:svg-attrs {:optional true} :map] [:svg-defs {:optional true} :map] [:svg-transform {:optional true} :map] - [:svg-viewbox {:optional true} :map]]) + [:svg-viewbox {:optional true} :map] + + ;; Video proof of concept: the source played into the shape's image fill, + ;; either a name under the static asset folder or a full URL. Only the + ;; render-wasm renderer paints it; every other consumer keeps showing the + ;; image fill, which doubles as the poster frame. + [:video {:optional true} [:maybe :string]]]) (def schema:group-attrs [:map {:title "GroupAttrs"} @@ -516,7 +522,7 @@ :hidden :masked-group :fills :proportion :proportion-lock :constraints-h :constraints-v :fixed-scroll :r1 :r2 :r3 :r4 :rotation :opacity :grids :exports :strokes :blend-mode :interactions :shadow :blur :background-blur :grow-type :applied-tokens - :plugin-data}) + :plugin-data :video}) (def ^:private allowed-shape-geom-attrs #{:x :y :width :height}) (def ^:private allowed-shape-base-attrs #{:id :name :type :selrect :points :transform diff --git a/docker/devenv/files/nginx.conf b/docker/devenv/files/nginx.conf index 3d3d70263f..afbdf11000 100644 --- a/docker/devenv/files/nginx.conf +++ b/docker/devenv/files/nginx.conf @@ -278,7 +278,7 @@ http { add_header X-Cache-Status $upstream_cache_status; } - location ~* \.(jpg|png|svg|ttf|woff|woff2|gif)$ { + location ~* \.(jpg|png|svg|ttf|woff|woff2|gif|mp4|webm|mov)$ { include /home/penpot/penpot/docker/devenv/files/nginx-security-headers.conf; add_header Cache-Control "public, max-age=604800" always; # 7 days } diff --git a/frontend/src/app/main/data/media.cljs b/frontend/src/app/main/data/media.cljs index 22b2f9f13b..392b977cf0 100644 --- a/frontend/src/app/main/data/media.cljs +++ b/frontend/src/app/main/data/media.cljs @@ -40,7 +40,7 @@ (defn validate-file "Check that a file obtained with the file javascript API is valid." [file] - (when-not (contains? cm/image-types (.-type file)) + (when-not (contains? cm/upload-types (.-type file)) (ex/raise :type :validation :code :media-type-not-allowed :hint (str/ffmt "media type % is not supported" (.-type file)))) diff --git a/frontend/src/app/main/data/workspace/media.cljs b/frontend/src/app/main/data/workspace/media.cljs index f3a395a504..4aac2d2e4e 100644 --- a/frontend/src/app/main/data/workspace/media.cljs +++ b/frontend/src/app/main/data/workspace/media.cljs @@ -38,7 +38,32 @@ [tubax.core :as tubax])) (def accept-image-types - (str/join "," media/image-types)) + (str/join "," media/upload-types)) + +(defn- read-video-dimensions + "Resolves the intrinsic size of a video blob. Nothing on the backend decodes + video, so the client is what reports the dimensions an upload is stored + with." + [blob] + (p/create + (fn [resolve reject] + (let [url (js/URL.createObjectURL blob) + element (js/document.createElement "video") + done (fn [f value] + (js/URL.revokeObjectURL url) + (f value))] + (set! (.-preload element) "metadata") + (set! (.-muted element) true) + (set! (.-onloadedmetadata element) + (fn [_] + (done resolve {:width (.-videoWidth element) + :height (.-videoHeight element)}))) + (set! (.-onerror element) + (fn [_] + (done reject (ex/error :type :validation + :code :invalid-video + :hint "could not read the video dimensions")))) + (set! (.-src element) url))))) (defn- optimize [input] @@ -123,17 +148,19 @@ "Uploads `blob` to `file-id` as a chunked media object using the three-step session API. Returns an observable that emits the assembled file-media-object map." - [{:keys [file-id name is-local blob]}] + [{:keys [file-id name is-local blob width height]}] (let [mtype (.-type blob)] (->> (uploads/upload-blob-chunked blob) (rx/mapcat (fn [{:keys [session-id]}] (rp/cmd! :assemble-file-media-object - {:session-id session-id - :file-id file-id - :is-local is-local - :name name - :mtype mtype})))))) + (cond-> {:session-id session-id + :file-id file-id + :is-local is-local + :name name + :mtype mtype} + (some? width) (assoc :width width) + (some? height) (assoc :height height)))))))) (defn process-uris [{:keys [file-id local? name uris mtype on-image on-svg]}] @@ -175,18 +202,26 @@ (and (not force-media) (= (.-type blob) "image/svg+xml"))) - (upload-blob [blob] - (let [params {:file-id file-id - :name (or name (if (dmm/file? blob) (media/strip-image-extension (.-name blob)) "blob")) - :is-local local? - :blob blob}] + (upload-blob* [blob dimensions] + (let [params (merge {:file-id file-id + :name (or name (if (dmm/file? blob) (media/strip-image-extension (.-name blob)) "blob")) + :is-local local? + :blob blob} + dimensions)] (if (>= (.-size blob) chunk-size) (upload-blob-chunked params) (rp/cmd! :upload-file-media-object - {:file-id file-id - :name (:name params) - :is-local local? - :content blob})))) + (merge {:file-id file-id + :name (:name params) + :is-local local? + :content blob} + dimensions))))) + + (upload-blob [blob] + (if (media/video-type? (.-type blob)) + (->> (rx/from (read-video-dimensions blob)) + (rx/mapcat #(upload-blob* blob %))) + (upload-blob* blob nil))) (extract-content [blob] (let [name (or name (.-name blob))] diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs b/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs new file mode 100644 index 0000000000..eed4f5bbf5 --- /dev/null +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs @@ -0,0 +1,92 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.main.ui.workspace.sidebar.options.menus.video + "Proof of concept: plays a video into a shape's image fill. + + Nothing uploads video, so the source is a path the browser can already + reach — a file under `frontend/resources/public/images/` or a full URL. Only + the render-wasm renderer paints it; elsewhere the image fill still shows." + (:require-macros [app.main.style :as stl]) + (:require + [app.main.data.workspace.shapes :as dwsh] + [app.main.store :as st] + [app.main.ui.components.title-bar :refer [title-bar*]] + [app.main.ui.ds.buttons.icon-button :refer [icon-button*]] + [app.main.ui.ds.controls.input :refer [input*]] + [app.main.ui.ds.foundations.assets.icon :as i] + [app.render-wasm.api.video :as video] + [app.util.dom :as dom] + [app.util.i18n :refer [tr]] + [clojure.string :as str] + [rumext.v2 :as mf])) + +(defn set-video + "Stores the source on the shape. `app.render-wasm.shape` picks the change up + and attaches or detaches the video, so undo and reload both behave." + [ids source] + (dwsh/update-shapes ids (fn [shape] + (if (str/blank? source) + (dissoc shape :video) + (assoc shape :video source))))) + +(mf/defc video-menu* + [{:keys [ids image-id source is-asset]}] + (let [;; `video/playing?` reads the element, which is outside app state, so + ;; the button tracks it locally. + playing* (mf/use-state #(video/playing? image-id)) + playing (deref playing*) + + has-source (not (str/blank? source)) + + on-change + (mf/use-fn + (mf/deps ids) + (fn [event] + (let [value (-> event dom/get-target dom/get-value str/trim)] + (st/emit! (set-video ids value))))) + + on-toggle-play + (mf/use-fn + (mf/deps image-id) + (fn [] + (reset! playing* (video/toggle-play! image-id)))) + + on-remove + (mf/use-fn + (mf/deps ids) + (fn [] + (st/emit! (set-video ids nil))))] + + [:section {:class (stl/css :element-set) + :aria-label (tr "workspace.options.video")} + [:div {:class (stl/css :element-title)} + [:> title-bar* {:collapsable false + :title (tr "workspace.options.video")} + (when has-source + [:> icon-button* {:variant "ghost" + :aria-label (if playing + (tr "workspace.options.video.pause") + (tr "workspace.options.video.play")) + :on-click on-toggle-play + :selected playing + :tooltip-placement "top-left" + :icon i/play}])]] + ;; An uploaded video has nothing to type: its source is the asset itself, + ;; and removing it means deleting the shape. + (when-not is-asset + [:div {:class (stl/css :row)} + [:> input* {:class (stl/css :source-input) + :placeholder (tr "workspace.options.video.placeholder") + :default-value (or source "") + :aria-label (tr "workspace.options.video.source") + :on-blur on-change}] + [:> icon-button* {:variant "ghost" + :aria-label (tr "workspace.options.video.remove") + :on-click on-remove + :disabled (not has-source) + :tooltip-placement "top-left" + :icon i/remove}]])])) diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss b/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss new file mode 100644 index 0000000000..d382a84c92 --- /dev/null +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss @@ -0,0 +1,23 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// Copyright (c) KALEIDOS SUBSIDIARY SL + +@use "../../../sidebar/common/sidebar.scss" as sidebar; + +.element-set { + max-width: var(--options-width); +} + +.row { + display: flex; + align-items: center; + gap: var(--sp-xs); + margin-bottom: var(--sp-s); +} + +.source-input { + flex-grow: 1; + min-width: 0; +} diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs b/frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs index dd19747911..13d319b4a3 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs @@ -8,6 +8,7 @@ (:require [app.common.data.macros :as dm] [app.common.types.shape.layout :as ctl] + [app.main.features :as features] [app.main.refs :as refs] [app.main.ui.workspace.sidebar.options.menus.blur :refer [blur-menu*]] [app.main.ui.workspace.sidebar.options.menus.constraints :refer [constraint-attrs constraints-menu*]] @@ -21,6 +22,8 @@ [app.main.ui.workspace.sidebar.options.menus.shadow :refer [shadow-menu*]] [app.main.ui.workspace.sidebar.options.menus.stroke :refer [stroke-attrs stroke-menu*]] [app.main.ui.workspace.sidebar.options.menus.svg-attrs :refer [svg-attrs-menu*]] + [app.main.ui.workspace.sidebar.options.menus.video :refer [video-menu*]] + [app.render-wasm.api.video :as video] [rumext.v2 :as mf])) (mf/defc options* @@ -80,7 +83,19 @@ (refs/parents-by-ids ids)) parents - (mf/deref parents-by-ids-ref)] + (mf/deref parents-by-ids-ref) + + ;; Video is painted by re-uploading frames into the shape's image fill, + ;; and only the wasm renderer does that. + render-wasm? + (features/use-feature "render-wasm/v1") + + video-image-id + (when render-wasm? (video/image-fill-id shape)) + + ;; An uploaded video asset needs no source field, only playback. + video-asset? + (and render-wasm? (some? (video/video-fill shape)))] [:* [:> layer-menu* {:ids ids @@ -127,6 +142,12 @@ :values shape :applied-tokens applied-tokens}] + (when (some? video-image-id) + [:> video-menu* {:ids ids + :image-id video-image-id + :source (video/shape-source shape) + :is-asset video-asset?}]) + [:> stroke-menu* {:ids ids :type type :values stroke-values diff --git a/frontend/src/app/render_wasm/api.cljs b/frontend/src/app/render_wasm/api.cljs index 64d19c56d8..61521a940c 100644 --- a/frontend/src/app/render_wasm/api.cljs +++ b/frontend/src/app/render_wasm/api.cljs @@ -16,6 +16,7 @@ [app.common.fonts :as cfnt] [app.common.logging :as log] [app.common.math :as mth] + [app.common.media :as cm] [app.common.render-wasm.api.props :as props] [app.common.render-wasm.api.upload :as upload] [app.common.render-wasm.helpers :as h] @@ -45,6 +46,7 @@ [app.render-wasm.api.enums] [app.render-wasm.api.fonts :as f] [app.render-wasm.api.texts :as t] + [app.render-wasm.api.video :as video] [app.render-wasm.api.webgl :as webgl] [app.render-wasm.deserializers :as dr] [app.render-wasm.gesture :as wasm-gesture] @@ -490,6 +492,10 @@ (defn- render [timestamp] (when (wasm/live?) + ;; Upload pending video frames before drawing, so this frame paints the + ;; current one instead of the previous. + (video/tick!) + ;; SYNC-TILES makes WASM keep the last presented frame while the new tiles ;; are rasterized, rather than clearing to the background first. The flag is ;; one-shot on both sides: WASM clears it when the render loop starts, so @@ -510,6 +516,12 @@ (js/console.error "text-editor overlay/update failed:" e))) (set! wasm/internal-frame-id nil) + + ;; Playing video has no other trigger to keep the loop alive. Requested + ;; after the frame id is cleared so the scheduled frame is the one kept. + (when (video/active?) + (request-render "video-frame")) + (ug/dispatch! (ug/event "penpot:wasm:render")))) (defn render-ui-only @@ -663,6 +675,8 @@ (throw e))))))] (set! wasm/internal-frame-id frame-id)))))) +(video/set-render-requester! request-render) + (defn request-render-preserving-target "Like `request-render`, but keeps the previously presented frame on screen while the new tiles are rasterized instead of blanking the canvas first. @@ -845,15 +859,6 @@ nil) -(defn- get-texture-id-for-gl-object - "Registers a WebGL texture with Emscripten's GL object system and returns its ID" - [texture] - (let [gl-obj (unchecked-get wasm/internal-module "GL") - textures (.-textures ^js gl-obj) - new-id (.getNewId ^js gl-obj textures)] - (aset textures new-id texture) - new-id)) - (defn- svg-blob? [^js blob] (str/starts-with? (.-type blob) "image/svg")) @@ -905,7 +910,7 @@ [shape-id image-id thumbnail? img] (when-let [gl (webgl/get-webgl-context)] (let [texture (webgl/create-webgl-texture-from-image gl img) - texture-id (get-texture-id-for-gl-object texture) + texture-id (webgl/register-texture! texture) width (.-width ^js img) height (.-height ^js img) ;; Header: 32 bytes (2 UUIDs) + 4 bytes (thumbnail) @@ -964,6 +969,17 @@ :cause cause) (rx/empty)))))})) +(defn- still-image-ids + "Image ids the renderer has to fetch. Video fills are skipped: their frames + are uploaded by `app.render-wasm.api.video`, so downloading the container as + an image would burn the bandwidth and then fail to decode." + [fills] + (into #{} + (comp (keep :fill-image) + (remove #(cm/video-type? (:mtype %))) + (map :id)) + (seq fills))) + (defn- get-fill-images [leaf] (filter :fill-image (:fills leaf))) @@ -1018,7 +1034,7 @@ (store-image-url! id (cf/resolve-file-media {:id id} thumbnail?)) (when (zero? cached-image?) (fetch-image shape-id id thumbnail?)))) - (types.fills/get-image-ids fills)))))) + (still-image-ids fills)))))) (defn- stroke-image-ids [strokes] @@ -1743,6 +1759,9 @@ update-text-layouts fires for all text shapes after fonts load — not just the first shape that triggered the fetch." [shapes] + ;; Newly added shapes never reach `set-wasm-attr!`, which is what starts a + ;; video on an edit, so a shape created with one is started here instead. + (run! video/sync-shape! shapes) (let [total-shapes (count shapes) {:keys [thumbnails full text-font-state]} (loop [index 0 @@ -2035,6 +2054,9 @@ (set-objects objects render-callback nil false)) ([objects render-callback on-shapes-ready force-sync] (when (wasm/live?) + ;; Start (and stop) the videos this page asks for. Cheap: one pass over + ;; the objects, and attachments that did not change keep playing. + (video/sync-shapes! objects) (perf/begin-measure "set-objects") (let [shapes (shapes-in-tree-order objects) total-shapes (count shapes)] @@ -2611,6 +2633,9 @@ :as payload}] (ug/dispatch! (ug/event "penpot:wasm:reload-start")) (reset! wasm/reloading? true) + ;; Attached videos hold texture ids from the context that is about to be + ;; destroyed; a frame uploaded after the reload would wrap a stale texture. + (video/detach-all!) (let [fonts (derive-font-resources base-objects fonts)] (-> (p/resolved nil) ;; Keep teardown strict (`_clean_up` + deleteContext) but do not diff --git a/frontend/src/app/render_wasm/api/video.cljs b/frontend/src/app/render_wasm/api/video.cljs new file mode 100644 index 0000000000..031cdea496 --- /dev/null +++ b/frontend/src/app/render_wasm/api/video.cljs @@ -0,0 +1,282 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.render-wasm.api.video + "Proof of concept: video frames painted as image fills. + + Skia has no video decoder, so the browser decodes and we only hand Skia a + texture. Each attached video owns one WebGL texture; on every rendered frame + the current frame is uploaded into that texture and WASM re-wraps it, so + every shape whose image fill points at `image-id` paints it. + + The source is stored on the shape as `:video`, so it persists with the file + and comes back on load. Drop a file into `frontend/resources/public/images/`, + where it is served like any other static asset, and name it from the Video + section of the design sidebar. `penpotAttachVideo(\"clip.mp4\")` does the same + from the console for the selected shape, without persisting it. + + There is no backend support: nothing uploads video, so the source is a path + the browser can reach on its own." + (:require + [app.common.logging :as log] + [app.common.media :as cm] + [app.common.render-wasm.helpers :as h] + [app.common.render-wasm.mem :as mem] + [app.common.render-wasm.mem.heap32 :as mem.h32] + [app.common.render-wasm.wasm :as wasm] + [app.common.types.fills :as types.fills] + [app.config :as cf] + [app.main.data.helpers :as dsh] + [app.main.store :as st] + [app.render-wasm.api.webgl :as webgl] + [clojure.string :as str])) + +;; HTMLMediaElement.readyState: the element has data for the current position. +(def ^:private HAVE-CURRENT-DATA 2) + +;; Videos live next to the other static assets, so a bare file name is enough. +(def ^:private assets-path "/images/") + +;; image-id -> {:element :source :texture :texture-id :shape-id :image-id :last-time} +;; The texture keys arrive later, once the video has decoded its first frame. +(defonce ^:private videos (atom {})) + +;; `api.cljs` owns the render loop and requires this namespace, so it installs +;; its requester here rather than being required back. +(defonce ^:private render-requester (atom nil)) + +(defn set-render-requester! + [f] + (reset! render-requester f)) + +(defn- request-render! + [] + (when-let [request @render-requester] + (request "video-frame"))) + +(defn- playable? + "False in the render worker, which runs the same renderer with no DOM to + build a video element in." + [] + (exists? js/document)) + +(defn- send-frame! + "Hands the texture to WASM, which re-wraps it as a Skia image and invalidates + the tiles of every shape painting it. Same layout as `_store_image_from_texture`." + [{:keys [shape-id image-id texture-id ^js element]}] + (let [offset (mem/alloc->offset-32 48) + heap32 (mem/get-heap-u32)] + (mem.h32/write-uuid offset heap32 shape-id) + (mem.h32/write-uuid (+ offset 4) heap32 image-id) + (aset heap32 (+ offset 8) 0) ;; thumbnail flag + (aset heap32 (+ offset 9) texture-id) + (aset heap32 (+ offset 10) (.-videoWidth element)) + (aset heap32 (+ offset 11) (.-videoHeight element)) + (h/call wasm/internal-module "_update_image_from_texture"))) + +(defn- decoded-frame? + [{:keys [^js element image-id texture-id]}] + (and (some? texture-id) + (>= (.-readyState element) HAVE-CURRENT-DATA) + (pos? (.-videoWidth element)) + ;; rAF runs faster than most videos decode; skip the upload when + ;; playback has not advanced since the last one. + (not= (.-currentTime element) (get-in @videos [image-id :last-time])))) + +(defn- upload-frame! + [gl {:keys [^js element texture image-id] :as entry}] + (when (decoded-frame? entry) + (swap! videos assoc-in [image-id :last-time] (.-currentTime element)) + (webgl/upload-texture-source! gl texture element) + (send-frame! entry) + true)) + +(defn tick! + "Uploads a frame for every attached video that advanced. Called from the + renderer's rAF, before the frame is drawn." + [] + (when (seq @videos) + (when-let [gl (webgl/get-webgl-context)] + (reduce (fn [uploaded entry] + (or (upload-frame! gl entry) uploaded)) + false + (vals @videos))))) + +(defn active? + "True while some attached video is playing, so the renderer keeps scheduling + frames instead of settling." + [] + (boolean (some (fn [{:keys [^js element]}] (not (.-paused element))) + (vals @videos)))) + +(defn detach! + "Stops a video. The last frame stays on screen: the texture is deliberately + kept alive because WASM holds a Skia image borrowing it." + [image-id] + (when-let [{:keys [^js element]} (get @videos image-id)] + (.pause element) + (.removeAttribute element "src") + (.load element) + (swap! videos dissoc image-id))) + +(defn detach-all! + [] + (run! detach! (keys @videos))) + +(defn- register! + "Gives the video a texture once it has decoded something to fill it with." + [image-id ^js element] + (if-let [gl (webgl/get-webgl-context)] + (let [texture (webgl/create-webgl-texture-from-image gl element)] + (swap! videos update image-id merge + {:texture texture + :texture-id (webgl/register-texture! texture)}) + (-> (.play element) + ;; Nothing else would ask for a frame: `tick!` only runs inside a + ;; render, and the loop only keeps itself alive once it has started. + (.then (fn [_] (request-render!))) + (.catch (fn [cause] + (log/error :hint "Could not play video" :cause cause))))) + (log/error :hint "No WebGL context available for video"))) + +(defn resolve-url + "Bare names resolve against the static asset folder; absolute paths and full + URLs are used as given." + [source] + (if (or (str/starts-with? source "/") + (str/includes? source "://")) + source + (str assets-path source))) + +(defn- attach-element! + [shape-id image-id source] + (let [url (resolve-url source) + element (js/document.createElement "video")] + ;; Registered before it loads, so a second `sync-shapes!` does not start a + ;; competing element for the same fill. + (swap! videos assoc image-id + {:element element + :source source + :shape-id shape-id + :image-id image-id + :last-time nil}) + ;; Same-origin assets need no CORS, but a remote video served without the + ;; headers would taint the canvas and make `texImage2D` throw. + (set! (.-crossOrigin element) "anonymous") + (set! (.-muted element) true) + (set! (.-loop element) true) + (set! (.-playsInline element) true) + (set! (.-onerror element) + (fn [_] + (log/error :hint "Could not load video" :url url) + (detach! image-id))) + (.addEventListener element "loadeddata" + (fn [] (register! image-id element)) + #js {:once true}) + (set! (.-src element) url) + nil)) + +(defn attach! + "Plays `source` into the image fill `image-id` of `shape-id`. + + The video is muted and looped: browsers refuse to autoplay audible video, and + there is no playback UI to unmute it with." + [shape-id image-id source] + (detach! image-id) + (when (playable?) + (attach-element! shape-id image-id source))) + +(defn image-fill-id + "Image painted by `shape`, from its fills or, for image shapes, its metadata." + [shape] + (or (first (types.fills/get-image-ids (types.fills/coerce (:fills shape)))) + (get-in shape [:metadata :id]))) + +(defn video-fill + "The shape's fill image, when the uploaded media is video rather than a still." + [shape] + (->> (types.fills/coerce (:fills shape)) + (seq) + (keep :fill-image) + (filter #(cm/video-type? (:mtype %))) + (first))) + +(defn shape-source + "Where `shape` plays its video from: an uploaded video asset if it has one, + otherwise the `:video` attribute naming a file the browser can reach." + [shape] + (if-let [image (video-fill shape)] + (cf/resolve-file-media image) + (:video shape))) + +(defn playing? + [image-id] + (boolean (when-let [^js element (get-in @videos [image-id :element])] + (not (.-paused element))))) + +(defn toggle-play! + [image-id] + (when-let [^js element (get-in @videos [image-id :element])] + (if (.-paused element) + (-> (.play element) + (.then (fn [_] (request-render!))) + (.catch (fn [cause] + (log/error :hint "Could not play video" :cause cause)))) + (.pause element)) + (not (.-paused element)))) + +(defn sync-shape! + "Reconciles one shape with its `:video` attribute. Called whenever the + attribute changes, so attaching is a plain shape edit and undo comes free. + + Returns nil: `set-wasm-attr!` treats what it gets back as pending image + loads, and `detach!` would otherwise hand it the video registry." + [shape] + (when-let [image-id (image-fill-id shape)] + (let [source (shape-source shape)] + (cond + (str/blank? source) + (detach! image-id) + + (not= source (get-in @videos [image-id :source])) + (attach! (:id shape) image-id source)))) + nil) + +(defn sync-shapes! + "Reconciles every attachment with `objects`: starts the videos the shapes ask + for and stops the ones no shape wants any more. Run on page load, so a video + survives a reload; an unchanged attachment keeps playing." + [objects] + (let [wanted (into {} + (keep (fn [[shape-id shape]] + (let [source (shape-source shape)] + (when-not (str/blank? source) + (when-let [image-id (image-fill-id shape)] + [image-id {:shape-id shape-id :source source}]))))) + objects)] + (doseq [[image-id {:keys [source]}] @videos] + (when (not= source (get-in wanted [image-id :source])) + (detach! image-id))) + (doseq [[image-id {:keys [shape-id source]}] wanted] + (when-not (contains? @videos image-id) + (attach! shape-id image-id source))))) + +(defn attach-to-selected! + "Console entry point: attaches `source` to the selected image shape." + [source] + (let [state (deref st/state) + shape (some->> (first (dsh/lookup-selected state)) + (dsh/lookup-shape state))] + (if-let [image-id (some-> shape image-fill-id)] + (do (attach! (:id shape) image-id source) true) + (do (log/error :hint "Select a shape with an image fill first" + :shape-id (:id shape) + :shape-type (:type shape)) + false)))) + +;; Reachable from the browser console while there is no UI for this. +(unchecked-set js/globalThis "penpotAttachVideo" attach-to-selected!) +(unchecked-set js/globalThis "penpotDetachVideos" detach-all!) diff --git a/frontend/src/app/render_wasm/api/webgl.cljs b/frontend/src/app/render_wasm/api/webgl.cljs index 862374ec01..fa2266a341 100644 --- a/frontend/src/app/render_wasm/api/webgl.cljs +++ b/frontend/src/app/render_wasm/api/webgl.cljs @@ -23,6 +23,24 @@ (when current-ctx (.-GLctx ^js current-ctx))))))) +(defn register-texture! + "Registers a WebGL texture with Emscripten's GL object system and returns its ID, + which is what WASM needs to wrap the texture as a Skia image." + [texture] + (let [gl-obj (unchecked-get wasm/internal-module "GL") + textures (.-textures ^js gl-obj) + new-id (.getNewId ^js gl-obj textures)] + (aset textures new-id texture) + new-id)) + +(defn upload-texture-source! + "Uploads `source` — an HTMLImageElement, ImageBitmap or HTMLVideoElement — into + an existing texture." + [gl texture source] + (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) texture) + (.texImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 (.-RGBA ^js gl) (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) source) + (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil)) + (defn create-webgl-texture-from-image "Creates a WebGL texture from an HTMLImageElement or ImageBitmap and returns the texture object" [gl image-element] @@ -32,8 +50,8 @@ (.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_WRAP_T ^js gl) (.-CLAMP_TO_EDGE ^js gl)) (.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_MIN_FILTER ^js gl) (.-LINEAR ^js gl)) (.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_MAG_FILTER ^js gl) (.-LINEAR ^js gl)) - (.texImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 (.-RGBA ^js gl) (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) image-element) (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil) + (upload-texture-source! gl texture image-element) texture)) ;; FIXME: temporary function until we are able to keep the same across pages. diff --git a/frontend/src/app/render_wasm/shape.cljs b/frontend/src/app/render_wasm/shape.cljs index 0dd5870675..6f419360c9 100644 --- a/frontend/src/app/render_wasm/shape.cljs +++ b/frontend/src/app/render_wasm/shape.cljs @@ -14,6 +14,7 @@ [app.common.types.shape.layout :as ctl] [app.main.refs :as refs] [app.render-wasm.api :as api] + [app.render-wasm.api.video :as video] [app.render-wasm.svg-filters :as svg-filters] [beicon.v2.core :as rx] [cljs.core :as c] @@ -313,6 +314,9 @@ (ctl/flex-layout? shape) (api/set-flex-layout shape)) + :video + (video/sync-shape! shape) + ;; Property not in WASM nil)))) diff --git a/frontend/translations/en.po b/frontend/translations/en.po index 128f102aa1..5e1c18fcc4 100644 --- a/frontend/translations/en.po +++ b/frontend/translations/en.po @@ -8895,6 +8895,30 @@ msgstr "Uppercase" msgid "workspace.options.use-play-button" msgstr "Use the play button at the header to run the prototype view." +#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:66 +msgid "workspace.options.video" +msgstr "Video" + +#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:73 +msgid "workspace.options.video.pause" +msgstr "Pause video" + +#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:80 +msgid "workspace.options.video.placeholder" +msgstr "clip.mp4" + +#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:74 +msgid "workspace.options.video.play" +msgstr "Play video" + +#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:86 +msgid "workspace.options.video.remove" +msgstr "Remove video" + +#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:82 +msgid "workspace.options.video.source" +msgstr "Video source" + #: src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:538, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:560 msgid "workspace.options.width" msgstr "Width" diff --git a/render-wasm/src/render/gpu_state.rs b/render-wasm/src/render/gpu_state.rs index 7a158b5fc6..81e3b6193d 100644 --- a/render-wasm/src/render/gpu_state.rs +++ b/render-wasm/src/render/gpu_state.rs @@ -55,6 +55,14 @@ impl GpuState { }) } + /// Drops the texture bindings Skia believes are current. Required whenever + /// JS writes a texture behind Ganesh's back (`texImage2D` from the CLJS + /// side): Skia caches which texture is bound to each unit, so after an + /// external bind it would keep drawing with a stale one. + pub fn reset_texture_bindings(&mut self) { + self.context.reset_gl_texture_bindings(); + } + pub fn max_texture_size(&self) -> i32 { self.context .max_texture_size() diff --git a/render-wasm/src/render/images.rs b/render-wasm/src/render/images.rs index 4b1ce66754..fb5090eb83 100644 --- a/render-wasm/src/render/images.rs +++ b/render-wasm/src/render/images.rs @@ -398,6 +398,38 @@ impl ImageStore { Ok(()) } + /// Rebinds an already-registered image to a different GL texture, or to the + /// same texture after its contents changed. Video frames go through here: + /// the CLJS side uploads each decoded frame into one texture it owns and + /// calls this, which re-wraps it as a fresh `skia::Image`. Mutating the + /// texture under a live `Image` is not enough — Skia keys its caches + /// (filters, mipmaps, tile contents) on the image's unique id, so it would + /// keep serving the frame it first saw. + pub fn rebind_gl_texture( + &mut self, + id: Uuid, + is_thumbnail: bool, + texture_id: u32, + width: i32, + height: i32, + ) -> Result<()> { + let Some(context) = self.context.as_mut() else { + return Err(crate::error::Error::CriticalError( + "Cannot rebind a GL texture without a GPU context".to_string(), + )); + }; + let image = create_image_from_gl_texture(context, texture_id, width, height)?; + + let key = (id, is_thumbnail); + if let Some(previous) = self.images.remove(&key) { + self.total_bytes -= previous.bytes; + } + let bytes = (width as usize) * (height as usize) * 4; + self.insert_entry(key, StoredImage::Gpu(image), bytes); + + Ok(()) + } + pub fn contains(&self, id: &Uuid, is_thumbnail: bool) -> bool { self.images.contains_key(&(*id, is_thumbnail)) } diff --git a/render-wasm/src/wasm/fills/image.rs b/render-wasm/src/wasm/fills/image.rs index e6b2e634c5..52c4d9f6a9 100644 --- a/render-wasm/src/wasm/fills/image.rs +++ b/render-wasm/src/wasm/fills/image.rs @@ -1,4 +1,5 @@ use crate::error::{Error, Result}; +use crate::get_gpu_state; use crate::get_resources; use crate::mem; use crate::shapes::Fill; @@ -183,6 +184,56 @@ pub extern "C" fn store_image_url(a: u32, b: u32, c: u32, d: u32) -> Result<()> Ok(()) } +/// A GL texture handed over from JS, as written by `store-image-texture` / +/// `upload-video-frame!` on the CLJS side. +struct TextureUpload { + ids: ShapeImageIds, + is_thumbnail: bool, + texture_id: u32, + width: i32, + height: i32, +} + +const TEXTURE_UPLOAD_SIZE: usize = 48; // header + texture id + width + height + +/// Reads a `TextureUpload` out of the shared buffer. The buffer is freed by the +/// caller on success and here on failure, so a malformed payload cannot leak it. +fn read_texture_upload() -> Result { + let bytes = mem::bytes(); + + if bytes.len() < TEXTURE_UPLOAD_SIZE { + // FIXME: Review if this should be an critical or a recoverable error. + eprintln!("read_texture_upload: insufficient data"); + mem::free_bytes()?; + return Err(Error::RecoverableError( + "read_texture_upload: insufficient data".to_string(), + )); + } + + let ids = ShapeImageIds::try_from(&bytes[0..IMAGE_IDS_SIZE]) + .map_err(|_| Error::CriticalError("Invalid image ids".to_string()))?; + + // FIXME: read bytes in a safe way + let read_u32 = |range: std::ops::Range, what: &str| -> Result { + Ok(u32::from_le_bytes((&bytes[range]).try_into().map_err( + |_| Error::CriticalError(format!("Invalid bytes for {}", what)), + )?)) + }; + + let is_thumbnail = read_u32(IMAGE_IDS_SIZE..IMAGE_HEADER_SIZE, "is_thumbnail flag")? != 0; + let texture_id = read_u32(36..40, "texture id")?; + let width = read_u32(40..44, "width")? as i32; + let height = read_u32(44..48, "height")? as i32; + + Ok(TextureUpload { + ids, + is_thumbnail, + texture_id, + width, + height, + }) +} + /// Stores an image from an existing WebGL texture, avoiding re-decoding /// Expected memory layout: /// - bytes 0-15: shape UUID @@ -194,66 +245,52 @@ pub extern "C" fn store_image_url(a: u32, b: u32, c: u32, d: u32) -> Result<()> #[no_mangle] #[wasm_error] pub extern "C" fn store_image_from_texture() -> Result<()> { - let bytes = mem::bytes(); - - // FIXME: where does this 48 come from? - if bytes.len() < 48 { - // FIXME: Review if this should be an critical or a recoverable error. - eprintln!("store_image_from_texture: insufficient data"); - mem::free_bytes()?; - return Err(Error::RecoverableError( - "store_image_from_texture: insufficient data".to_string(), - )); - } - - let ids = ShapeImageIds::try_from(&bytes[0..IMAGE_IDS_SIZE]) - .map_err(|_| Error::CriticalError("Invalid image ids".to_string()))?; - - // FIXME: read bytes in a safe way - - // Read is_thumbnail flag (4 bytes as u32) - let is_thumbnail_bytes = &bytes[IMAGE_IDS_SIZE..IMAGE_HEADER_SIZE]; - let is_thumbnail_value = - u32::from_le_bytes(is_thumbnail_bytes.try_into().map_err(|_| { - Error::CriticalError("Invalid bytes for is_thumbnail flag".to_string()) - })?); - let is_thumbnail = is_thumbnail_value != 0; - - // Read GL texture ID (4 bytes as u32) - let texture_id_bytes = &bytes[36..40]; - let texture_id = u32::from_le_bytes( - texture_id_bytes - .try_into() - .map_err(|_| Error::CriticalError("Invalid bytes for texture id".to_string()))?, - ); - - // Read width and height (8 bytes as two i32s) - let width_bytes = &bytes[40..44]; - let width = i32::from_le_bytes( - width_bytes - .try_into() - .map_err(|_| Error::CriticalError("Invalid bytes for width".to_string()))?, - ); - - let height_bytes = &bytes[44..48]; - let height = i32::from_le_bytes( - height_bytes - .try_into() - .map_err(|_| Error::CriticalError("Invalid bytes for height".to_string()))?, - ); + let upload = read_texture_upload()?; with_state!(state, { if let Err(msg) = get_resources().images.add_image_from_gl_texture( - ids.image_id, - is_thumbnail, - texture_id, - width, - height, + upload.ids.image_id, + upload.is_thumbnail, + upload.texture_id, + upload.width, + upload.height, ) { // FIXME: Review if we should return a RecoverableError eprintln!("store_image_from_texture error: {}", msg); } - touch_shapes_with_image(state, ids.image_id); + touch_shapes_with_image(state, upload.ids.image_id); + }); + + mem::free_bytes()?; + Ok(()) +} + +/// Rebinds an already-stored image to the current contents of a GL texture and +/// invalidates the tiles of every shape that paints it. Same memory layout as +/// `store_image_from_texture`. +/// +/// This is the per-frame entry point for video: the CLJS side owns one texture +/// per playing video, uploads the decoded frame into it and calls this. +#[no_mangle] +#[wasm_error] +pub extern "C" fn update_image_from_texture() -> Result<()> { + let upload = read_texture_upload()?; + + // The caller just bound and wrote the texture from JS, which Skia has no + // way to observe. Drop its cached bindings before it samples the texture. + get_gpu_state().reset_texture_bindings(); + + with_state!(state, { + if let Err(msg) = get_resources().images.rebind_gl_texture( + upload.ids.image_id, + upload.is_thumbnail, + upload.texture_id, + upload.width, + upload.height, + ) { + eprintln!("update_image_from_texture error: {}", msg); + } + touch_shapes_with_image(state, upload.ids.image_id); }); mem::free_bytes()?;