wip

backend/src/app/media/validation.clj

@@ -33,7 +33,9 @@
   (sm/check-fn schema:input))
 
 (defn validate-media-type!
-  ([upload] (validate-media-type! upload cm/image-types))
+  ;; `upload-types` rather than `image-types`: a file can also hold video.
+  ;; Callers that must stay image-only pass `cm/image-types` explicitly.
+  ([upload] (validate-media-type! upload cm/upload-types))
   ([upload allowed]
    (when-not (contains? allowed (:mtype upload))
      (ex/raise :type :validation

backend/src/app/rpc/commands/media.clj

@@ -9,6 +9,7 @@
    [app.common.data :as d]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
+   [app.common.media :as cm]
    [app.common.schema :as sm]
    [app.common.time :as ct]
    [app.common.uuid :as uuid]
@@ -52,7 +53,11 @@
    [:file-id ::sm/uuid]
    [:is-local ::sm/boolean]
    [:name [:string {:max 250}]]
-   [:content media.v/schema:upload]])
+   [:content media.v/schema:upload]
+   ;; Only video sends these: nothing on the backend can read them from the
+   ;; file, so the client reports what it decoded.
+   [:width {:optional true} ::sm/int]
+   [:height {:optional true} ::sm/int]])
 
 (sv/defmethod ::upload-file-media-object
   {::doc/added "1.17"
@@ -170,9 +175,31 @@
       :always
       (assoc ::image (process-main-image info)))))
 
+(defn- process-video
+  "Video is stored and served, never decoded: nothing here reads its
+   dimensions or renders a frame to thumbnail. The client sends the dimensions
+   it read from the file. Shaped like `process-image` so the caller does not
+   care which one ran."
+  [content width height]
+  (when-not (and width height)
+    (ex/raise :type :validation
+              :code :missing-video-dimensions
+              :hint "video uploads must report the dimensions the client decoded"))
+  (let [info (assoc content
+                    :ts (ct/now)
+                    :width width
+                    :height height)]
+    (assoc info ::image (process-main-image info))))
+
+(defn- process-content
+  [cfg {:keys [mtype] :as content} width height]
+  (if (cm/video-type? mtype)
+    (process-video content width height)
+    (process-image cfg content)))
+
 (defn- create-file-media-object
   [{:keys [::sto/storage ::db/conn] :as cfg}
-   {:keys [id file-id is-local name content from-url? from-chunks?]}]
+   {:keys [id file-id is-local name content width height from-url? from-chunks?]}]
 
   (let [tpoint (ct/tpoint)
         id     (or id (uuid/next))
@@ -192,7 +219,7 @@
            :path (str (:path content))
            :origin origin)
 
-    (let [result  (process-image cfg content)
+    (let [result  (process-content cfg content width height)
           image   (sto/put-object! storage (::image result))
           thumb   (when-let [params (::thumb result)]
                     (sto/put-object! storage params))
@@ -453,7 +480,9 @@
    [:is-local   ::sm/boolean]
    [:name       [:string {:max 250}]]
    [:mtype      :string]
-   [:id         {:optional true} ::sm/uuid]])
+   [:id         {:optional true} ::sm/uuid]
+   [:width      {:optional true} ::sm/int]
+   [:height     {:optional true} ::sm/int]])
 
 (sv/defmethod ::assemble-file-media-object
   {::doc/added "2.17"

common/src/app/common/media.cljc

@@ -22,6 +22,20 @@
     "image/gif"
     "image/svg+xml"})
 
+(def video-types
+  #{"video/mp4"
+    "video/webm"})
+
+(defn video-type?
+  [mtype]
+  (contains? video-types mtype))
+
+;; Media a file can hold. Video is a proof of concept: the backend stores and
+;; serves it but decodes nothing, so it carries the dimensions the client read
+;; from the file.
+(def upload-types
+  (into image-types video-types))
+
 (def tempfile-types
   (conj image-types "application/pdf" "application/zip"))
 
@@ -32,7 +46,9 @@
     :jpeg ".jpg"
     :webp ".webp"
     :gif ".gif"
-    :svg  ".svg"))
+    :svg  ".svg"
+    :mp4  ".mp4"
+    :webm ".webm"))
 
 (defn format->mtype
   [format]
@@ -43,6 +59,8 @@
     :webp "image/webp"
     :gif "image/gif"
     :svg  "image/svg+xml"
+    :mp4  "video/mp4"
+    :webm "video/webm"
     "application/octet-stream"))
 
 (defn mtype->format
@@ -53,6 +71,8 @@
     "image/webp"    :webp
     "image/gif"     :gif
     "image/svg+xml" :svg
+    "video/mp4"     :mp4
+    "video/webm"    :webm
     nil))
 
 (defn mtype->extension [mtype]
@@ -65,6 +85,8 @@
     "image/png"                ".png"
     "image/svg+xml"            ".svg"
     "image/webp"               ".webp"
+    "video/mp4"                ".mp4"
+    "video/webm"               ".webm"
     "application/zip"          ".zip"
     "application/penpot"       ".penpot"
     "application/pdf"          ".pdf"
@@ -78,7 +100,7 @@
 
 (defn strip-image-extension
   [filename]
-  (let [image-extensions-re #"(\.png)|(\.jpg)|(\.jpeg)|(\.webp)|(\.gif)|(\.svg)$"]
+  (let [image-extensions-re #"(\.png)|(\.jpg)|(\.jpeg)|(\.webp)|(\.gif)|(\.svg)|(\.mp4)|(\.webm)$"]
     (str/replace filename image-extensions-re "")))
 
 (defn parse-font-weight

common/src/app/common/types/fills/impl.cljc

@@ -160,7 +160,9 @@
                   "image/png"     0x02
                   "image/gif"     0x03
                   "image/webp"    0x04
-                  "image/svg+xml" 0x05)]
+                  "image/svg+xml" 0x05
+                  "video/mp4"     0x06
+                  "video/webm"    0x07)]
         (buf/write-short buffer (+ offset 2) val)))
 
     (if (and (some? ref-file)
@@ -241,7 +243,10 @@
                                     0x02 "image/png"
                                     0x03 "image/gif"
                                     0x04 "image/webp"
-                                    0x05 "image/svg+xml")]
+                                    0x05 "image/svg+xml"
+                                    0x06 "video/mp4"
+                                    0x07 "video/webm"
+                                    nil)]
                     {:fill-opacity opacity
                      :fill-image   (cond-> {:id                id
                                             :width             width

common/src/app/common/types/shape.cljc

@@ -276,7 +276,13 @@
    [:svg-attrs {:optional true} :map]
    [:svg-defs {:optional true} :map]
    [:svg-transform {:optional true} :map]
-   [:svg-viewbox {:optional true} :map]])
+   [:svg-viewbox {:optional true} :map]
+
+   ;; Video proof of concept: the source played into the shape's image fill,
+   ;; either a name under the static asset folder or a full URL. Only the
+   ;; render-wasm renderer paints it; every other consumer keeps showing the
+   ;; image fill, which doubles as the poster frame.
+   [:video {:optional true} [:maybe :string]]])
 
 (def schema:group-attrs
   [:map {:title "GroupAttrs"}
@@ -516,7 +522,7 @@
     :hidden :masked-group :fills :proportion :proportion-lock :constraints-h
     :constraints-v :fixed-scroll :r1 :r2 :r3 :r4 :rotation :opacity :grids :exports
     :strokes :blend-mode :interactions :shadow :blur :background-blur :grow-type :applied-tokens
-    :plugin-data})
+    :plugin-data :video})
 
 (def ^:private allowed-shape-geom-attrs #{:x :y :width :height})
 (def ^:private allowed-shape-base-attrs #{:id :name :type :selrect :points :transform

docker/devenv/files/nginx.conf

@@ -278,7 +278,7 @@ http {
                 add_header X-Cache-Status $upstream_cache_status;
             }
 
-            location ~* \.(jpg|png|svg|ttf|woff|woff2|gif)$ {
+            location ~* \.(jpg|png|svg|ttf|woff|woff2|gif|mp4|webm|mov)$ {
                 include /home/penpot/penpot/docker/devenv/files/nginx-security-headers.conf;
                 add_header Cache-Control "public, max-age=604800" always; # 7 days
             }

frontend/src/app/main/data/media.cljs

@@ -40,7 +40,7 @@
 (defn validate-file
   "Check that a file obtained with the file javascript API is valid."
   [file]
-  (when-not (contains? cm/image-types (.-type file))
+  (when-not (contains? cm/upload-types (.-type file))
     (ex/raise :type :validation
               :code :media-type-not-allowed
               :hint (str/ffmt "media type % is not supported" (.-type file))))

frontend/src/app/main/data/workspace/media.cljs

@@ -38,7 +38,32 @@
    [tubax.core :as tubax]))
 
 (def accept-image-types
-  (str/join "," media/image-types))
+  (str/join "," media/upload-types))
+
+(defn- read-video-dimensions
+  "Resolves the intrinsic size of a video blob. Nothing on the backend decodes
+   video, so the client is what reports the dimensions an upload is stored
+   with."
+  [blob]
+  (p/create
+   (fn [resolve reject]
+     (let [url (js/URL.createObjectURL blob)
+           element (js/document.createElement "video")
+           done (fn [f value]
+                  (js/URL.revokeObjectURL url)
+                  (f value))]
+       (set! (.-preload element) "metadata")
+       (set! (.-muted element) true)
+       (set! (.-onloadedmetadata element)
+             (fn [_]
+               (done resolve {:width (.-videoWidth element)
+                              :height (.-videoHeight element)})))
+       (set! (.-onerror element)
+             (fn [_]
+               (done reject (ex/error :type :validation
+                                      :code :invalid-video
+                                      :hint "could not read the video dimensions"))))
+       (set! (.-src element) url)))))
 
 (defn- optimize
   [input]
@@ -123,17 +148,19 @@
   "Uploads `blob` to `file-id` as a chunked media object using the
   three-step session API.  Returns an observable that emits the
   assembled file-media-object map."
-  [{:keys [file-id name is-local blob]}]
+  [{:keys [file-id name is-local blob width height]}]
   (let [mtype (.-type blob)]
     (->> (uploads/upload-blob-chunked blob)
          (rx/mapcat
           (fn [{:keys [session-id]}]
             (rp/cmd! :assemble-file-media-object
-                     {:session-id session-id
+                     (cond-> {:session-id session-id
-                      :file-id    file-id
+                              :file-id    file-id
-                      :is-local   is-local
+                              :is-local   is-local
-                      :name       name
+                              :name       name
-                      :mtype      mtype}))))))
+                              :mtype      mtype}
+                       (some? width)  (assoc :width width)
+                       (some? height) (assoc :height height))))))))
 
 (defn process-uris
   [{:keys [file-id local? name uris mtype on-image on-svg]}]
@@ -175,18 +202,26 @@
             (and (not force-media)
                  (= (.-type blob) "image/svg+xml")))
 
-          (upload-blob [blob]
+          (upload-blob* [blob dimensions]
-            (let [params {:file-id  file-id
+            (let [params (merge {:file-id  file-id
-                          :name     (or name (if (dmm/file? blob) (media/strip-image-extension (.-name blob)) "blob"))
+                                 :name     (or name (if (dmm/file? blob) (media/strip-image-extension (.-name blob)) "blob"))
-                          :is-local local?
+                                 :is-local local?
-                          :blob     blob}]
+                                 :blob     blob}
+                                dimensions)]
               (if (>= (.-size blob) chunk-size)
                 (upload-blob-chunked params)
                 (rp/cmd! :upload-file-media-object
-                         {:file-id  file-id
+                         (merge {:file-id  file-id
-                          :name     (:name params)
+                                 :name     (:name params)
-                          :is-local local?
+                                 :is-local local?
-                          :content  blob}))))
+                                 :content  blob}
+                                dimensions)))))
+
+          (upload-blob [blob]
+            (if (media/video-type? (.-type blob))
+              (->> (rx/from (read-video-dimensions blob))
+                   (rx/mapcat #(upload-blob* blob %)))
+              (upload-blob* blob nil)))
 
           (extract-content [blob]
             (let [name (or name (.-name blob))]

frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs

@@ -0,0 +1,92 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS SUBSIDIARY SL
+
+(ns app.main.ui.workspace.sidebar.options.menus.video
+  "Proof of concept: plays a video into a shape's image fill.
+
+   Nothing uploads video, so the source is a path the browser can already
+   reach — a file under `frontend/resources/public/images/` or a full URL. Only
+   the render-wasm renderer paints it; elsewhere the image fill still shows."
+  (:require-macros [app.main.style :as stl])
+  (:require
+   [app.main.data.workspace.shapes :as dwsh]
+   [app.main.store :as st]
+   [app.main.ui.components.title-bar :refer [title-bar*]]
+   [app.main.ui.ds.buttons.icon-button :refer [icon-button*]]
+   [app.main.ui.ds.controls.input :refer [input*]]
+   [app.main.ui.ds.foundations.assets.icon :as i]
+   [app.render-wasm.api.video :as video]
+   [app.util.dom :as dom]
+   [app.util.i18n :refer [tr]]
+   [clojure.string :as str]
+   [rumext.v2 :as mf]))
+
+(defn set-video
+  "Stores the source on the shape. `app.render-wasm.shape` picks the change up
+   and attaches or detaches the video, so undo and reload both behave."
+  [ids source]
+  (dwsh/update-shapes ids (fn [shape]
+                            (if (str/blank? source)
+                              (dissoc shape :video)
+                              (assoc shape :video source)))))
+
+(mf/defc video-menu*
+  [{:keys [ids image-id source is-asset]}]
+  (let [;; `video/playing?` reads the element, which is outside app state, so
+        ;; the button tracks it locally.
+        playing* (mf/use-state #(video/playing? image-id))
+        playing  (deref playing*)
+
+        has-source (not (str/blank? source))
+
+        on-change
+        (mf/use-fn
+         (mf/deps ids)
+         (fn [event]
+           (let [value (-> event dom/get-target dom/get-value str/trim)]
+             (st/emit! (set-video ids value)))))
+
+        on-toggle-play
+        (mf/use-fn
+         (mf/deps image-id)
+         (fn []
+           (reset! playing* (video/toggle-play! image-id))))
+
+        on-remove
+        (mf/use-fn
+         (mf/deps ids)
+         (fn []
+           (st/emit! (set-video ids nil))))]
+
+    [:section {:class (stl/css :element-set)
+               :aria-label (tr "workspace.options.video")}
+     [:div {:class (stl/css :element-title)}
+      [:> title-bar* {:collapsable false
+                      :title (tr "workspace.options.video")}
+       (when has-source
+         [:> icon-button* {:variant "ghost"
+                           :aria-label (if playing
+                                         (tr "workspace.options.video.pause")
+                                         (tr "workspace.options.video.play"))
+                           :on-click on-toggle-play
+                           :selected playing
+                           :tooltip-placement "top-left"
+                           :icon i/play}])]]
+     ;; An uploaded video has nothing to type: its source is the asset itself,
+     ;; and removing it means deleting the shape.
+     (when-not is-asset
+       [:div {:class (stl/css :row)}
+        [:> input* {:class (stl/css :source-input)
+                    :placeholder (tr "workspace.options.video.placeholder")
+                    :default-value (or source "")
+                    :aria-label (tr "workspace.options.video.source")
+                    :on-blur on-change}]
+        [:> icon-button* {:variant "ghost"
+                          :aria-label (tr "workspace.options.video.remove")
+                          :on-click on-remove
+                          :disabled (not has-source)
+                          :tooltip-placement "top-left"
+                          :icon i/remove}]])]))

frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss

@@ -0,0 +1,23 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// Copyright (c) KALEIDOS SUBSIDIARY SL
+
+@use "../../../sidebar/common/sidebar.scss" as sidebar;
+
+.element-set {
+  max-width: var(--options-width);
+}
+
+.row {
+  display: flex;
+  align-items: center;
+  gap: var(--sp-xs);
+  margin-bottom: var(--sp-s);
+}
+
+.source-input {
+  flex-grow: 1;
+  min-width: 0;
+}

frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs

@@ -8,6 +8,7 @@
   (:require
    [app.common.data.macros :as dm]
    [app.common.types.shape.layout :as ctl]
+   [app.main.features :as features]
    [app.main.refs :as refs]
    [app.main.ui.workspace.sidebar.options.menus.blur :refer [blur-menu*]]
    [app.main.ui.workspace.sidebar.options.menus.constraints :refer [constraint-attrs constraints-menu*]]
@@ -21,6 +22,8 @@
    [app.main.ui.workspace.sidebar.options.menus.shadow :refer [shadow-menu*]]
    [app.main.ui.workspace.sidebar.options.menus.stroke :refer [stroke-attrs stroke-menu*]]
    [app.main.ui.workspace.sidebar.options.menus.svg-attrs :refer [svg-attrs-menu*]]
+   [app.main.ui.workspace.sidebar.options.menus.video :refer [video-menu*]]
+   [app.render-wasm.api.video :as video]
    [rumext.v2 :as mf]))
 
 (mf/defc options*
@@ -80,7 +83,19 @@
           (refs/parents-by-ids ids))
 
         parents
-        (mf/deref parents-by-ids-ref)]
+        (mf/deref parents-by-ids-ref)
+
+        ;; Video is painted by re-uploading frames into the shape's image fill,
+        ;; and only the wasm renderer does that.
+        render-wasm?
+        (features/use-feature "render-wasm/v1")
+
+        video-image-id
+        (when render-wasm? (video/image-fill-id shape))
+
+        ;; An uploaded video asset needs no source field, only playback.
+        video-asset?
+        (and render-wasm? (some? (video/video-fill shape)))]
 
     [:*
      [:> layer-menu* {:ids ids
@@ -127,6 +142,12 @@
        :values shape
        :applied-tokens applied-tokens}]
 
+     (when (some? video-image-id)
+       [:> video-menu* {:ids ids
+                        :image-id video-image-id
+                        :source (video/shape-source shape)
+                        :is-asset video-asset?}])
+
      [:> stroke-menu* {:ids ids
                        :type type
                        :values stroke-values

frontend/src/app/render_wasm/api.cljs

@@ -16,6 +16,7 @@
    [app.common.fonts :as cfnt]
    [app.common.logging :as log]
    [app.common.math :as mth]
+   [app.common.media :as cm]
    [app.common.render-wasm.api.props :as props]
    [app.common.render-wasm.api.upload :as upload]
    [app.common.render-wasm.helpers :as h]
@@ -45,6 +46,7 @@
    [app.render-wasm.api.enums]
    [app.render-wasm.api.fonts :as f]
    [app.render-wasm.api.texts :as t]
+   [app.render-wasm.api.video :as video]
    [app.render-wasm.api.webgl :as webgl]
    [app.render-wasm.deserializers :as dr]
    [app.render-wasm.gesture :as wasm-gesture]
@@ -490,6 +492,10 @@
 (defn- render
   [timestamp]
   (when (wasm/live?)
+    ;; Upload pending video frames before drawing, so this frame paints the
+    ;; current one instead of the previous.
+    (video/tick!)
+
     ;; SYNC-TILES makes WASM keep the last presented frame while the new tiles
     ;; are rasterized, rather than clearing to the background first. The flag is
     ;; one-shot on both sides: WASM clears it when the render loop starts, so
@@ -510,6 +516,12 @@
         (js/console.error "text-editor overlay/update failed:" e)))
 
     (set! wasm/internal-frame-id nil)
+
+    ;; Playing video has no other trigger to keep the loop alive. Requested
+    ;; after the frame id is cleared so the scheduled frame is the one kept.
+    (when (video/active?)
+      (request-render "video-frame"))
+
     (ug/dispatch! (ug/event "penpot:wasm:render"))))
 
 (defn render-ui-only
@@ -663,6 +675,8 @@
                                 (throw e))))))]
           (set! wasm/internal-frame-id frame-id))))))
 
+(video/set-render-requester! request-render)
+
 (defn request-render-preserving-target
   "Like `request-render`, but keeps the previously presented frame on screen
    while the new tiles are rasterized instead of blanking the canvas first.
@@ -845,15 +859,6 @@
   nil)
 
 
-(defn- get-texture-id-for-gl-object
-  "Registers a WebGL texture with Emscripten's GL object system and returns its ID"
-  [texture]
-  (let [gl-obj (unchecked-get wasm/internal-module "GL")
-        textures (.-textures ^js gl-obj)
-        new-id (.getNewId ^js gl-obj textures)]
-    (aset textures new-id texture)
-    new-id))
-
 (defn- svg-blob?
   [^js blob]
   (str/starts-with? (.-type blob) "image/svg"))
@@ -905,7 +910,7 @@
   [shape-id image-id thumbnail? img]
   (when-let [gl (webgl/get-webgl-context)]
     (let [texture (webgl/create-webgl-texture-from-image gl img)
-          texture-id (get-texture-id-for-gl-object texture)
+          texture-id (webgl/register-texture! texture)
           width  (.-width ^js img)
           height (.-height ^js img)
           ;; Header: 32 bytes (2 UUIDs) + 4 bytes (thumbnail)
@@ -964,6 +969,17 @@
                           :cause cause)
                (rx/empty)))))}))
 
+(defn- still-image-ids
+  "Image ids the renderer has to fetch. Video fills are skipped: their frames
+   are uploaded by `app.render-wasm.api.video`, so downloading the container as
+   an image would burn the bandwidth and then fail to decode."
+  [fills]
+  (into #{}
+        (comp (keep :fill-image)
+              (remove #(cm/video-type? (:mtype %)))
+              (map :id))
+        (seq fills)))
+
 (defn- get-fill-images
   [leaf]
   (filter :fill-image (:fills leaf)))
@@ -1018,7 +1034,7 @@
                  (store-image-url! id (cf/resolve-file-media {:id id} thumbnail?))
                  (when (zero? cached-image?)
                    (fetch-image shape-id id thumbnail?))))
-             (types.fills/get-image-ids fills))))))
+             (still-image-ids fills))))))
 
 (defn- stroke-image-ids
   [strokes]
@@ -1743,6 +1759,9 @@
    update-text-layouts fires for all text shapes after fonts load — not
    just the first shape that triggered the fetch."
   [shapes]
+  ;; Newly added shapes never reach `set-wasm-attr!`, which is what starts a
+  ;; video on an edit, so a shape created with one is started here instead.
+  (run! video/sync-shape! shapes)
   (let [total-shapes (count shapes)
         {:keys [thumbnails full text-font-state]}
         (loop [index 0
@@ -2035,6 +2054,9 @@
    (set-objects objects render-callback nil false))
   ([objects render-callback on-shapes-ready force-sync]
    (when (wasm/live?)
+     ;; Start (and stop) the videos this page asks for. Cheap: one pass over
+     ;; the objects, and attachments that did not change keep playing.
+     (video/sync-shapes! objects)
      (perf/begin-measure "set-objects")
      (let [shapes (shapes-in-tree-order objects)
            total-shapes (count shapes)]
@@ -2611,6 +2633,9 @@
     :as payload}]
   (ug/dispatch! (ug/event "penpot:wasm:reload-start"))
   (reset! wasm/reloading? true)
+  ;; Attached videos hold texture ids from the context that is about to be
+  ;; destroyed; a frame uploaded after the reload would wrap a stale texture.
+  (video/detach-all!)
   (let [fonts (derive-font-resources base-objects fonts)]
     (-> (p/resolved nil)
         ;; Keep teardown strict (`_clean_up` + deleteContext) but do not

frontend/src/app/render_wasm/api/video.cljs

@@ -0,0 +1,282 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS SUBSIDIARY SL
+
+(ns app.render-wasm.api.video
+  "Proof of concept: video frames painted as image fills.
+
+   Skia has no video decoder, so the browser decodes and we only hand Skia a
+   texture. Each attached video owns one WebGL texture; on every rendered frame
+   the current frame is uploaded into that texture and WASM re-wraps it, so
+   every shape whose image fill points at `image-id` paints it.
+
+   The source is stored on the shape as `:video`, so it persists with the file
+   and comes back on load. Drop a file into `frontend/resources/public/images/`,
+   where it is served like any other static asset, and name it from the Video
+   section of the design sidebar. `penpotAttachVideo(\"clip.mp4\")` does the same
+   from the console for the selected shape, without persisting it.
+
+   There is no backend support: nothing uploads video, so the source is a path
+   the browser can reach on its own."
+  (:require
+   [app.common.logging :as log]
+   [app.common.media :as cm]
+   [app.common.render-wasm.helpers :as h]
+   [app.common.render-wasm.mem :as mem]
+   [app.common.render-wasm.mem.heap32 :as mem.h32]
+   [app.common.render-wasm.wasm :as wasm]
+   [app.common.types.fills :as types.fills]
+   [app.config :as cf]
+   [app.main.data.helpers :as dsh]
+   [app.main.store :as st]
+   [app.render-wasm.api.webgl :as webgl]
+   [clojure.string :as str]))
+
+;; HTMLMediaElement.readyState: the element has data for the current position.
+(def ^:private HAVE-CURRENT-DATA 2)
+
+;; Videos live next to the other static assets, so a bare file name is enough.
+(def ^:private assets-path "/images/")
+
+;; image-id -> {:element :source :texture :texture-id :shape-id :image-id :last-time}
+;; The texture keys arrive later, once the video has decoded its first frame.
+(defonce ^:private videos (atom {}))
+
+;; `api.cljs` owns the render loop and requires this namespace, so it installs
+;; its requester here rather than being required back.
+(defonce ^:private render-requester (atom nil))
+
+(defn set-render-requester!
+  [f]
+  (reset! render-requester f))
+
+(defn- request-render!
+  []
+  (when-let [request @render-requester]
+    (request "video-frame")))
+
+(defn- playable?
+  "False in the render worker, which runs the same renderer with no DOM to
+   build a video element in."
+  []
+  (exists? js/document))
+
+(defn- send-frame!
+  "Hands the texture to WASM, which re-wraps it as a Skia image and invalidates
+   the tiles of every shape painting it. Same layout as `_store_image_from_texture`."
+  [{:keys [shape-id image-id texture-id ^js element]}]
+  (let [offset (mem/alloc->offset-32 48)
+        heap32 (mem/get-heap-u32)]
+    (mem.h32/write-uuid offset heap32 shape-id)
+    (mem.h32/write-uuid (+ offset 4) heap32 image-id)
+    (aset heap32 (+ offset 8) 0) ;; thumbnail flag
+    (aset heap32 (+ offset 9) texture-id)
+    (aset heap32 (+ offset 10) (.-videoWidth element))
+    (aset heap32 (+ offset 11) (.-videoHeight element))
+    (h/call wasm/internal-module "_update_image_from_texture")))
+
+(defn- decoded-frame?
+  [{:keys [^js element image-id texture-id]}]
+  (and (some? texture-id)
+       (>= (.-readyState element) HAVE-CURRENT-DATA)
+       (pos? (.-videoWidth element))
+       ;; rAF runs faster than most videos decode; skip the upload when
+       ;; playback has not advanced since the last one.
+       (not= (.-currentTime element) (get-in @videos [image-id :last-time]))))
+
+(defn- upload-frame!
+  [gl {:keys [^js element texture image-id] :as entry}]
+  (when (decoded-frame? entry)
+    (swap! videos assoc-in [image-id :last-time] (.-currentTime element))
+    (webgl/upload-texture-source! gl texture element)
+    (send-frame! entry)
+    true))
+
+(defn tick!
+  "Uploads a frame for every attached video that advanced. Called from the
+   renderer's rAF, before the frame is drawn."
+  []
+  (when (seq @videos)
+    (when-let [gl (webgl/get-webgl-context)]
+      (reduce (fn [uploaded entry]
+                (or (upload-frame! gl entry) uploaded))
+              false
+              (vals @videos)))))
+
+(defn active?
+  "True while some attached video is playing, so the renderer keeps scheduling
+   frames instead of settling."
+  []
+  (boolean (some (fn [{:keys [^js element]}] (not (.-paused element)))
+                 (vals @videos))))
+
+(defn detach!
+  "Stops a video. The last frame stays on screen: the texture is deliberately
+   kept alive because WASM holds a Skia image borrowing it."
+  [image-id]
+  (when-let [{:keys [^js element]} (get @videos image-id)]
+    (.pause element)
+    (.removeAttribute element "src")
+    (.load element)
+    (swap! videos dissoc image-id)))
+
+(defn detach-all!
+  []
+  (run! detach! (keys @videos)))
+
+(defn- register!
+  "Gives the video a texture once it has decoded something to fill it with."
+  [image-id ^js element]
+  (if-let [gl (webgl/get-webgl-context)]
+    (let [texture (webgl/create-webgl-texture-from-image gl element)]
+      (swap! videos update image-id merge
+             {:texture texture
+              :texture-id (webgl/register-texture! texture)})
+      (-> (.play element)
+          ;; Nothing else would ask for a frame: `tick!` only runs inside a
+          ;; render, and the loop only keeps itself alive once it has started.
+          (.then (fn [_] (request-render!)))
+          (.catch (fn [cause]
+                    (log/error :hint "Could not play video" :cause cause)))))
+    (log/error :hint "No WebGL context available for video")))
+
+(defn resolve-url
+  "Bare names resolve against the static asset folder; absolute paths and full
+   URLs are used as given."
+  [source]
+  (if (or (str/starts-with? source "/")
+          (str/includes? source "://"))
+    source
+    (str assets-path source)))
+
+(defn- attach-element!
+  [shape-id image-id source]
+  (let [url (resolve-url source)
+        element (js/document.createElement "video")]
+    ;; Registered before it loads, so a second `sync-shapes!` does not start a
+    ;; competing element for the same fill.
+    (swap! videos assoc image-id
+           {:element element
+            :source source
+            :shape-id shape-id
+            :image-id image-id
+            :last-time nil})
+    ;; Same-origin assets need no CORS, but a remote video served without the
+    ;; headers would taint the canvas and make `texImage2D` throw.
+    (set! (.-crossOrigin element) "anonymous")
+    (set! (.-muted element) true)
+    (set! (.-loop element) true)
+    (set! (.-playsInline element) true)
+    (set! (.-onerror element)
+          (fn [_]
+            (log/error :hint "Could not load video" :url url)
+            (detach! image-id)))
+    (.addEventListener element "loadeddata"
+                       (fn [] (register! image-id element))
+                       #js {:once true})
+    (set! (.-src element) url)
+    nil))
+
+(defn attach!
+  "Plays `source` into the image fill `image-id` of `shape-id`.
+
+   The video is muted and looped: browsers refuse to autoplay audible video, and
+   there is no playback UI to unmute it with."
+  [shape-id image-id source]
+  (detach! image-id)
+  (when (playable?)
+    (attach-element! shape-id image-id source)))
+
+(defn image-fill-id
+  "Image painted by `shape`, from its fills or, for image shapes, its metadata."
+  [shape]
+  (or (first (types.fills/get-image-ids (types.fills/coerce (:fills shape))))
+      (get-in shape [:metadata :id])))
+
+(defn video-fill
+  "The shape's fill image, when the uploaded media is video rather than a still."
+  [shape]
+  (->> (types.fills/coerce (:fills shape))
+       (seq)
+       (keep :fill-image)
+       (filter #(cm/video-type? (:mtype %)))
+       (first)))
+
+(defn shape-source
+  "Where `shape` plays its video from: an uploaded video asset if it has one,
+   otherwise the `:video` attribute naming a file the browser can reach."
+  [shape]
+  (if-let [image (video-fill shape)]
+    (cf/resolve-file-media image)
+    (:video shape)))
+
+(defn playing?
+  [image-id]
+  (boolean (when-let [^js element (get-in @videos [image-id :element])]
+             (not (.-paused element)))))
+
+(defn toggle-play!
+  [image-id]
+  (when-let [^js element (get-in @videos [image-id :element])]
+    (if (.-paused element)
+      (-> (.play element)
+          (.then (fn [_] (request-render!)))
+          (.catch (fn [cause]
+                    (log/error :hint "Could not play video" :cause cause))))
+      (.pause element))
+    (not (.-paused element))))
+
+(defn sync-shape!
+  "Reconciles one shape with its `:video` attribute. Called whenever the
+   attribute changes, so attaching is a plain shape edit and undo comes free.
+
+   Returns nil: `set-wasm-attr!` treats what it gets back as pending image
+   loads, and `detach!` would otherwise hand it the video registry."
+  [shape]
+  (when-let [image-id (image-fill-id shape)]
+    (let [source (shape-source shape)]
+      (cond
+        (str/blank? source)
+        (detach! image-id)
+
+        (not= source (get-in @videos [image-id :source]))
+        (attach! (:id shape) image-id source))))
+  nil)
+
+(defn sync-shapes!
+  "Reconciles every attachment with `objects`: starts the videos the shapes ask
+   for and stops the ones no shape wants any more. Run on page load, so a video
+   survives a reload; an unchanged attachment keeps playing."
+  [objects]
+  (let [wanted (into {}
+                     (keep (fn [[shape-id shape]]
+                             (let [source (shape-source shape)]
+                               (when-not (str/blank? source)
+                                 (when-let [image-id (image-fill-id shape)]
+                                   [image-id {:shape-id shape-id :source source}])))))
+                     objects)]
+    (doseq [[image-id {:keys [source]}] @videos]
+      (when (not= source (get-in wanted [image-id :source]))
+        (detach! image-id)))
+    (doseq [[image-id {:keys [shape-id source]}] wanted]
+      (when-not (contains? @videos image-id)
+        (attach! shape-id image-id source)))))
+
+(defn attach-to-selected!
+  "Console entry point: attaches `source` to the selected image shape."
+  [source]
+  (let [state (deref st/state)
+        shape (some->> (first (dsh/lookup-selected state))
+                       (dsh/lookup-shape state))]
+    (if-let [image-id (some-> shape image-fill-id)]
+      (do (attach! (:id shape) image-id source) true)
+      (do (log/error :hint "Select a shape with an image fill first"
+                     :shape-id (:id shape)
+                     :shape-type (:type shape))
+          false))))
+
+;; Reachable from the browser console while there is no UI for this.
+(unchecked-set js/globalThis "penpotAttachVideo" attach-to-selected!)
+(unchecked-set js/globalThis "penpotDetachVideos" detach-all!)

frontend/src/app/render_wasm/api/webgl.cljs

@@ -23,6 +23,24 @@
           (when current-ctx
             (.-GLctx ^js current-ctx)))))))
 
+(defn register-texture!
+  "Registers a WebGL texture with Emscripten's GL object system and returns its ID,
+   which is what WASM needs to wrap the texture as a Skia image."
+  [texture]
+  (let [gl-obj (unchecked-get wasm/internal-module "GL")
+        textures (.-textures ^js gl-obj)
+        new-id (.getNewId ^js gl-obj textures)]
+    (aset textures new-id texture)
+    new-id))
+
+(defn upload-texture-source!
+  "Uploads `source` — an HTMLImageElement, ImageBitmap or HTMLVideoElement — into
+   an existing texture."
+  [gl texture source]
+  (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) texture)
+  (.texImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 (.-RGBA ^js gl) (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) source)
+  (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil))
+
 (defn create-webgl-texture-from-image
   "Creates a WebGL texture from an HTMLImageElement or ImageBitmap and returns the texture object"
   [gl image-element]
@@ -32,8 +50,8 @@
     (.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_WRAP_T ^js gl) (.-CLAMP_TO_EDGE ^js gl))
     (.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_MIN_FILTER ^js gl) (.-LINEAR ^js gl))
     (.texParameteri ^js gl (.-TEXTURE_2D ^js gl) (.-TEXTURE_MAG_FILTER ^js gl) (.-LINEAR ^js gl))
-    (.texImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 (.-RGBA ^js gl) (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) image-element)
     (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil)
+    (upload-texture-source! gl texture image-element)
     texture))
 
 ;; FIXME: temporary function until we are able to keep the same <canvas> across pages.

frontend/src/app/render_wasm/shape.cljs

@@ -14,6 +14,7 @@
    [app.common.types.shape.layout :as ctl]
    [app.main.refs :as refs]
    [app.render-wasm.api :as api]
+   [app.render-wasm.api.video :as video]
    [app.render-wasm.svg-filters :as svg-filters]
    [beicon.v2.core :as rx]
    [cljs.core :as c]
@@ -313,6 +314,9 @@
           (ctl/flex-layout? shape)
           (api/set-flex-layout shape))
 
+        :video
+        (video/sync-shape! shape)
+
         ;; Property not in WASM
         nil))))
 

frontend/translations/en.po

@@ -8895,6 +8895,30 @@ msgstr "Uppercase"
 msgid "workspace.options.use-play-button"
 msgstr "Use the play button at the header to run the prototype view."
 
+#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:66
+msgid "workspace.options.video"
+msgstr "Video"
+
+#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:73
+msgid "workspace.options.video.pause"
+msgstr "Pause video"
+
+#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:80
+msgid "workspace.options.video.placeholder"
+msgstr "clip.mp4"
+
+#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:74
+msgid "workspace.options.video.play"
+msgstr "Play video"
+
+#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:86
+msgid "workspace.options.video.remove"
+msgstr "Remove video"
+
+#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:82
+msgid "workspace.options.video.source"
+msgstr "Video source"
+
 #: src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:538, src/app/main/ui/workspace/sidebar/options/menus/measures.cljs:560
 msgid "workspace.options.width"
 msgstr "Width"

render-wasm/src/render/gpu_state.rs

@@ -55,6 +55,14 @@ impl GpuState {
         })
     }
 
+    /// Drops the texture bindings Skia believes are current. Required whenever
+    /// JS writes a texture behind Ganesh's back (`texImage2D` from the CLJS
+    /// side): Skia caches which texture is bound to each unit, so after an
+    /// external bind it would keep drawing with a stale one.
+    pub fn reset_texture_bindings(&mut self) {
+        self.context.reset_gl_texture_bindings();
+    }
+
     pub fn max_texture_size(&self) -> i32 {
         self.context
             .max_texture_size()

render-wasm/src/render/images.rs

@@ -398,6 +398,38 @@ impl ImageStore {
         Ok(())
     }
 
+    /// Rebinds an already-registered image to a different GL texture, or to the
+    /// same texture after its contents changed. Video frames go through here:
+    /// the CLJS side uploads each decoded frame into one texture it owns and
+    /// calls this, which re-wraps it as a fresh `skia::Image`. Mutating the
+    /// texture under a live `Image` is not enough — Skia keys its caches
+    /// (filters, mipmaps, tile contents) on the image's unique id, so it would
+    /// keep serving the frame it first saw.
+    pub fn rebind_gl_texture(
+        &mut self,
+        id: Uuid,
+        is_thumbnail: bool,
+        texture_id: u32,
+        width: i32,
+        height: i32,
+    ) -> Result<()> {
+        let Some(context) = self.context.as_mut() else {
+            return Err(crate::error::Error::CriticalError(
+                "Cannot rebind a GL texture without a GPU context".to_string(),
+            ));
+        };
+        let image = create_image_from_gl_texture(context, texture_id, width, height)?;
+
+        let key = (id, is_thumbnail);
+        if let Some(previous) = self.images.remove(&key) {
+            self.total_bytes -= previous.bytes;
+        }
+        let bytes = (width as usize) * (height as usize) * 4;
+        self.insert_entry(key, StoredImage::Gpu(image), bytes);
+
+        Ok(())
+    }
+
     pub fn contains(&self, id: &Uuid, is_thumbnail: bool) -> bool {
         self.images.contains_key(&(*id, is_thumbnail))
     }

render-wasm/src/wasm/fills/image.rs

@@ -1,4 +1,5 @@
 use crate::error::{Error, Result};
+use crate::get_gpu_state;
 use crate::get_resources;
 use crate::mem;
 use crate::shapes::Fill;
@@ -183,6 +184,56 @@ pub extern "C" fn store_image_url(a: u32, b: u32, c: u32, d: u32) -> Result<()>
     Ok(())
 }
 
+/// A GL texture handed over from JS, as written by `store-image-texture` /
+/// `upload-video-frame!` on the CLJS side.
+struct TextureUpload {
+    ids: ShapeImageIds,
+    is_thumbnail: bool,
+    texture_id: u32,
+    width: i32,
+    height: i32,
+}
+
+const TEXTURE_UPLOAD_SIZE: usize = 48; // header + texture id + width + height
+
+/// Reads a `TextureUpload` out of the shared buffer. The buffer is freed by the
+/// caller on success and here on failure, so a malformed payload cannot leak it.
+fn read_texture_upload() -> Result<TextureUpload> {
+    let bytes = mem::bytes();
+
+    if bytes.len() < TEXTURE_UPLOAD_SIZE {
+        // FIXME: Review if this should be an critical or a recoverable error.
+        eprintln!("read_texture_upload: insufficient data");
+        mem::free_bytes()?;
+        return Err(Error::RecoverableError(
+            "read_texture_upload: insufficient data".to_string(),
+        ));
+    }
+
+    let ids = ShapeImageIds::try_from(&bytes[0..IMAGE_IDS_SIZE])
+        .map_err(|_| Error::CriticalError("Invalid image ids".to_string()))?;
+
+    // FIXME: read bytes in a safe way
+    let read_u32 = |range: std::ops::Range<usize>, what: &str| -> Result<u32> {
+        Ok(u32::from_le_bytes((&bytes[range]).try_into().map_err(
+            |_| Error::CriticalError(format!("Invalid bytes for {}", what)),
+        )?))
+    };
+
+    let is_thumbnail = read_u32(IMAGE_IDS_SIZE..IMAGE_HEADER_SIZE, "is_thumbnail flag")? != 0;
+    let texture_id = read_u32(36..40, "texture id")?;
+    let width = read_u32(40..44, "width")? as i32;
+    let height = read_u32(44..48, "height")? as i32;
+
+    Ok(TextureUpload {
+        ids,
+        is_thumbnail,
+        texture_id,
+        width,
+        height,
+    })
+}
+
 /// Stores an image from an existing WebGL texture, avoiding re-decoding
 /// Expected memory layout:
 /// - bytes 0-15: shape UUID
@@ -194,66 +245,52 @@ pub extern "C" fn store_image_url(a: u32, b: u32, c: u32, d: u32) -> Result<()>
 #[no_mangle]
 #[wasm_error]
 pub extern "C" fn store_image_from_texture() -> Result<()> {
-    let bytes = mem::bytes();
+    let upload = read_texture_upload()?;
-
-    // FIXME: where does this 48 come from?
-    if bytes.len() < 48 {
-        // FIXME: Review if this should be an critical or a recoverable error.
-        eprintln!("store_image_from_texture: insufficient data");
-        mem::free_bytes()?;
-        return Err(Error::RecoverableError(
-            "store_image_from_texture: insufficient data".to_string(),
-        ));
-    }
-
-    let ids = ShapeImageIds::try_from(&bytes[0..IMAGE_IDS_SIZE])
-        .map_err(|_| Error::CriticalError("Invalid image ids".to_string()))?;
-
-    // FIXME: read bytes in a safe way
-
-    // Read is_thumbnail flag (4 bytes as u32)
-    let is_thumbnail_bytes = &bytes[IMAGE_IDS_SIZE..IMAGE_HEADER_SIZE];
-    let is_thumbnail_value =
-        u32::from_le_bytes(is_thumbnail_bytes.try_into().map_err(|_| {
-            Error::CriticalError("Invalid bytes for is_thumbnail flag".to_string())
-        })?);
-    let is_thumbnail = is_thumbnail_value != 0;
-
-    // Read GL texture ID (4 bytes as u32)
-    let texture_id_bytes = &bytes[36..40];
-    let texture_id = u32::from_le_bytes(
-        texture_id_bytes
-            .try_into()
-            .map_err(|_| Error::CriticalError("Invalid bytes for texture id".to_string()))?,
-    );
-
-    // Read width and height (8 bytes as two i32s)
-    let width_bytes = &bytes[40..44];
-    let width = i32::from_le_bytes(
-        width_bytes
-            .try_into()
-            .map_err(|_| Error::CriticalError("Invalid bytes for width".to_string()))?,
-    );
-
-    let height_bytes = &bytes[44..48];
-    let height = i32::from_le_bytes(
-        height_bytes
-            .try_into()
-            .map_err(|_| Error::CriticalError("Invalid bytes for height".to_string()))?,
-    );
 
     with_state!(state, {
         if let Err(msg) = get_resources().images.add_image_from_gl_texture(
-            ids.image_id,
+            upload.ids.image_id,
-            is_thumbnail,
+            upload.is_thumbnail,
-            texture_id,
+            upload.texture_id,
-            width,
+            upload.width,
-            height,
+            upload.height,
         ) {
             // FIXME: Review if we should return a RecoverableError
             eprintln!("store_image_from_texture error: {}", msg);
         }
-        touch_shapes_with_image(state, ids.image_id);
+        touch_shapes_with_image(state, upload.ids.image_id);
+    });
+
+    mem::free_bytes()?;
+    Ok(())
+}
+
+/// Rebinds an already-stored image to the current contents of a GL texture and
+/// invalidates the tiles of every shape that paints it. Same memory layout as
+/// `store_image_from_texture`.
+///
+/// This is the per-frame entry point for video: the CLJS side owns one texture
+/// per playing video, uploads the decoded frame into it and calls this.
+#[no_mangle]
+#[wasm_error]
+pub extern "C" fn update_image_from_texture() -> Result<()> {
+    let upload = read_texture_upload()?;
+
+    // The caller just bound and wrote the texture from JS, which Skia has no
+    // way to observe. Drop its cached bindings before it samples the texture.
+    get_gpu_state().reset_texture_bindings();
+
+    with_state!(state, {
+        if let Err(msg) = get_resources().images.rebind_gl_texture(
+            upload.ids.image_id,
+            upload.is_thumbnail,
+            upload.texture_id,
+            upload.width,
+            upload.height,
+        ) {
+            eprintln!("update_image_from_texture error: {}", msg);
+        }
+        touch_shapes_with_image(state, upload.ids.image_id);
     });
 
     mem::free_bytes()?;