wip

frontend/src/app/main/data/workspace/media.cljs

@@ -65,6 +65,16 @@
                                       :hint "could not read the video dimensions"))))
        (set! (.-src element) url)))))
 
+(defn- media-dimensions
+  "Stream of the dimensions an upload has to be stored with: the size the
+   client decoded for a video, and nothing at all for a still, which the
+   backend measures itself. Every upload path runs through this, so a video
+   cannot reach the backend without them."
+  [blob]
+  (if (media/video-type? (.-type blob))
+    (rx/from (read-video-dimensions blob))
+    (rx/of nil)))
+
 (defn- optimize
   [input]
   (svgo/optimize input svgo/defaultOptions))
@@ -218,10 +228,8 @@
                                 dimensions)))))
 
           (upload-blob [blob]
-            (if (media/video-type? (.-type blob))
+            (->> (media-dimensions blob)
-              (->> (rx/from (read-video-dimensions blob))
+                 (rx/mapcat #(upload-blob* blob %))))
-                   (rx/mapcat #(upload-blob* blob %)))
-              (upload-blob* blob nil)))
 
           (extract-content [blob]
             (let [name (or name (.-name blob))]
@@ -341,7 +349,9 @@
         (dmm/notify-start-loading)
         (->> (rx/of file)
              (rx/map dmm/validate-file)
-             (rx/map prepare)
+             (rx/mapcat (fn [content]
+                          (->> (media-dimensions content)
+                               (rx/map #(merge (prepare content) %)))))
              (rx/mapcat #(rp/cmd! :upload-file-media-object %))
              (rx/tap on-upload-success)
              (rx/catch handle-media-error))))))

frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs

@@ -5,16 +5,11 @@
 ;; Copyright (c) KALEIDOS SUBSIDIARY SL
 
 (ns app.main.ui.workspace.sidebar.options.menus.video
-  "Plays a video into a shape's image fill.
+  "Proof of concept: plays a video into a shape's image fill.
 
-   The source is an uploaded video asset, or a path the browser can already
+   Nothing uploads video, so the source is a path the browser can already
    reach — a file under `frontend/resources/public/images/` or a full URL. Only
-   the render-wasm renderer paints it; elsewhere the image fill still shows.
+   the render-wasm renderer paints it; elsewhere the image fill still shows."
-
-   A video is stamped when the frame is composed, which is a flat draw: it
-   cannot carry opacity, a blend mode, a blur, a shadow or a stroke. A shape
-   with one of those does not play, and this menu says which one is in the
-   way."
   (:require-macros [app.main.style :as stl])
   (:require
    [app.main.data.workspace.shapes :as dwsh]
@@ -38,25 +33,14 @@
                               (dissoc shape :video)
                               (assoc shape :video source)))))
 
-(def ^:private reason-labels
-  {:opacity "workspace.options.video.blocked.opacity"
-   :blend-mode "workspace.options.video.blocked.blend-mode"
-   :blur "workspace.options.video.blocked.blur"
-   :shadow "workspace.options.video.blocked.shadow"
-   :stroke "workspace.options.video.blocked.stroke"
-   :masked "workspace.options.video.blocked.masked"})
-
 (mf/defc video-menu*
-  [{:keys [ids image-id source is-asset blocked-reason]}]
+  [{:keys [ids image-id source is-asset]}]
   (let [;; `video/playing?` reads the element, which is outside app state, so
         ;; the button tracks it locally.
         playing* (mf/use-state #(video/playing? image-id))
         playing  (deref playing*)
 
-        blocked-label (get reason-labels blocked-reason)
+        has-source (not (str/blank? source))
-
-        has-source (and (not (str/blank? source))
-                        (nil? blocked-label))
 
         on-change
         (mf/use-fn
@@ -91,11 +75,6 @@
                            :selected playing
                            :tooltip-placement "top-left"
                            :icon i/play}])]]
-     ;; Playback was refused: say which property is in the way, so a video that
-     ;; stops after a shadow is added does not look broken.
-     (when (some? blocked-label)
-       [:div {:class (stl/css :blocked)}
-        (tr blocked-label)])
      ;; An uploaded video has nothing to type: its source is the asset itself,
      ;; and removing it means deleting the shape.
      (when-not is-asset

frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss

@@ -4,7 +4,6 @@
 //
 // Copyright (c) KALEIDOS SUBSIDIARY SL
 
-@use "ds/typography.scss" as t;
 @use "../../../sidebar/common/sidebar.scss" as sidebar;
 
 .element-set {
@@ -22,10 +21,3 @@
   flex-grow: 1;
   min-width: 0;
 }
-
-.blocked {
-  @include t.use-typography("body-small");
-
-  color: var(--input-foreground-color-disabled);
-  margin-bottom: var(--sp-s);
-}

frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs

@@ -95,12 +95,7 @@
 
         ;; An uploaded video asset needs no source field, only playback.
         video-asset?
-        (and render-wasm? (some? (video/video-fill shape)))
+        (and render-wasm? (some? (video/video-fill shape)))]
-
-        ;; The composited path is a flat draw, so a shape carrying opacity, a
-        ;; blend mode, a blur, a shadow or a stroke does not play at all.
-        video-blocked-reason
-        (when (some? video-image-id) (video/refuses-to-play shape))]
 
     [:*
      [:> layer-menu* {:ids ids
@@ -151,8 +146,7 @@
        [:> video-menu* {:ids ids
                         :image-id video-image-id
                         :source (video/shape-source shape)
-                        :is-asset video-asset?
+                        :is-asset video-asset?}])
-                        :blocked-reason video-blocked-reason}])
 
      [:> stroke-menu* {:ids ids
                        :type type

frontend/src/app/render_wasm/api/video.cljs

@@ -12,19 +12,15 @@
    the current frame is uploaded into that texture and WASM re-wraps it, so
    every shape whose image fill points at `image-id` paints it.
 
-   With `video-overlay-wasm/v1` on, an eligible shape has its frames stamped
+   The source is stored on the shape as `:video`, so it persists with the file
-   when the frame is composed rather than rastered into the tiles, so playback
+   and comes back on load. Drop a file into `frontend/resources/public/images/`,
-   costs one textured quad instead of a tile re-raster. A shape carrying
+   where it is served like any other static asset, and name it from the Video
-   opacity, a blend mode, a blur, a shadow or a stroke cannot be stamped that
+   section of the design sidebar. `penpotAttachVideo(\"clip.mp4\")` does the same
-   way and does not play at all; `ineligible-reason` says which one is in the
+   from the console for the selected shape, without persisting it.
-   way so the sidebar can explain it.
 
-   The source is either an uploaded video asset or the shape's `:video`
+   There is no backend support: nothing uploads video, so the source is a path
-   attribute — a file under `frontend/resources/public/images/` or a full URL.
+   the browser can reach on its own."
-   `penpotAttachVideo(\"clip.mp4\")` does the same from the console for the
-   selected shape, without persisting it."
   (:require
-   [app.common.geom.rect :as grc]
    [app.common.logging :as log]
    [app.common.media :as cm]
    [app.common.render-wasm.helpers :as h]
@@ -32,7 +28,6 @@
    [app.common.render-wasm.mem.heap32 :as mem.h32]
    [app.common.render-wasm.wasm :as wasm]
    [app.common.types.fills :as types.fills]
-   [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.main.data.helpers :as dsh]
    [app.main.store :as st]
@@ -45,26 +40,10 @@
 ;; Videos live next to the other static assets, so a bare file name is enough.
 (def ^:private assets-path "/images/")
 
-;; image-id -> {:element :source :texture :texture-id :shape-id :image-id
+;; image-id -> {:element :source :texture :texture-id :shape-id :image-id :last-time}
-;;              :tex-width :tex-height :pending? :frame-handle :last-time}
 ;; The texture keys arrive later, once the video has decoded its first frame.
 (defonce ^:private videos (atom {}))
 
-(defn- update-entry!
-  "Applies `f` to an attachment, and does nothing when it is already detached —
-   frame callbacks and promises resolve after a detach."
-  [image-id f & args]
-  (swap! videos (fn [videos]
-                  (if (contains? videos image-id)
-                    (apply update videos image-id f args)
-                    videos))))
-
-(defn- frame-callbacks?
-  "True when the element reports each presented frame on its own. Widely
-   available, but the rAF poll stays as the fallback."
-  [^js element]
-  (fn? (.-requestVideoFrameCallback element)))
-
 ;; `api.cljs` owns the render loop and requires this namespace, so it installs
 ;; its requester here rather than being required back.
 (defonce ^:private render-requester (atom nil))
@@ -84,52 +63,6 @@
   []
   (exists? js/document))
 
-(def ineligible-reasons
-  "Codes returned by `_get_video_eligibility`, matching `VideoIneligible` in
-   `render-wasm/src/render/video.rs`. `0` means the video can be composited."
-  {1 :no-video-fill
-   2 :opacity
-   3 :blend-mode
-   4 :blur
-   5 :shadow
-   6 :stroke
-   7 :masked})
-
-(defn- call-with-uuid!
-  [export id]
-  (let [buffer (uuid/get-u32 id)]
-    (h/call wasm/internal-module export
-            (aget buffer 0)
-            (aget buffer 1)
-            (aget buffer 2)
-            (aget buffer 3))))
-
-(defn- register-image!
-  "Tells the renderer this image is backed by a playing video, so its frames are
-   stamped when the frame is composed instead of rastered into the tiles."
-  [image-id]
-  (when (wasm/live?)
-    (call-with-uuid! "_register_video_image" image-id)))
-
-(defn- unregister-image!
-  [image-id]
-  (when (wasm/live?)
-    (call-with-uuid! "_unregister_video_image" image-id)))
-
-(defn set-overlay-enabled!
-  "Threads `video-overlay-wasm/v1` to the renderer. With it off, video frames
-   keep going through the tiles."
-  [enabled]
-  (when (wasm/live?)
-    (h/call wasm/internal-module "_set_video_overlay_enabled" (boolean enabled))))
-
-(defn ineligible-reason
-  "Why `shape-id` cannot have its video composited, or nil when it can. The
-   renderer owns the rule, so the sidebar and the render path cannot disagree."
-  [shape-id]
-  (when (wasm/live?)
-    (get ineligible-reasons (call-with-uuid! "_get_video_eligibility" shape-id))))
-
 (defn- send-frame!
   "Hands the texture to WASM, which re-wraps it as a Skia image and invalidates
    the tiles of every shape painting it. Same layout as `_store_image_from_texture`."
@@ -145,51 +78,21 @@
     (h/call wasm/internal-module "_update_image_from_texture")))
 
 (defn- decoded-frame?
-  [{:keys [^js element texture-id pending? last-time]}]
+  [{:keys [^js element image-id texture-id]}]
   (and (some? texture-id)
        (>= (.-readyState element) HAVE-CURRENT-DATA)
        (pos? (.-videoWidth element))
-       (if (frame-callbacks? element)
+       ;; rAF runs faster than most videos decode; skip the upload when
-         pending?
+       ;; playback has not advanced since the last one.
-         ;; rAF runs faster than most videos decode; without the frame callback
+       (not= (.-currentTime element) (get-in @videos [image-id :last-time]))))
-         ;; skip the upload when playback has not advanced since the last one.
-         (not= (.-currentTime element) last-time))))
-
-(defn visible-in-viewport?
-  "True when `selrect` meets `vbox`. An unknown viewport or unknown bounds —
-   the viewer, a page still loading — count as visible, so the video keeps
-   painting rather than going blank."
-  [vbox selrect]
-  (or (nil? vbox)
-      (nil? selrect)
-      (grc/overlaps-rects? vbox selrect)))
-
-(defn- on-screen?
-  "A video outside the viewport keeps playing — so it stays in step with the
-   others and resumes at the right moment — but stops paying for a texture
-   upload."
-  [shape-id]
-  (let [state (deref st/state)]
-    (visible-in-viewport? (get-in state [:workspace-local :vbox])
-                          (:selrect (dsh/lookup-shape state shape-id)))))
 
 (defn- upload-frame!
-  [gl {:keys [^js element texture image-id shape-id tex-width tex-height] :as entry}]
+  [gl {:keys [^js element texture image-id] :as entry}]
   (when (decoded-frame? entry)
-    (if-not (on-screen? shape-id)
+    (swap! videos assoc-in [image-id :last-time] (.-currentTime element))
-      (update-entry! image-id assoc :pending? false :last-time (.-currentTime element))
+    (webgl/upload-texture-source! gl texture element)
-      (let [width  (.-videoWidth element)
+    (send-frame! entry)
-            height (.-videoHeight element)]
+    true))
-        ;; An adaptive stream can switch resolution mid-playback, and the
-        ;; texture storage has to be redefined when it does.
-        (if (and (= width tex-width) (= height tex-height))
-          (webgl/update-texture-source! gl texture element)
-          (do
-            (webgl/upload-texture-source! gl texture element)
-            (update-entry! image-id assoc :tex-width width :tex-height height)))
-        (update-entry! image-id assoc :pending? false :last-time (.-currentTime element))
-        (send-frame! entry)
-        true))))
 
 (defn tick!
   "Uploads a frame for every attached video that advanced. Called from the
@@ -203,49 +106,21 @@
               (vals @videos)))))
 
 (defn active?
-  "True while some attached video needs the renderer to keep scheduling frames
+  "True while some attached video is playing, so the renderer keeps scheduling
-   instead of settling. A video reporting its own frames asks for a render when
+   frames instead of settling."
-   it has one, so only the polling fallback keeps the loop awake."
   []
-  (boolean (some (fn [{:keys [^js element]}]
+  (boolean (some (fn [{:keys [^js element]}] (not (.-paused element)))
-                   (and (not (.-paused element))
-                        (not (frame-callbacks? element))))
                  (vals @videos))))
 
-(defn- request-frame-callback!
-  "Asks the element to report its next presented frame, and re-arms itself from
-   the callback so the chain lasts as long as the attachment does."
-  [image-id ^js element]
-  (when (and (frame-callbacks? element)
-             (contains? @videos image-id)
-             (nil? (get-in @videos [image-id :frame-handle])))
-    (let [handle (.requestVideoFrameCallback
-                  element
-                  (fn [_now _metadata]
-                    (when (contains? @videos image-id)
-                      (update-entry! image-id assoc :pending? true :frame-handle nil)
-                      (request-render!)
-                      (request-frame-callback! image-id element))))]
-      (update-entry! image-id assoc :frame-handle handle))))
-
-(defn- cancel-frame-callback!
-  [^js element handle]
-  (when (and (some? handle) (fn? (.-cancelVideoFrameCallback element)))
-    (.cancelVideoFrameCallback element handle)))
-
 (defn detach!
   "Stops a video. The last frame stays on screen: the texture is deliberately
    kept alive because WASM holds a Skia image borrowing it."
   [image-id]
-  (when-let [{:keys [^js element frame-handle]} (get @videos image-id)]
+  (when-let [{:keys [^js element]} (get @videos image-id)]
-    ;; Dropped from the registry first: the frame callback re-arms itself and
-    ;; checks the registry to know when to stop.
-    (swap! videos dissoc image-id)
-    (unregister-image! image-id)
-    (cancel-frame-callback! element frame-handle)
     (.pause element)
     (.removeAttribute element "src")
-    (.load element)))
+    (.load element)
+    (swap! videos dissoc image-id)))
 
 (defn detach-all!
   []
@@ -256,16 +131,9 @@
   [image-id ^js element]
   (if-let [gl (webgl/get-webgl-context)]
     (let [texture (webgl/create-webgl-texture-from-image gl element)]
-      (register-image! image-id)
+      (swap! videos update image-id merge
-      (update-entry! image-id merge
+             {:texture texture
-                     {:texture texture
+              :texture-id (webgl/register-texture! texture)})
-                      :texture-id (webgl/register-texture! texture)
-                      ;; `create-webgl-texture-from-image` allocated the storage
-                      ;; at this size; later frames only overwrite its pixels.
-                      :tex-width (.-videoWidth element)
-                      :tex-height (.-videoHeight element)
-                      :pending? true})
-      (request-frame-callback! image-id element)
       (-> (.play element)
           ;; Nothing else would ask for a frame: `tick!` only runs inside a
           ;; render, and the loop only keeps itself alive once it has started.
@@ -294,10 +162,6 @@
             :source source
             :shape-id shape-id
             :image-id image-id
-            :tex-width nil
-            :tex-height nil
-            :pending? false
-            :frame-handle nil
             :last-time nil})
     ;; Same-origin assets need no CORS, but a remote video served without the
     ;; headers would taint the canvas and make `texImage2D` throw.
@@ -357,32 +221,16 @@
   [image-id]
   (when-let [^js element (get-in @videos [image-id :element])]
     (if (.-paused element)
-      (do
+      (-> (.play element)
-        (request-frame-callback! image-id element)
+          (.then (fn [_] (request-render!)))
-        (-> (.play element)
+          (.catch (fn [cause]
-            (.then (fn [_] (request-render!)))
+                    (log/error :hint "Could not play video" :cause cause))))
-            (.catch (fn [cause]
-                      (log/error :hint "Could not play video" :cause cause)))))
       (.pause element))
     (not (.-paused element))))
 
-(defn- overlay-enabled?
-  []
-  (contains? (:features (deref st/state)) "video-overlay-wasm/v1"))
-
-(defn refuses-to-play
-  "Why `shape` will not play its video, or nil when it will. Only the composited
-   path refuses: with the flag off, frames still go through the tiles and any
-   shape can carry them."
-  [shape]
-  (when (overlay-enabled?)
-    (ineligible-reason (:id shape))))
-
 (defn sync-shape!
   "Reconciles one shape with its `:video` attribute. Called whenever the
    attribute changes, so attaching is a plain shape edit and undo comes free.
-   Also called when the shape's effects change, so adding a drop shadow to a
-   playing video stops it there and then.
 
    Returns nil: `set-wasm-attr!` treats what it gets back as pending image
    loads, and `detach!` would otherwise hand it the video registry."
@@ -393,11 +241,6 @@
         (str/blank? source)
         (detach! image-id)
 
-        ;; A flat stamp cannot reproduce opacity, blending, blur, a shadow or a
-        ;; stroke, so a shape carrying one shows its poster frame instead.
-        (some? (refuses-to-play shape))
-        (detach! image-id)
-
         (not= source (get-in @videos [image-id :source]))
         (attach! (:id shape) image-id source))))
   nil)
@@ -407,9 +250,6 @@
    for and stops the ones no shape wants any more. Run on page load, so a video
    survives a reload; an unchanged attachment keeps playing."
   [objects]
-  ;; Cheap, and this is where the renderer learns the flag: it runs on every
-  ;; page load, and the flag can change between them.
-  (set-overlay-enabled! (overlay-enabled?))
   (let [wanted (into {}
                      (keep (fn [[shape-id shape]]
                              (let [source (shape-source shape)]
@@ -422,8 +262,7 @@
         (detach! image-id)))
     (doseq [[image-id {:keys [shape-id source]}] wanted]
       (when-not (contains? @videos image-id)
-        (when-not (and (overlay-enabled?) (some? (ineligible-reason shape-id)))
+        (attach! shape-id image-id source)))))
-          (attach! shape-id image-id source))))))
 
 (defn attach-to-selected!
   "Console entry point: attaches `source` to the selected image shape."

frontend/src/app/render_wasm/api/webgl.cljs

@@ -34,25 +34,13 @@
     new-id))
 
 (defn upload-texture-source!
-  "Allocates the texture storage and fills it with `source` — an
+  "Uploads `source` — an HTMLImageElement, ImageBitmap or HTMLVideoElement — into
-   HTMLImageElement, ImageBitmap or HTMLVideoElement. Redefines the texture
+   an existing texture."
-   level, so it is the call to make once per texture (or when the source
-   changes size), not the one to repeat per frame."
   [gl texture source]
   (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) texture)
   (.texImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 (.-RGBA ^js gl) (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) source)
   (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil))
 
-(defn update-texture-source!
-  "Overwrites the pixels of an already-allocated texture with `source`, which
-   must have the same size the texture was allocated with. Used for video
-   frames: unlike `upload-texture-source!` it does not redefine the level, so
-   the storage is not reallocated on every frame."
-  [gl texture source]
-  (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) texture)
-  (.texSubImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 0 0 (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) source)
-  (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil))
-
 (defn create-webgl-texture-from-image
   "Creates a WebGL texture from an HTMLImageElement or ImageBitmap and returns the texture object"
   [gl image-element]

frontend/src/app/render_wasm/shape.cljs

@@ -166,8 +166,12 @@
         :transform
         (api/set-shape-transform v)
 
+        ;; An uploaded video carries its source in the fill, so replacing the
+        ;; fill is what starts or stops playback.
         :fills
-        (api/set-shape-fills id v false)
+        (let [pending (into [] (api/set-shape-fills id v false))]
+          (video/sync-shape! shape)
+          pending)
 
         :strokes
         (let [pending (into [] (api/set-shape-strokes id v false))]

render-wasm/src/render.rs

@@ -43,7 +43,7 @@ use crate::{get_gpu_state, get_resources, performance};
 
 pub use fonts::*;
 pub use images::*;
-pub use video::{shape_overlay_blockers, video_overlay_eligibility, VideoIneligible, VideoRegistry};
+pub use video::VideoRegistry;
 pub(crate) use resources::RenderResources;
 
 type ClipStack = Vec<(Rect, Option<Corners>, Matrix)>;

render-wasm/src/state/shapes_pool.rs

@@ -521,6 +521,8 @@ impl ShapesPoolImpl {
             modifier_uuids: Vec::new(),
             structure: HashMap::default(),
             scale_content: HashMap::default(),
+            revision: 0,
+            image_index: None,
         }
     }