diff --git a/frontend/src/app/main/data/workspace/media.cljs b/frontend/src/app/main/data/workspace/media.cljs index 4aac2d2e4e..f5536ae463 100644 --- a/frontend/src/app/main/data/workspace/media.cljs +++ b/frontend/src/app/main/data/workspace/media.cljs @@ -65,6 +65,16 @@ :hint "could not read the video dimensions")))) (set! (.-src element) url))))) +(defn- media-dimensions + "Stream of the dimensions an upload has to be stored with: the size the + client decoded for a video, and nothing at all for a still, which the + backend measures itself. Every upload path runs through this, so a video + cannot reach the backend without them." + [blob] + (if (media/video-type? (.-type blob)) + (rx/from (read-video-dimensions blob)) + (rx/of nil))) + (defn- optimize [input] (svgo/optimize input svgo/defaultOptions)) @@ -218,10 +228,8 @@ dimensions))))) (upload-blob [blob] - (if (media/video-type? (.-type blob)) - (->> (rx/from (read-video-dimensions blob)) - (rx/mapcat #(upload-blob* blob %))) - (upload-blob* blob nil))) + (->> (media-dimensions blob) + (rx/mapcat #(upload-blob* blob %)))) (extract-content [blob] (let [name (or name (.-name blob))] @@ -341,7 +349,9 @@ (dmm/notify-start-loading) (->> (rx/of file) (rx/map dmm/validate-file) - (rx/map prepare) + (rx/mapcat (fn [content] + (->> (media-dimensions content) + (rx/map #(merge (prepare content) %))))) (rx/mapcat #(rp/cmd! :upload-file-media-object %)) (rx/tap on-upload-success) (rx/catch handle-media-error)))))) diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs b/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs index ee6d1a98a9..eed4f5bbf5 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs @@ -5,16 +5,11 @@ ;; Copyright (c) KALEIDOS SUBSIDIARY SL (ns app.main.ui.workspace.sidebar.options.menus.video - "Plays a video into a shape's image fill. + "Proof of concept: plays a video into a shape's image fill. - The source is an uploaded video asset, or a path the browser can already + Nothing uploads video, so the source is a path the browser can already reach — a file under `frontend/resources/public/images/` or a full URL. Only - the render-wasm renderer paints it; elsewhere the image fill still shows. - - A video is stamped when the frame is composed, which is a flat draw: it - cannot carry opacity, a blend mode, a blur, a shadow or a stroke. A shape - with one of those does not play, and this menu says which one is in the - way." + the render-wasm renderer paints it; elsewhere the image fill still shows." (:require-macros [app.main.style :as stl]) (:require [app.main.data.workspace.shapes :as dwsh] @@ -38,25 +33,14 @@ (dissoc shape :video) (assoc shape :video source))))) -(def ^:private reason-labels - {:opacity "workspace.options.video.blocked.opacity" - :blend-mode "workspace.options.video.blocked.blend-mode" - :blur "workspace.options.video.blocked.blur" - :shadow "workspace.options.video.blocked.shadow" - :stroke "workspace.options.video.blocked.stroke" - :masked "workspace.options.video.blocked.masked"}) - (mf/defc video-menu* - [{:keys [ids image-id source is-asset blocked-reason]}] + [{:keys [ids image-id source is-asset]}] (let [;; `video/playing?` reads the element, which is outside app state, so ;; the button tracks it locally. playing* (mf/use-state #(video/playing? image-id)) playing (deref playing*) - blocked-label (get reason-labels blocked-reason) - - has-source (and (not (str/blank? source)) - (nil? blocked-label)) + has-source (not (str/blank? source)) on-change (mf/use-fn @@ -91,11 +75,6 @@ :selected playing :tooltip-placement "top-left" :icon i/play}])]] - ;; Playback was refused: say which property is in the way, so a video that - ;; stops after a shadow is added does not look broken. - (when (some? blocked-label) - [:div {:class (stl/css :blocked)} - (tr blocked-label)]) ;; An uploaded video has nothing to type: its source is the asset itself, ;; and removing it means deleting the shape. (when-not is-asset diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss b/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss index 495a53d235..d382a84c92 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss @@ -4,7 +4,6 @@ // // Copyright (c) KALEIDOS SUBSIDIARY SL -@use "ds/typography.scss" as t; @use "../../../sidebar/common/sidebar.scss" as sidebar; .element-set { @@ -22,10 +21,3 @@ flex-grow: 1; min-width: 0; } - -.blocked { - @include t.use-typography("body-small"); - - color: var(--input-foreground-color-disabled); - margin-bottom: var(--sp-s); -} diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs b/frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs index 8729091578..13d319b4a3 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs @@ -95,12 +95,7 @@ ;; An uploaded video asset needs no source field, only playback. video-asset? - (and render-wasm? (some? (video/video-fill shape))) - - ;; The composited path is a flat draw, so a shape carrying opacity, a - ;; blend mode, a blur, a shadow or a stroke does not play at all. - video-blocked-reason - (when (some? video-image-id) (video/refuses-to-play shape))] + (and render-wasm? (some? (video/video-fill shape)))] [:* [:> layer-menu* {:ids ids @@ -151,8 +146,7 @@ [:> video-menu* {:ids ids :image-id video-image-id :source (video/shape-source shape) - :is-asset video-asset? - :blocked-reason video-blocked-reason}]) + :is-asset video-asset?}]) [:> stroke-menu* {:ids ids :type type diff --git a/frontend/src/app/render_wasm/api/video.cljs b/frontend/src/app/render_wasm/api/video.cljs index d0d85368ae..031cdea496 100644 --- a/frontend/src/app/render_wasm/api/video.cljs +++ b/frontend/src/app/render_wasm/api/video.cljs @@ -12,19 +12,15 @@ the current frame is uploaded into that texture and WASM re-wraps it, so every shape whose image fill points at `image-id` paints it. - With `video-overlay-wasm/v1` on, an eligible shape has its frames stamped - when the frame is composed rather than rastered into the tiles, so playback - costs one textured quad instead of a tile re-raster. A shape carrying - opacity, a blend mode, a blur, a shadow or a stroke cannot be stamped that - way and does not play at all; `ineligible-reason` says which one is in the - way so the sidebar can explain it. + The source is stored on the shape as `:video`, so it persists with the file + and comes back on load. Drop a file into `frontend/resources/public/images/`, + where it is served like any other static asset, and name it from the Video + section of the design sidebar. `penpotAttachVideo(\"clip.mp4\")` does the same + from the console for the selected shape, without persisting it. - The source is either an uploaded video asset or the shape's `:video` - attribute — a file under `frontend/resources/public/images/` or a full URL. - `penpotAttachVideo(\"clip.mp4\")` does the same from the console for the - selected shape, without persisting it." + There is no backend support: nothing uploads video, so the source is a path + the browser can reach on its own." (:require - [app.common.geom.rect :as grc] [app.common.logging :as log] [app.common.media :as cm] [app.common.render-wasm.helpers :as h] @@ -32,7 +28,6 @@ [app.common.render-wasm.mem.heap32 :as mem.h32] [app.common.render-wasm.wasm :as wasm] [app.common.types.fills :as types.fills] - [app.common.uuid :as uuid] [app.config :as cf] [app.main.data.helpers :as dsh] [app.main.store :as st] @@ -45,26 +40,10 @@ ;; Videos live next to the other static assets, so a bare file name is enough. (def ^:private assets-path "/images/") -;; image-id -> {:element :source :texture :texture-id :shape-id :image-id -;; :tex-width :tex-height :pending? :frame-handle :last-time} +;; image-id -> {:element :source :texture :texture-id :shape-id :image-id :last-time} ;; The texture keys arrive later, once the video has decoded its first frame. (defonce ^:private videos (atom {})) -(defn- update-entry! - "Applies `f` to an attachment, and does nothing when it is already detached — - frame callbacks and promises resolve after a detach." - [image-id f & args] - (swap! videos (fn [videos] - (if (contains? videos image-id) - (apply update videos image-id f args) - videos)))) - -(defn- frame-callbacks? - "True when the element reports each presented frame on its own. Widely - available, but the rAF poll stays as the fallback." - [^js element] - (fn? (.-requestVideoFrameCallback element))) - ;; `api.cljs` owns the render loop and requires this namespace, so it installs ;; its requester here rather than being required back. (defonce ^:private render-requester (atom nil)) @@ -84,52 +63,6 @@ [] (exists? js/document)) -(def ineligible-reasons - "Codes returned by `_get_video_eligibility`, matching `VideoIneligible` in - `render-wasm/src/render/video.rs`. `0` means the video can be composited." - {1 :no-video-fill - 2 :opacity - 3 :blend-mode - 4 :blur - 5 :shadow - 6 :stroke - 7 :masked}) - -(defn- call-with-uuid! - [export id] - (let [buffer (uuid/get-u32 id)] - (h/call wasm/internal-module export - (aget buffer 0) - (aget buffer 1) - (aget buffer 2) - (aget buffer 3)))) - -(defn- register-image! - "Tells the renderer this image is backed by a playing video, so its frames are - stamped when the frame is composed instead of rastered into the tiles." - [image-id] - (when (wasm/live?) - (call-with-uuid! "_register_video_image" image-id))) - -(defn- unregister-image! - [image-id] - (when (wasm/live?) - (call-with-uuid! "_unregister_video_image" image-id))) - -(defn set-overlay-enabled! - "Threads `video-overlay-wasm/v1` to the renderer. With it off, video frames - keep going through the tiles." - [enabled] - (when (wasm/live?) - (h/call wasm/internal-module "_set_video_overlay_enabled" (boolean enabled)))) - -(defn ineligible-reason - "Why `shape-id` cannot have its video composited, or nil when it can. The - renderer owns the rule, so the sidebar and the render path cannot disagree." - [shape-id] - (when (wasm/live?) - (get ineligible-reasons (call-with-uuid! "_get_video_eligibility" shape-id)))) - (defn- send-frame! "Hands the texture to WASM, which re-wraps it as a Skia image and invalidates the tiles of every shape painting it. Same layout as `_store_image_from_texture`." @@ -145,51 +78,21 @@ (h/call wasm/internal-module "_update_image_from_texture"))) (defn- decoded-frame? - [{:keys [^js element texture-id pending? last-time]}] + [{:keys [^js element image-id texture-id]}] (and (some? texture-id) (>= (.-readyState element) HAVE-CURRENT-DATA) (pos? (.-videoWidth element)) - (if (frame-callbacks? element) - pending? - ;; rAF runs faster than most videos decode; without the frame callback - ;; skip the upload when playback has not advanced since the last one. - (not= (.-currentTime element) last-time)))) - -(defn visible-in-viewport? - "True when `selrect` meets `vbox`. An unknown viewport or unknown bounds — - the viewer, a page still loading — count as visible, so the video keeps - painting rather than going blank." - [vbox selrect] - (or (nil? vbox) - (nil? selrect) - (grc/overlaps-rects? vbox selrect))) - -(defn- on-screen? - "A video outside the viewport keeps playing — so it stays in step with the - others and resumes at the right moment — but stops paying for a texture - upload." - [shape-id] - (let [state (deref st/state)] - (visible-in-viewport? (get-in state [:workspace-local :vbox]) - (:selrect (dsh/lookup-shape state shape-id))))) + ;; rAF runs faster than most videos decode; skip the upload when + ;; playback has not advanced since the last one. + (not= (.-currentTime element) (get-in @videos [image-id :last-time])))) (defn- upload-frame! - [gl {:keys [^js element texture image-id shape-id tex-width tex-height] :as entry}] + [gl {:keys [^js element texture image-id] :as entry}] (when (decoded-frame? entry) - (if-not (on-screen? shape-id) - (update-entry! image-id assoc :pending? false :last-time (.-currentTime element)) - (let [width (.-videoWidth element) - height (.-videoHeight element)] - ;; An adaptive stream can switch resolution mid-playback, and the - ;; texture storage has to be redefined when it does. - (if (and (= width tex-width) (= height tex-height)) - (webgl/update-texture-source! gl texture element) - (do - (webgl/upload-texture-source! gl texture element) - (update-entry! image-id assoc :tex-width width :tex-height height))) - (update-entry! image-id assoc :pending? false :last-time (.-currentTime element)) - (send-frame! entry) - true)))) + (swap! videos assoc-in [image-id :last-time] (.-currentTime element)) + (webgl/upload-texture-source! gl texture element) + (send-frame! entry) + true)) (defn tick! "Uploads a frame for every attached video that advanced. Called from the @@ -203,49 +106,21 @@ (vals @videos))))) (defn active? - "True while some attached video needs the renderer to keep scheduling frames - instead of settling. A video reporting its own frames asks for a render when - it has one, so only the polling fallback keeps the loop awake." + "True while some attached video is playing, so the renderer keeps scheduling + frames instead of settling." [] - (boolean (some (fn [{:keys [^js element]}] - (and (not (.-paused element)) - (not (frame-callbacks? element)))) + (boolean (some (fn [{:keys [^js element]}] (not (.-paused element))) (vals @videos)))) -(defn- request-frame-callback! - "Asks the element to report its next presented frame, and re-arms itself from - the callback so the chain lasts as long as the attachment does." - [image-id ^js element] - (when (and (frame-callbacks? element) - (contains? @videos image-id) - (nil? (get-in @videos [image-id :frame-handle]))) - (let [handle (.requestVideoFrameCallback - element - (fn [_now _metadata] - (when (contains? @videos image-id) - (update-entry! image-id assoc :pending? true :frame-handle nil) - (request-render!) - (request-frame-callback! image-id element))))] - (update-entry! image-id assoc :frame-handle handle)))) - -(defn- cancel-frame-callback! - [^js element handle] - (when (and (some? handle) (fn? (.-cancelVideoFrameCallback element))) - (.cancelVideoFrameCallback element handle))) - (defn detach! "Stops a video. The last frame stays on screen: the texture is deliberately kept alive because WASM holds a Skia image borrowing it." [image-id] - (when-let [{:keys [^js element frame-handle]} (get @videos image-id)] - ;; Dropped from the registry first: the frame callback re-arms itself and - ;; checks the registry to know when to stop. - (swap! videos dissoc image-id) - (unregister-image! image-id) - (cancel-frame-callback! element frame-handle) + (when-let [{:keys [^js element]} (get @videos image-id)] (.pause element) (.removeAttribute element "src") - (.load element))) + (.load element) + (swap! videos dissoc image-id))) (defn detach-all! [] @@ -256,16 +131,9 @@ [image-id ^js element] (if-let [gl (webgl/get-webgl-context)] (let [texture (webgl/create-webgl-texture-from-image gl element)] - (register-image! image-id) - (update-entry! image-id merge - {:texture texture - :texture-id (webgl/register-texture! texture) - ;; `create-webgl-texture-from-image` allocated the storage - ;; at this size; later frames only overwrite its pixels. - :tex-width (.-videoWidth element) - :tex-height (.-videoHeight element) - :pending? true}) - (request-frame-callback! image-id element) + (swap! videos update image-id merge + {:texture texture + :texture-id (webgl/register-texture! texture)}) (-> (.play element) ;; Nothing else would ask for a frame: `tick!` only runs inside a ;; render, and the loop only keeps itself alive once it has started. @@ -294,10 +162,6 @@ :source source :shape-id shape-id :image-id image-id - :tex-width nil - :tex-height nil - :pending? false - :frame-handle nil :last-time nil}) ;; Same-origin assets need no CORS, but a remote video served without the ;; headers would taint the canvas and make `texImage2D` throw. @@ -357,32 +221,16 @@ [image-id] (when-let [^js element (get-in @videos [image-id :element])] (if (.-paused element) - (do - (request-frame-callback! image-id element) - (-> (.play element) - (.then (fn [_] (request-render!))) - (.catch (fn [cause] - (log/error :hint "Could not play video" :cause cause))))) + (-> (.play element) + (.then (fn [_] (request-render!))) + (.catch (fn [cause] + (log/error :hint "Could not play video" :cause cause)))) (.pause element)) (not (.-paused element)))) -(defn- overlay-enabled? - [] - (contains? (:features (deref st/state)) "video-overlay-wasm/v1")) - -(defn refuses-to-play - "Why `shape` will not play its video, or nil when it will. Only the composited - path refuses: with the flag off, frames still go through the tiles and any - shape can carry them." - [shape] - (when (overlay-enabled?) - (ineligible-reason (:id shape)))) - (defn sync-shape! "Reconciles one shape with its `:video` attribute. Called whenever the attribute changes, so attaching is a plain shape edit and undo comes free. - Also called when the shape's effects change, so adding a drop shadow to a - playing video stops it there and then. Returns nil: `set-wasm-attr!` treats what it gets back as pending image loads, and `detach!` would otherwise hand it the video registry." @@ -393,11 +241,6 @@ (str/blank? source) (detach! image-id) - ;; A flat stamp cannot reproduce opacity, blending, blur, a shadow or a - ;; stroke, so a shape carrying one shows its poster frame instead. - (some? (refuses-to-play shape)) - (detach! image-id) - (not= source (get-in @videos [image-id :source])) (attach! (:id shape) image-id source)))) nil) @@ -407,9 +250,6 @@ for and stops the ones no shape wants any more. Run on page load, so a video survives a reload; an unchanged attachment keeps playing." [objects] - ;; Cheap, and this is where the renderer learns the flag: it runs on every - ;; page load, and the flag can change between them. - (set-overlay-enabled! (overlay-enabled?)) (let [wanted (into {} (keep (fn [[shape-id shape]] (let [source (shape-source shape)] @@ -422,8 +262,7 @@ (detach! image-id))) (doseq [[image-id {:keys [shape-id source]}] wanted] (when-not (contains? @videos image-id) - (when-not (and (overlay-enabled?) (some? (ineligible-reason shape-id))) - (attach! shape-id image-id source)))))) + (attach! shape-id image-id source))))) (defn attach-to-selected! "Console entry point: attaches `source` to the selected image shape." diff --git a/frontend/src/app/render_wasm/api/webgl.cljs b/frontend/src/app/render_wasm/api/webgl.cljs index 4ca2a9c8d3..fa2266a341 100644 --- a/frontend/src/app/render_wasm/api/webgl.cljs +++ b/frontend/src/app/render_wasm/api/webgl.cljs @@ -34,25 +34,13 @@ new-id)) (defn upload-texture-source! - "Allocates the texture storage and fills it with `source` — an - HTMLImageElement, ImageBitmap or HTMLVideoElement. Redefines the texture - level, so it is the call to make once per texture (or when the source - changes size), not the one to repeat per frame." + "Uploads `source` — an HTMLImageElement, ImageBitmap or HTMLVideoElement — into + an existing texture." [gl texture source] (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) texture) (.texImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 (.-RGBA ^js gl) (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) source) (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil)) -(defn update-texture-source! - "Overwrites the pixels of an already-allocated texture with `source`, which - must have the same size the texture was allocated with. Used for video - frames: unlike `upload-texture-source!` it does not redefine the level, so - the storage is not reallocated on every frame." - [gl texture source] - (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) texture) - (.texSubImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 0 0 (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) source) - (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil)) - (defn create-webgl-texture-from-image "Creates a WebGL texture from an HTMLImageElement or ImageBitmap and returns the texture object" [gl image-element] diff --git a/frontend/src/app/render_wasm/shape.cljs b/frontend/src/app/render_wasm/shape.cljs index ce82eced85..fe581f2e87 100644 --- a/frontend/src/app/render_wasm/shape.cljs +++ b/frontend/src/app/render_wasm/shape.cljs @@ -166,8 +166,12 @@ :transform (api/set-shape-transform v) + ;; An uploaded video carries its source in the fill, so replacing the + ;; fill is what starts or stops playback. :fills - (api/set-shape-fills id v false) + (let [pending (into [] (api/set-shape-fills id v false))] + (video/sync-shape! shape) + pending) :strokes (let [pending (into [] (api/set-shape-strokes id v false))] diff --git a/render-wasm/src/render.rs b/render-wasm/src/render.rs index 092c313ec7..38fc9468eb 100644 --- a/render-wasm/src/render.rs +++ b/render-wasm/src/render.rs @@ -43,7 +43,7 @@ use crate::{get_gpu_state, get_resources, performance}; pub use fonts::*; pub use images::*; -pub use video::{shape_overlay_blockers, video_overlay_eligibility, VideoIneligible, VideoRegistry}; +pub use video::VideoRegistry; pub(crate) use resources::RenderResources; type ClipStack = Vec<(Rect, Option, Matrix)>; diff --git a/render-wasm/src/state/shapes_pool.rs b/render-wasm/src/state/shapes_pool.rs index 3a742141e9..a494203b16 100644 --- a/render-wasm/src/state/shapes_pool.rs +++ b/render-wasm/src/state/shapes_pool.rs @@ -521,6 +521,8 @@ impl ShapesPoolImpl { modifier_uuids: Vec::new(), structure: HashMap::default(), scale_content: HashMap::default(), + revision: 0, + image_index: None, } }