diff --git a/common/src/app/common/features.cljc b/common/src/app/common/features.cljc index e1b12729e0..c7eacef81f 100644 --- a/common/src/app/common/features.cljc +++ b/common/src/app/common/features.cljc @@ -57,6 +57,7 @@ "text-editor/v2" "text-editor-wasm/v1" "render-wasm/v1" + "video-overlay-wasm/v1" "wasm-export/v1" "variants/v1"}) @@ -84,6 +85,7 @@ "text-editor-wasm/v1" "tokens/numeric-input" "render-wasm/v1" + "video-overlay-wasm/v1" "wasm-export/v1"}) ;; Features that are mainly backend only or there are a proper @@ -134,6 +136,7 @@ :feature-text-editor-v2-html-paste "text-editor/v2-html-paste" :feature-text-editor-wasm "text-editor-wasm/v1" :feature-render-wasm "render-wasm/v1" + :feature-video-overlay-wasm "video-overlay-wasm/v1" :feature-variants "variants/v1" :feature-token-input "tokens/numeric-input" nil)) diff --git a/common/src/app/common/flags.cljc b/common/src/app/common/flags.cljc index 7ef1c0c6df..2aafb004d5 100644 --- a/common/src/app/common/flags.cljc +++ b/common/src/app/common/flags.cljc @@ -214,6 +214,7 @@ :enable-inspect-styles :enable-feature-fdata-objects-map :enable-feature-render-wasm + :enable-feature-video-overlay-wasm :enable-token-import-from-library :enable-render-switch :enable-render-wasm-info diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs b/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs index eed4f5bbf5..ee6d1a98a9 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs @@ -5,11 +5,16 @@ ;; Copyright (c) KALEIDOS SUBSIDIARY SL (ns app.main.ui.workspace.sidebar.options.menus.video - "Proof of concept: plays a video into a shape's image fill. + "Plays a video into a shape's image fill. - Nothing uploads video, so the source is a path the browser can already + The source is an uploaded video asset, or a path the browser can already reach — a file under `frontend/resources/public/images/` or a full URL. Only - the render-wasm renderer paints it; elsewhere the image fill still shows." + the render-wasm renderer paints it; elsewhere the image fill still shows. + + A video is stamped when the frame is composed, which is a flat draw: it + cannot carry opacity, a blend mode, a blur, a shadow or a stroke. A shape + with one of those does not play, and this menu says which one is in the + way." (:require-macros [app.main.style :as stl]) (:require [app.main.data.workspace.shapes :as dwsh] @@ -33,14 +38,25 @@ (dissoc shape :video) (assoc shape :video source))))) +(def ^:private reason-labels + {:opacity "workspace.options.video.blocked.opacity" + :blend-mode "workspace.options.video.blocked.blend-mode" + :blur "workspace.options.video.blocked.blur" + :shadow "workspace.options.video.blocked.shadow" + :stroke "workspace.options.video.blocked.stroke" + :masked "workspace.options.video.blocked.masked"}) + (mf/defc video-menu* - [{:keys [ids image-id source is-asset]}] + [{:keys [ids image-id source is-asset blocked-reason]}] (let [;; `video/playing?` reads the element, which is outside app state, so ;; the button tracks it locally. playing* (mf/use-state #(video/playing? image-id)) playing (deref playing*) - has-source (not (str/blank? source)) + blocked-label (get reason-labels blocked-reason) + + has-source (and (not (str/blank? source)) + (nil? blocked-label)) on-change (mf/use-fn @@ -75,6 +91,11 @@ :selected playing :tooltip-placement "top-left" :icon i/play}])]] + ;; Playback was refused: say which property is in the way, so a video that + ;; stops after a shadow is added does not look broken. + (when (some? blocked-label) + [:div {:class (stl/css :blocked)} + (tr blocked-label)]) ;; An uploaded video has nothing to type: its source is the asset itself, ;; and removing it means deleting the shape. (when-not is-asset diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss b/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss index d382a84c92..495a53d235 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss @@ -4,6 +4,7 @@ // // Copyright (c) KALEIDOS SUBSIDIARY SL +@use "ds/typography.scss" as t; @use "../../../sidebar/common/sidebar.scss" as sidebar; .element-set { @@ -21,3 +22,10 @@ flex-grow: 1; min-width: 0; } + +.blocked { + @include t.use-typography("body-small"); + + color: var(--input-foreground-color-disabled); + margin-bottom: var(--sp-s); +} diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs b/frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs index 13d319b4a3..8729091578 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs @@ -95,7 +95,12 @@ ;; An uploaded video asset needs no source field, only playback. video-asset? - (and render-wasm? (some? (video/video-fill shape)))] + (and render-wasm? (some? (video/video-fill shape))) + + ;; The composited path is a flat draw, so a shape carrying opacity, a + ;; blend mode, a blur, a shadow or a stroke does not play at all. + video-blocked-reason + (when (some? video-image-id) (video/refuses-to-play shape))] [:* [:> layer-menu* {:ids ids @@ -146,7 +151,8 @@ [:> video-menu* {:ids ids :image-id video-image-id :source (video/shape-source shape) - :is-asset video-asset?}]) + :is-asset video-asset? + :blocked-reason video-blocked-reason}]) [:> stroke-menu* {:ids ids :type type diff --git a/frontend/src/app/render_wasm/api/video.cljs b/frontend/src/app/render_wasm/api/video.cljs index 031cdea496..d0d85368ae 100644 --- a/frontend/src/app/render_wasm/api/video.cljs +++ b/frontend/src/app/render_wasm/api/video.cljs @@ -12,15 +12,19 @@ the current frame is uploaded into that texture and WASM re-wraps it, so every shape whose image fill points at `image-id` paints it. - The source is stored on the shape as `:video`, so it persists with the file - and comes back on load. Drop a file into `frontend/resources/public/images/`, - where it is served like any other static asset, and name it from the Video - section of the design sidebar. `penpotAttachVideo(\"clip.mp4\")` does the same - from the console for the selected shape, without persisting it. + With `video-overlay-wasm/v1` on, an eligible shape has its frames stamped + when the frame is composed rather than rastered into the tiles, so playback + costs one textured quad instead of a tile re-raster. A shape carrying + opacity, a blend mode, a blur, a shadow or a stroke cannot be stamped that + way and does not play at all; `ineligible-reason` says which one is in the + way so the sidebar can explain it. - There is no backend support: nothing uploads video, so the source is a path - the browser can reach on its own." + The source is either an uploaded video asset or the shape's `:video` + attribute — a file under `frontend/resources/public/images/` or a full URL. + `penpotAttachVideo(\"clip.mp4\")` does the same from the console for the + selected shape, without persisting it." (:require + [app.common.geom.rect :as grc] [app.common.logging :as log] [app.common.media :as cm] [app.common.render-wasm.helpers :as h] @@ -28,6 +32,7 @@ [app.common.render-wasm.mem.heap32 :as mem.h32] [app.common.render-wasm.wasm :as wasm] [app.common.types.fills :as types.fills] + [app.common.uuid :as uuid] [app.config :as cf] [app.main.data.helpers :as dsh] [app.main.store :as st] @@ -40,10 +45,26 @@ ;; Videos live next to the other static assets, so a bare file name is enough. (def ^:private assets-path "/images/") -;; image-id -> {:element :source :texture :texture-id :shape-id :image-id :last-time} +;; image-id -> {:element :source :texture :texture-id :shape-id :image-id +;; :tex-width :tex-height :pending? :frame-handle :last-time} ;; The texture keys arrive later, once the video has decoded its first frame. (defonce ^:private videos (atom {})) +(defn- update-entry! + "Applies `f` to an attachment, and does nothing when it is already detached — + frame callbacks and promises resolve after a detach." + [image-id f & args] + (swap! videos (fn [videos] + (if (contains? videos image-id) + (apply update videos image-id f args) + videos)))) + +(defn- frame-callbacks? + "True when the element reports each presented frame on its own. Widely + available, but the rAF poll stays as the fallback." + [^js element] + (fn? (.-requestVideoFrameCallback element))) + ;; `api.cljs` owns the render loop and requires this namespace, so it installs ;; its requester here rather than being required back. (defonce ^:private render-requester (atom nil)) @@ -63,6 +84,52 @@ [] (exists? js/document)) +(def ineligible-reasons + "Codes returned by `_get_video_eligibility`, matching `VideoIneligible` in + `render-wasm/src/render/video.rs`. `0` means the video can be composited." + {1 :no-video-fill + 2 :opacity + 3 :blend-mode + 4 :blur + 5 :shadow + 6 :stroke + 7 :masked}) + +(defn- call-with-uuid! + [export id] + (let [buffer (uuid/get-u32 id)] + (h/call wasm/internal-module export + (aget buffer 0) + (aget buffer 1) + (aget buffer 2) + (aget buffer 3)))) + +(defn- register-image! + "Tells the renderer this image is backed by a playing video, so its frames are + stamped when the frame is composed instead of rastered into the tiles." + [image-id] + (when (wasm/live?) + (call-with-uuid! "_register_video_image" image-id))) + +(defn- unregister-image! + [image-id] + (when (wasm/live?) + (call-with-uuid! "_unregister_video_image" image-id))) + +(defn set-overlay-enabled! + "Threads `video-overlay-wasm/v1` to the renderer. With it off, video frames + keep going through the tiles." + [enabled] + (when (wasm/live?) + (h/call wasm/internal-module "_set_video_overlay_enabled" (boolean enabled)))) + +(defn ineligible-reason + "Why `shape-id` cannot have its video composited, or nil when it can. The + renderer owns the rule, so the sidebar and the render path cannot disagree." + [shape-id] + (when (wasm/live?) + (get ineligible-reasons (call-with-uuid! "_get_video_eligibility" shape-id)))) + (defn- send-frame! "Hands the texture to WASM, which re-wraps it as a Skia image and invalidates the tiles of every shape painting it. Same layout as `_store_image_from_texture`." @@ -78,21 +145,51 @@ (h/call wasm/internal-module "_update_image_from_texture"))) (defn- decoded-frame? - [{:keys [^js element image-id texture-id]}] + [{:keys [^js element texture-id pending? last-time]}] (and (some? texture-id) (>= (.-readyState element) HAVE-CURRENT-DATA) (pos? (.-videoWidth element)) - ;; rAF runs faster than most videos decode; skip the upload when - ;; playback has not advanced since the last one. - (not= (.-currentTime element) (get-in @videos [image-id :last-time])))) + (if (frame-callbacks? element) + pending? + ;; rAF runs faster than most videos decode; without the frame callback + ;; skip the upload when playback has not advanced since the last one. + (not= (.-currentTime element) last-time)))) + +(defn visible-in-viewport? + "True when `selrect` meets `vbox`. An unknown viewport or unknown bounds — + the viewer, a page still loading — count as visible, so the video keeps + painting rather than going blank." + [vbox selrect] + (or (nil? vbox) + (nil? selrect) + (grc/overlaps-rects? vbox selrect))) + +(defn- on-screen? + "A video outside the viewport keeps playing — so it stays in step with the + others and resumes at the right moment — but stops paying for a texture + upload." + [shape-id] + (let [state (deref st/state)] + (visible-in-viewport? (get-in state [:workspace-local :vbox]) + (:selrect (dsh/lookup-shape state shape-id))))) (defn- upload-frame! - [gl {:keys [^js element texture image-id] :as entry}] + [gl {:keys [^js element texture image-id shape-id tex-width tex-height] :as entry}] (when (decoded-frame? entry) - (swap! videos assoc-in [image-id :last-time] (.-currentTime element)) - (webgl/upload-texture-source! gl texture element) - (send-frame! entry) - true)) + (if-not (on-screen? shape-id) + (update-entry! image-id assoc :pending? false :last-time (.-currentTime element)) + (let [width (.-videoWidth element) + height (.-videoHeight element)] + ;; An adaptive stream can switch resolution mid-playback, and the + ;; texture storage has to be redefined when it does. + (if (and (= width tex-width) (= height tex-height)) + (webgl/update-texture-source! gl texture element) + (do + (webgl/upload-texture-source! gl texture element) + (update-entry! image-id assoc :tex-width width :tex-height height))) + (update-entry! image-id assoc :pending? false :last-time (.-currentTime element)) + (send-frame! entry) + true)))) (defn tick! "Uploads a frame for every attached video that advanced. Called from the @@ -106,21 +203,49 @@ (vals @videos))))) (defn active? - "True while some attached video is playing, so the renderer keeps scheduling - frames instead of settling." + "True while some attached video needs the renderer to keep scheduling frames + instead of settling. A video reporting its own frames asks for a render when + it has one, so only the polling fallback keeps the loop awake." [] - (boolean (some (fn [{:keys [^js element]}] (not (.-paused element))) + (boolean (some (fn [{:keys [^js element]}] + (and (not (.-paused element)) + (not (frame-callbacks? element)))) (vals @videos)))) +(defn- request-frame-callback! + "Asks the element to report its next presented frame, and re-arms itself from + the callback so the chain lasts as long as the attachment does." + [image-id ^js element] + (when (and (frame-callbacks? element) + (contains? @videos image-id) + (nil? (get-in @videos [image-id :frame-handle]))) + (let [handle (.requestVideoFrameCallback + element + (fn [_now _metadata] + (when (contains? @videos image-id) + (update-entry! image-id assoc :pending? true :frame-handle nil) + (request-render!) + (request-frame-callback! image-id element))))] + (update-entry! image-id assoc :frame-handle handle)))) + +(defn- cancel-frame-callback! + [^js element handle] + (when (and (some? handle) (fn? (.-cancelVideoFrameCallback element))) + (.cancelVideoFrameCallback element handle))) + (defn detach! "Stops a video. The last frame stays on screen: the texture is deliberately kept alive because WASM holds a Skia image borrowing it." [image-id] - (when-let [{:keys [^js element]} (get @videos image-id)] + (when-let [{:keys [^js element frame-handle]} (get @videos image-id)] + ;; Dropped from the registry first: the frame callback re-arms itself and + ;; checks the registry to know when to stop. + (swap! videos dissoc image-id) + (unregister-image! image-id) + (cancel-frame-callback! element frame-handle) (.pause element) (.removeAttribute element "src") - (.load element) - (swap! videos dissoc image-id))) + (.load element))) (defn detach-all! [] @@ -131,9 +256,16 @@ [image-id ^js element] (if-let [gl (webgl/get-webgl-context)] (let [texture (webgl/create-webgl-texture-from-image gl element)] - (swap! videos update image-id merge - {:texture texture - :texture-id (webgl/register-texture! texture)}) + (register-image! image-id) + (update-entry! image-id merge + {:texture texture + :texture-id (webgl/register-texture! texture) + ;; `create-webgl-texture-from-image` allocated the storage + ;; at this size; later frames only overwrite its pixels. + :tex-width (.-videoWidth element) + :tex-height (.-videoHeight element) + :pending? true}) + (request-frame-callback! image-id element) (-> (.play element) ;; Nothing else would ask for a frame: `tick!` only runs inside a ;; render, and the loop only keeps itself alive once it has started. @@ -162,6 +294,10 @@ :source source :shape-id shape-id :image-id image-id + :tex-width nil + :tex-height nil + :pending? false + :frame-handle nil :last-time nil}) ;; Same-origin assets need no CORS, but a remote video served without the ;; headers would taint the canvas and make `texImage2D` throw. @@ -221,16 +357,32 @@ [image-id] (when-let [^js element (get-in @videos [image-id :element])] (if (.-paused element) - (-> (.play element) - (.then (fn [_] (request-render!))) - (.catch (fn [cause] - (log/error :hint "Could not play video" :cause cause)))) + (do + (request-frame-callback! image-id element) + (-> (.play element) + (.then (fn [_] (request-render!))) + (.catch (fn [cause] + (log/error :hint "Could not play video" :cause cause))))) (.pause element)) (not (.-paused element)))) +(defn- overlay-enabled? + [] + (contains? (:features (deref st/state)) "video-overlay-wasm/v1")) + +(defn refuses-to-play + "Why `shape` will not play its video, or nil when it will. Only the composited + path refuses: with the flag off, frames still go through the tiles and any + shape can carry them." + [shape] + (when (overlay-enabled?) + (ineligible-reason (:id shape)))) + (defn sync-shape! "Reconciles one shape with its `:video` attribute. Called whenever the attribute changes, so attaching is a plain shape edit and undo comes free. + Also called when the shape's effects change, so adding a drop shadow to a + playing video stops it there and then. Returns nil: `set-wasm-attr!` treats what it gets back as pending image loads, and `detach!` would otherwise hand it the video registry." @@ -241,6 +393,11 @@ (str/blank? source) (detach! image-id) + ;; A flat stamp cannot reproduce opacity, blending, blur, a shadow or a + ;; stroke, so a shape carrying one shows its poster frame instead. + (some? (refuses-to-play shape)) + (detach! image-id) + (not= source (get-in @videos [image-id :source])) (attach! (:id shape) image-id source)))) nil) @@ -250,6 +407,9 @@ for and stops the ones no shape wants any more. Run on page load, so a video survives a reload; an unchanged attachment keeps playing." [objects] + ;; Cheap, and this is where the renderer learns the flag: it runs on every + ;; page load, and the flag can change between them. + (set-overlay-enabled! (overlay-enabled?)) (let [wanted (into {} (keep (fn [[shape-id shape]] (let [source (shape-source shape)] @@ -262,7 +422,8 @@ (detach! image-id))) (doseq [[image-id {:keys [shape-id source]}] wanted] (when-not (contains? @videos image-id) - (attach! shape-id image-id source))))) + (when-not (and (overlay-enabled?) (some? (ineligible-reason shape-id))) + (attach! shape-id image-id source)))))) (defn attach-to-selected! "Console entry point: attaches `source` to the selected image shape." diff --git a/frontend/src/app/render_wasm/api/webgl.cljs b/frontend/src/app/render_wasm/api/webgl.cljs index fa2266a341..4ca2a9c8d3 100644 --- a/frontend/src/app/render_wasm/api/webgl.cljs +++ b/frontend/src/app/render_wasm/api/webgl.cljs @@ -34,13 +34,25 @@ new-id)) (defn upload-texture-source! - "Uploads `source` — an HTMLImageElement, ImageBitmap or HTMLVideoElement — into - an existing texture." + "Allocates the texture storage and fills it with `source` — an + HTMLImageElement, ImageBitmap or HTMLVideoElement. Redefines the texture + level, so it is the call to make once per texture (or when the source + changes size), not the one to repeat per frame." [gl texture source] (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) texture) (.texImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 (.-RGBA ^js gl) (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) source) (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil)) +(defn update-texture-source! + "Overwrites the pixels of an already-allocated texture with `source`, which + must have the same size the texture was allocated with. Used for video + frames: unlike `upload-texture-source!` it does not redefine the level, so + the storage is not reallocated on every frame." + [gl texture source] + (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) texture) + (.texSubImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 0 0 (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) source) + (.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil)) + (defn create-webgl-texture-from-image "Creates a WebGL texture from an HTMLImageElement or ImageBitmap and returns the texture object" [gl image-element] diff --git a/frontend/src/app/render_wasm/shape.cljs b/frontend/src/app/render_wasm/shape.cljs index 6f419360c9..ce82eced85 100644 --- a/frontend/src/app/render_wasm/shape.cljs +++ b/frontend/src/app/render_wasm/shape.cljs @@ -170,13 +170,17 @@ (api/set-shape-fills id v false) :strokes - (into [] (api/set-shape-strokes id v false)) + (let [pending (into [] (api/set-shape-strokes id v false))] + (video/sync-shape! shape) + pending) :blend-mode - (api/set-shape-blend-mode v) + (do (api/set-shape-blend-mode v) + (video/sync-shape! shape)) :opacity - (api/set-shape-opacity v) + (do (api/set-shape-opacity v) + (video/sync-shape! shape)) :hidden (api/set-shape-hidden v) @@ -185,13 +189,15 @@ (api/set-shape-children v) :blur - (api/set-shape-blur v) + (do (api/set-shape-blur v) + (video/sync-shape! shape)) :background-blur (api/set-shape-background-blur v) :shadow - (api/set-shape-shadows v) + (do (api/set-shape-shadows v) + (video/sync-shape! shape)) :constraints-h (api/set-constraints-h v) @@ -314,6 +320,9 @@ (ctl/flex-layout? shape) (api/set-flex-layout shape)) + ;; Also reached from the effect cases above: a flat video stamp cannot + ;; carry opacity, blending, a blur, a shadow or a stroke, so gaining one + ;; has to stop playback. :video (video/sync-shape! shape) diff --git a/frontend/test/frontend_tests/render_wasm/video_test.cljs b/frontend/test/frontend_tests/render_wasm/video_test.cljs new file mode 100644 index 0000000000..a3485a1205 --- /dev/null +++ b/frontend/test/frontend_tests/render_wasm/video_test.cljs @@ -0,0 +1,40 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.render-wasm.video-test + (:require + [app.common.geom.rect :as grc] + [app.render-wasm.api.video :as video] + [cljs.test :as t :include-macros true])) + +(t/deftest video-inside-the-viewport-is-visible + (let [vbox (grc/make-rect 0 0 800 600) + selrect (grc/make-rect 100 100 200 200)] + (t/is (true? (boolean (video/visible-in-viewport? vbox selrect)))))) + +(t/deftest video-outside-the-viewport-is-not-visible + (let [vbox (grc/make-rect 0 0 800 600) + selrect (grc/make-rect 2000 2000 200 200)] + (t/is (false? (boolean (video/visible-in-viewport? vbox selrect)))))) + +(t/deftest video-partly-inside-the-viewport-is-visible + (let [vbox (grc/make-rect 0 0 800 600) + selrect (grc/make-rect 700 500 400 400)] + (t/is (true? (boolean (video/visible-in-viewport? vbox selrect)))))) + +(t/deftest an-unknown-viewport-counts-as-visible + (let [selrect (grc/make-rect 2000 2000 200 200)] + (t/is (true? (boolean (video/visible-in-viewport? nil selrect)))))) + +(t/deftest unknown-bounds-count-as-visible + (let [vbox (grc/make-rect 0 0 800 600)] + (t/is (true? (boolean (video/visible-in-viewport? vbox nil)))))) + +(t/deftest every-ineligible-code-maps-to-a-reason + ;; The codes are the `VideoIneligible` discriminants in + ;; render-wasm/src/render/video.rs; a gap would silently show no explanation. + (t/is (= (set (range 1 8)) + (set (keys video/ineligible-reasons))))) diff --git a/frontend/test/frontend_tests/runner.cljs b/frontend/test/frontend_tests/runner.cljs index e085b8f2ec..ea58881174 100644 --- a/frontend/test/frontend_tests/runner.cljs +++ b/frontend/test/frontend_tests/runner.cljs @@ -70,6 +70,7 @@ [frontend-tests.render-wasm.process-objects-test] [frontend-tests.render-wasm.text-editor-apply-styles-test] [frontend-tests.render-wasm.text-editor-caret-color-test] + [frontend-tests.render-wasm.video-test] [frontend-tests.svg-fills-test] [frontend-tests.text-editor-paste-guard-test] [frontend-tests.tokens.copy-paste-props-test] @@ -179,6 +180,7 @@ 'frontend-tests.render-wasm.process-objects-test 'frontend-tests.render-wasm.text-editor-apply-styles-test 'frontend-tests.render-wasm.text-editor-caret-color-test + 'frontend-tests.render-wasm.video-test 'frontend-tests.svg-fills-test 'frontend-tests.tokens.copy-paste-props-test 'frontend-tests.tokens.import-export-test diff --git a/frontend/translations/en.po b/frontend/translations/en.po index 5e1c18fcc4..81021a6ac4 100644 --- a/frontend/translations/en.po +++ b/frontend/translations/en.po @@ -8899,6 +8899,30 @@ msgstr "Use the play button at the header to run the prototype view." msgid "workspace.options.video" msgstr "Video" +#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:44 +msgid "workspace.options.video.blocked.blend-mode" +msgstr "Video does not play on a shape with a blend mode." + +#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:45 +msgid "workspace.options.video.blocked.blur" +msgstr "Video does not play on a shape with a blur." + +#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:47 +msgid "workspace.options.video.blocked.masked" +msgstr "Video does not play inside a mask." + +#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:43 +msgid "workspace.options.video.blocked.opacity" +msgstr "Video does not play on a shape below full opacity." + +#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:46 +msgid "workspace.options.video.blocked.shadow" +msgstr "Video does not play on a shape with a shadow." + +#: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:48 +msgid "workspace.options.video.blocked.stroke" +msgstr "Video does not play on a shape with a stroke." + #: src/app/main/ui/workspace/sidebar/options/menus/video.cljs:73 msgid "workspace.options.video.pause" msgstr "Pause video" diff --git a/render-wasm/src/render.rs b/render-wasm/src/render.rs index c2ee801c41..092c313ec7 100644 --- a/render-wasm/src/render.rs +++ b/render-wasm/src/render.rs @@ -18,6 +18,7 @@ pub mod text; pub mod text_editor; mod ui; mod vector; +pub mod video; use skia_safe::{self as skia, Matrix, RRect, Rect}; use std::borrow::Cow; @@ -42,10 +43,82 @@ use crate::{get_gpu_state, get_resources, performance}; pub use fonts::*; pub use images::*; +pub use video::{shape_overlay_blockers, video_overlay_eligibility, VideoIneligible, VideoRegistry}; pub(crate) use resources::RenderResources; type ClipStack = Vec<(Rect, Option, Matrix)>; +/// A shape's own contribution to a clip stack: its bounds, its corner radii and +/// the transform they are expressed in. Shared by the tree walk and by +/// `ancestor_clip_stack`, so a caller drawing outside the walk clips exactly the +/// same way. +fn clip_entry( + element: &Shape, + offset: Option<(f32, f32)>, + clip_inset: Option, +) -> (Rect, Option, Matrix) { + let mut bounds = element.selrect(); + if let Some(offset) = offset { + let x = bounds.x() - offset.0; + let y = bounds.y() - offset.1; + let width = bounds.width(); + let height = bounds.height(); + bounds.set_xywh(x, y, width, height); + } + let mut transform = element.transform; + transform.post_translate(bounds.center()); + transform.pre_translate(-bounds.center()); + + let corners = match &element.shape_type { + Type::Rect(data) => data.corners, + Type::Frame(data) => data.corners, + _ => None, + }; + + if let Some(clip_inset) = clip_inset.filter(|&e| e > 0.0) { + bounds.inset((clip_inset, clip_inset)); + } + + (bounds, corners, transform) +} + +/// The pool is not guaranteed acyclic while a file is loading, so ancestor +/// walks outside the render tree are bounded rather than trusting it. +const MAX_CLIP_ANCESTOR_DEPTH: u32 = 1024; + +/// The clip stack a shape inherits from its ancestors, outermost first — the +/// same stack `get_children_clip_bounds` accumulates while descending the tree. +/// Needed by callers that draw a shape outside that walk, such as the video +/// overlay, which would otherwise paint straight through a clipping board. +pub(crate) fn ancestor_clip_stack(shapes: ShapesPoolRef, shape: &Shape) -> Option { + let mut clips: ClipStack = Vec::new(); + let mut current = shape.parent_id; + let mut depth = 0; + + while let Some(parent_id) = current.filter(|id| !id.is_nil()) { + depth += 1; + if depth > MAX_CLIP_ANCESTOR_DEPTH { + break; + } + let Some(parent) = shapes.get(&parent_id) else { + break; + }; + if parent.clip() { + clips.push(clip_entry(parent, None, None)); + } + current = parent.parent_id; + } + + if clips.is_empty() { + return None; + } + + // Collected innermost first while walking up; the walk produces them + // outermost first. + clips.reverse(); + Some(clips) +} + #[repr(u8)] pub enum FrameType { None = 0, @@ -141,29 +214,10 @@ impl NodeRenderState { return self.clip_bounds.clone(); } - let mut bounds = element.selrect(); - if let Some(offset) = offset { - let x = bounds.x() - offset.0; - let y = bounds.y() - offset.1; - let width = bounds.width(); - let height = bounds.height(); - bounds.set_xywh(x, y, width, height); - } - let mut transform = element.transform; - transform.post_translate(bounds.center()); - transform.pre_translate(-bounds.center()); - - let corners = match &element.shape_type { - Type::Rect(data) => data.corners, - Type::Frame(data) => data.corners, - _ => None, - }; - - if let Some(clip_inset) = clip_inset.filter(|&e| e > 0.0) { - bounds.inset((clip_inset, clip_inset)); - } - - Self::append_clip(self.clip_bounds.clone(), (bounds, corners, transform)) + Self::append_clip( + self.clip_bounds.clone(), + clip_entry(element, offset, clip_inset), + ) } /// Calculates the clip bounds for shadow rendering of a given shape. @@ -391,6 +445,17 @@ pub(crate) struct RenderState { /// Frame id passed as `base_object` for viewer renders; always traversed. pub viewer_render_root: Option, pub touched_ids: HashSet, + /// Images currently backed by a playing video. Frames for these are + /// stamped during composition instead of rastered into the tiles. + pub videos: VideoRegistry, + /// Shape id -> video image id, for the shapes whose video is stamped at + /// compose time this frame. Their image fill is left out of the tiles so + /// the frame shows through. Refreshed on attach/detach and at the start of + /// every render loop. + pub composited_videos: HashMap, + /// `video-overlay-wasm/v1`. With it off the renderer keeps painting video + /// frames through the tiles, exactly as before. + pub video_overlay_enabled: bool, /// Pre-edit extrects for old∪new tile eviction (captured on first touch). touched_prev_extrects: HashMap, /// Temporary flag used for off-screen passes (drop-shadow masks, filter surfaces, etc.) @@ -613,6 +678,9 @@ impl RenderState { include_filter: None, viewer_render_root: None, touched_ids: HashSet::default(), + videos: VideoRegistry::new(), + composited_videos: HashMap::default(), + video_overlay_enabled: false, touched_prev_extrects: HashMap::default(), ignore_nested_blurs: false, preview_mode: false, @@ -988,6 +1056,13 @@ impl RenderState { if self.viewer_masked_pass() { self.surfaces.clear_target(skia::Color::TRANSPARENT); self.surfaces.copy_backbuffer_to_target_replace(); + } else if self.has_composited_video() { + // Background, then the video frames, then the backbuffer over them. + // The backbuffer is transparent where an eligible video sits, so + // the frame shows through and anything stacked above it still wins. + self.surfaces.clear_target(self.background_color); + self.render_video_overlay(tree); + self.surfaces.draw_backbuffer_over_target(); } else { self.surfaces .copy_backbuffer_to_target(self.background_color); @@ -1003,6 +1078,144 @@ impl RenderState { debug::render_wasm_label(self); } + /// What the tile composite clears the backbuffer to. Transparent while a + /// video is stamped at compose time, so the hole left in the tiles is not + /// filled in with the page background before the frame is drawn under it. + /// `compose_frame` paints the background itself in that case. + pub fn backbuffer_clear_color(&self) -> skia::Color { + if self.has_composited_video() { + skia::Color::TRANSPARENT + } else { + self.background_color + } + } + + /// Recomputes which shapes have their video stamped at compose time. + /// Cheap: one pass over the registered video images, and each of them is + /// carried by a handful of shapes at most. + pub fn refresh_composited_videos(&mut self, tree: ShapesPoolRef) { + self.composited_videos.clear(); + if !self.video_overlay_enabled || self.videos.is_empty() { + return; + } + + for shape in tree.iter() { + if shape.id.is_nil() || shape.deleted() { + continue; + } + if let Ok(image_id) = video::video_overlay_eligibility(tree, shape, &self.videos) { + self.composited_videos.insert(shape.id, image_id); + } + } + } + + /// True when `image_id` is the video this shape has stamped at compose + /// time, so the tiles must leave a transparent hole instead of painting the + /// poster frame. + pub fn is_composited_video(&self, shape_id: &Uuid, image_id: &Uuid) -> bool { + self.composited_videos.get(shape_id) == Some(image_id) + } + + /// Whether anything is being stamped at compose time. While it is, the tile + /// composite leaves the backbuffer transparent so the stamp shows through. + pub fn has_composited_video(&self) -> bool { + !self.composited_videos.is_empty() + } + + /// Stamps each playing video onto Target, before the backbuffer is drawn + /// over it. The backbuffer carries a transparent hole where the video sits, + /// so shapes above the video still occlude it and shapes below stay hidden. + fn render_video_overlay(&mut self, tree: ShapesPoolRef) { + if self.composited_videos.is_empty() { + return; + } + + let zoom = self.viewbox.zoom * self.options.dpr; + let pan = self.viewbox.area; + let entries: Vec<(Uuid, Uuid)> = self + .composited_videos + .iter() + .map(|(shape_id, image_id)| (*shape_id, *image_id)) + .collect(); + + for (shape_id, image_id) in entries { + let Some(shape) = tree.get(&shape_id) else { + continue; + }; + let Some(image) = get_resources().images.get(&image_id) else { + continue; + }; + let Some(image_fill) = shape.fills().find_map(|fill| match fill { + Fill::Image(image_fill) if image_fill.id() == image_id => Some(image_fill.clone()), + _ => None, + }) else { + continue; + }; + + let clips = ancestor_clip_stack(tree, shape); + let container = &shape.selrect; + let dest_rect = images::get_image_dest_rect(container, &image_fill); + let src_rect = images::get_source_rect(image.dimensions(), &dest_rect, &image_fill); + let corners = match &shape.shape_type { + Type::Rect(data) => data.corners, + Type::Frame(data) => data.corners, + _ => None, + }; + + let mut transform = shape.transform; + transform.post_translate(container.center()); + transform.pre_translate(-container.center()); + + let sampling = get_resources().sampling_options; + let canvas = self.surfaces.canvas(SurfaceId::Target); + canvas.save(); + canvas.scale((zoom, zoom)); + canvas.translate((-pan.left, -pan.top)); + + if let Some(clips) = clips.as_ref() { + for (bounds, clip_corners, clip_transform) in clips.iter() { + canvas.concat(clip_transform); + match clip_corners { + Some(clip_corners) => { + canvas.clip_rrect( + RRect::new_rect_radii(*bounds, clip_corners), + skia::ClipOp::Intersect, + true, + ); + } + None => { + canvas.clip_rect(*bounds, skia::ClipOp::Intersect, true); + } + } + canvas.concat(&clip_transform.invert().unwrap_or_default()); + } + } + + canvas.concat(&transform); + match corners { + Some(corners) => { + canvas.clip_rrect( + RRect::new_rect_radii(*container, &corners), + skia::ClipOp::Intersect, + true, + ); + } + None => { + canvas.clip_rect(*container, skia::ClipOp::Intersect, true); + } + } + + canvas.draw_image_rect_with_sampling_options( + image, + Some((&src_rect, skia::canvas::SrcRectConstraint::Strict)), + dest_rect, + sampling, + &skia::Paint::default(), + ); + canvas.restore(); + } + } + /// Drawn on Target before the UI surface is composited, so rulers and guides /// stay above the selection band fn render_text_editor_overlay(&mut self, tree: ShapesPoolRef) { @@ -2323,11 +2536,9 @@ impl RenderState { pub fn render_from_cache(&mut self, shapes: ShapesPoolRef) { let _start = performance::begin_timed_log!("render_from_cache"); performance::begin_measure!("render_from_cache"); - self.surfaces.draw_combined_atlas_to_backbuffer( - &self.viewbox, - &self.tile_viewbox, - self.background_color, - ); + let clear_color = self.backbuffer_clear_color(); + self.surfaces + .draw_combined_atlas_to_backbuffer(&self.viewbox, &self.tile_viewbox, clear_color); self.present_frame(shapes); performance::end_measure!("render_from_cache"); @@ -2394,6 +2605,9 @@ impl RenderState { sync_render: bool, ) -> Result { self.clear(tree); + // Shape edits can make a playing video eligible or not, so the set is + // settled before any tile is rastered with (or without) its hole. + self.refresh_composited_videos(tree); let _start = performance::begin_timed_log!("start_render_loop"); let scale = self.get_scale(); @@ -2566,17 +2780,18 @@ impl RenderState { if should_compose { // Fast mode skips the tile atlas; use the same doc-atlas + scale // overlays as render_from_cache instead of composing empty slots. + let clear_color = self.backbuffer_clear_color(); if self.options.is_fast_mode() { self.surfaces.draw_combined_atlas_to_backbuffer( &self.viewbox, &self.tile_viewbox, - self.background_color, + clear_color, ); } else { self.surfaces.draw_tile_atlas_to_backbuffer( &self.viewbox, &self.tile_viewbox, - self.background_color, + clear_color, ); } } @@ -2634,11 +2849,9 @@ impl RenderState { // Same composition as `continue_render_loop` for full frames: snapshot only the // drawable tile rect into the atlas (no blur-margin overlap), then blit once. if !self.viewer_masked_pass() { - self.surfaces.draw_tile_atlas_to_backbuffer( - &self.viewbox, - &self.tile_viewbox, - self.background_color, - ); + let clear_color = self.backbuffer_clear_color(); + self.surfaces + .draw_tile_atlas_to_backbuffer(&self.viewbox, &self.tile_viewbox, clear_color); } let saved_preview_mode = self.preview_mode; @@ -4587,3 +4800,79 @@ impl RenderState { get_gpu_state().context.free_gpu_resources(); } } + +#[cfg(test)] +mod clip_stack_tests { + use super::*; + use crate::shapes::Type; + use crate::state::ShapesPool; + + fn add_board(pool: &mut ShapesPool, clip: bool, x: f32, y: f32) -> Uuid { + let id = Uuid::new_v4(); + let board = pool.add_shape(id); + board.set_shape_type(Type::Frame(Default::default())); + board.set_selrect(x, y, x + 100.0, y + 100.0); + board.set_clip(clip); + id + } + + fn add_child(pool: &mut ShapesPool, parent_id: Uuid) -> Uuid { + let id = Uuid::new_v4(); + let child = pool.add_shape(id); + child.set_shape_type(Type::Rect(Default::default())); + child.set_selrect(0.0, 0.0, 10.0, 10.0); + child.parent_id = Some(parent_id); + id + } + + #[test] + fn a_shape_without_clipping_ancestors_has_no_clip_stack() { + let mut pool = ShapesPool::new(); + let board = add_board(&mut pool, false, 0.0, 0.0); + let child = add_child(&mut pool, board); + + let shape = pool.get(&child).unwrap(); + assert!(ancestor_clip_stack(&pool, shape).is_none()); + } + + #[test] + fn a_clipping_board_contributes_its_bounds() { + let mut pool = ShapesPool::new(); + let board = add_board(&mut pool, true, 5.0, 7.0); + let child = add_child(&mut pool, board); + + let shape = pool.get(&child).unwrap(); + let clips = ancestor_clip_stack(&pool, shape).expect("a clip stack"); + assert_eq!(clips.len(), 1); + assert_eq!(clips[0].0, pool.get(&board).unwrap().selrect()); + } + + #[test] + fn nested_boards_are_ordered_outermost_first() { + let mut pool = ShapesPool::new(); + let outer = add_board(&mut pool, true, 0.0, 0.0); + let inner = add_board(&mut pool, true, 20.0, 20.0); + pool.get_mut(&inner).unwrap().parent_id = Some(outer); + let child = add_child(&mut pool, inner); + + let shape = pool.get(&child).unwrap(); + let clips = ancestor_clip_stack(&pool, shape).expect("a clip stack"); + assert_eq!(clips.len(), 2); + assert_eq!(clips[0].0, pool.get(&outer).unwrap().selrect()); + assert_eq!(clips[1].0, pool.get(&inner).unwrap().selrect()); + } + + #[test] + fn a_board_showing_overflow_contributes_nothing() { + let mut pool = ShapesPool::new(); + let outer = add_board(&mut pool, true, 0.0, 0.0); + let inner = add_board(&mut pool, false, 20.0, 20.0); + pool.get_mut(&inner).unwrap().parent_id = Some(outer); + let child = add_child(&mut pool, inner); + + let shape = pool.get(&child).unwrap(); + let clips = ancestor_clip_stack(&pool, shape).expect("a clip stack"); + assert_eq!(clips.len(), 1); + assert_eq!(clips[0].0, pool.get(&outer).unwrap().selrect()); + } +} diff --git a/render-wasm/src/render/fills.rs b/render-wasm/src/render/fills.rs index ac740b064d..fe3cd66cff 100644 --- a/render-wasm/src/render/fills.rs +++ b/render-wasm/src/render/fills.rs @@ -73,6 +73,12 @@ fn draw_image_fill( antialias: bool, surface_id: SurfaceId, ) { + // A video stamped at compose time leaves a transparent hole here, so the + // frame drawn underneath the backbuffer shows through. + if render_state.is_composited_video(&shape.id, &image_fill.id()) { + return; + } + if draw_svg_image_fill( render_state, shape, diff --git a/render-wasm/src/render/surfaces.rs b/render-wasm/src/render/surfaces.rs index 36e5c02008..a59d4720d0 100644 --- a/render-wasm/src/render/surfaces.rs +++ b/render-wasm/src/render/surfaces.rs @@ -988,6 +988,19 @@ impl Surfaces { ); } + /// Draw `Backbuffer` over whatever `Target` already holds, without clearing + /// it first. Used when something was painted underneath — a video frame + /// showing through the backbuffer's transparent hole. + pub fn draw_backbuffer_over_target(&mut self) { + let sampling_options = self.sampling_options; + self.backbuffer.draw( + self.target.canvas(), + (0.0, 0.0), + sampling_options, + Some(&skia::Paint::default()), + ); + } + /// Replace `Target` pixels with `Backbuffer` (Src blend). /// /// Used for viewer masked passes: transparent backbuffer regions must not diff --git a/render-wasm/src/render/video.rs b/render-wasm/src/render/video.rs new file mode 100644 index 0000000000..9f11d5f133 --- /dev/null +++ b/render-wasm/src/render/video.rs @@ -0,0 +1,317 @@ +//! Video painted as a composited overlay rather than as part of the tiles. +//! +//! A video frame changes the picture 30 to 60 times a second. Painting it +//! through the normal path means invalidating the shape's tiles that often, +//! and a tile re-raster redraws every shape intersecting it — so the cost of +//! playing a video grows with how crowded the page is around it. +//! +//! Instead the renderer keeps a transparent hole where an eligible video sits +//! and stamps the frame during composition. That only produces the right +//! picture when the shape composites trivially, so anything carrying opacity, +//! a blend mode, a blur, a shadow or a stroke is refused outright and keeps +//! showing its poster frame. + +use std::collections::HashSet; + +use crate::shapes::{BlendMode, Blur, Fill, Shape, Type}; +use crate::state::ShapesPoolRef; +use crate::uuid::Uuid; + +/// The pool is not guaranteed acyclic while a file is loading, so the ancestor +/// walk is bounded rather than trusting it. +const MAX_ANCESTOR_DEPTH: u32 = 1024; + +/// Why a shape cannot have its video stamped during composition. Reported back +/// to the bridge so the sidebar can name the property that is in the way. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VideoIneligible { + NoVideoFill = 1, + Opacity = 2, + BlendMode = 3, + Blur = 4, + Shadow = 5, + Stroke = 6, + Masked = 7, +} + +/// The image ids currently backed by a playing video. +#[derive(Default)] +pub struct VideoRegistry { + images: HashSet, +} + +impl VideoRegistry { + pub fn new() -> Self { + Self::default() + } + + pub fn register(&mut self, image_id: Uuid) { + self.images.insert(image_id); + } + + pub fn unregister(&mut self, image_id: Uuid) { + self.images.remove(&image_id); + } + + pub fn is_empty(&self) -> bool { + self.images.is_empty() + } + + pub fn contains(&self, image_id: &Uuid) -> bool { + self.images.contains(image_id) + } + + pub fn iter(&self) -> impl Iterator { + self.images.iter() + } +} + +/// The video image a shape paints, if it paints one. +pub fn video_fill_id(shape: &Shape, videos: &VideoRegistry) -> Option { + shape.fills().find_map(|fill| match fill { + Fill::Image(image) if videos.contains(&image.id()) => Some(image.id()), + _ => None, + }) +} + +fn has_masked_ancestor(shapes: ShapesPoolRef, shape: &Shape) -> bool { + let mut current = shape.parent_id; + let mut depth = 0; + while let Some(parent_id) = current.filter(|id| !id.is_nil()) { + depth += 1; + if depth > MAX_ANCESTOR_DEPTH { + return true; + } + let Some(parent) = shapes.get_raw(&parent_id) else { + return false; + }; + if matches!(&parent.shape_type, Type::Group(group) if group.masked) { + return true; + } + current = parent.parent_id; + } + false +} + +/// Whether `shape` can have its video stamped during composition instead of +/// rastered into the tiles. +/// +/// The overlay is a flat stamp: it cannot blend, blur, cast a shadow, or sit +/// under an inner stroke. Rotation and corner radii are fine — the stamp +/// carries the shape's transform and clips to its rounded rect. +pub fn video_overlay_eligibility( + shapes: ShapesPoolRef, + shape: &Shape, + videos: &VideoRegistry, +) -> Result { + let image_id = video_fill_id(shape, videos).ok_or(VideoIneligible::NoVideoFill)?; + shape_overlay_blockers(shapes, shape)?; + Ok(image_id) +} + +/// The properties that stop a shape being stamped, independent of whether a +/// video is attached to it yet. The bridge asks this before starting a video, +/// so the sidebar can refuse with a reason rather than starting and stopping. +pub fn shape_overlay_blockers( + shapes: ShapesPoolRef, + shape: &Shape, +) -> Result<(), VideoIneligible> { + if shape.opacity < 1.0 { + return Err(VideoIneligible::Opacity); + } + if shape.blend_mode != BlendMode::default() { + return Err(VideoIneligible::BlendMode); + } + // Hidden or zero effects paint nothing, so they are no reason to refuse. + let visible_blur = |blur: &Option| { + blur.map(|blur| !blur.hidden && blur.value > 0.0) + .unwrap_or(false) + }; + if visible_blur(&shape.blur) || visible_blur(&shape.background_blur) { + return Err(VideoIneligible::Blur); + } + if shape.shadows.iter().any(|shadow| !shadow.hidden()) { + return Err(VideoIneligible::Shadow); + } + if shape.has_visible_strokes() { + return Err(VideoIneligible::Stroke); + } + if has_masked_ancestor(shapes, shape) { + return Err(VideoIneligible::Masked); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::shapes::{BlurType, Group, ImageFill, Shadow, ShadowStyle, Stroke, StrokeStyle}; + use crate::state::ShapesPool; + + fn registry(image_id: Uuid) -> VideoRegistry { + let mut videos = VideoRegistry::new(); + videos.register(image_id); + videos + } + + fn pool_with_video(image_id: Uuid) -> (ShapesPool, Uuid) { + let mut pool = ShapesPool::new(); + let shape_id = Uuid::new_v4(); + let shape = pool.add_shape(shape_id); + shape.set_shape_type(Type::Rect(Default::default())); + shape.add_fill(Fill::Image(ImageFill::new(image_id, 255, 10, 10, false))); + (pool, shape_id) + } + + fn eligibility( + pool: &ShapesPool, + shape_id: Uuid, + videos: &VideoRegistry, + ) -> Result { + let shape = pool.get(&shape_id).unwrap(); + video_overlay_eligibility(pool, shape, videos) + } + + #[test] + fn a_plain_video_shape_is_eligible() { + let image_id = Uuid::new_v4(); + let (pool, shape_id) = pool_with_video(image_id); + + assert_eq!( + eligibility(&pool, shape_id, ®istry(image_id)), + Ok(image_id) + ); + } + + #[test] + fn rotation_and_corner_radii_stay_eligible() { + let image_id = Uuid::new_v4(); + let (mut pool, shape_id) = pool_with_video(image_id); + { + let shape = pool.get_mut(&shape_id).unwrap(); + shape.rotation = 33.0; + shape.set_corners((8.0, 8.0, 8.0, 8.0)); + } + + assert_eq!( + eligibility(&pool, shape_id, ®istry(image_id)), + Ok(image_id) + ); + } + + #[test] + fn a_shape_without_a_registered_video_is_not_eligible() { + let image_id = Uuid::new_v4(); + let (pool, shape_id) = pool_with_video(image_id); + + assert_eq!( + eligibility(&pool, shape_id, &VideoRegistry::new()), + Err(VideoIneligible::NoVideoFill) + ); + } + + #[test] + fn opacity_below_one_is_refused() { + let image_id = Uuid::new_v4(); + let (mut pool, shape_id) = pool_with_video(image_id); + pool.get_mut(&shape_id).unwrap().opacity = 0.4; + + assert_eq!( + eligibility(&pool, shape_id, ®istry(image_id)), + Err(VideoIneligible::Opacity) + ); + } + + #[test] + fn a_blend_mode_is_refused() { + let image_id = Uuid::new_v4(); + let (mut pool, shape_id) = pool_with_video(image_id); + pool.get_mut(&shape_id).unwrap().blend_mode = BlendMode(skia_safe::BlendMode::Multiply); + + assert_eq!( + eligibility(&pool, shape_id, ®istry(image_id)), + Err(VideoIneligible::BlendMode) + ); + } + + #[test] + fn a_blur_is_refused() { + let image_id = Uuid::new_v4(); + let (mut pool, shape_id) = pool_with_video(image_id); + pool.get_mut(&shape_id).unwrap().blur = + Some(Blur::new(BlurType::LayerBlur, false, 4.0)); + + assert_eq!( + eligibility(&pool, shape_id, ®istry(image_id)), + Err(VideoIneligible::Blur) + ); + } + + #[test] + fn a_shadow_is_refused() { + let image_id = Uuid::new_v4(); + let (mut pool, shape_id) = pool_with_video(image_id); + pool.get_mut(&shape_id).unwrap().shadows.push(Shadow::new( + skia_safe::Color::BLACK, + 4.0, + 0.0, + (2.0, 2.0), + ShadowStyle::Drop, + false, + )); + + assert_eq!( + eligibility(&pool, shape_id, ®istry(image_id)), + Err(VideoIneligible::Shadow) + ); + } + + #[test] + fn a_stroke_is_refused() { + let image_id = Uuid::new_v4(); + let (mut pool, shape_id) = pool_with_video(image_id); + pool.get_mut(&shape_id).unwrap().add_stroke( + Stroke::new_center_stroke(2.0, StrokeStyle::Solid, None, None, None, None), + ); + + assert_eq!( + eligibility(&pool, shape_id, ®istry(image_id)), + Err(VideoIneligible::Stroke) + ); + } + + #[test] + fn a_masked_ancestor_is_refused() { + let image_id = Uuid::new_v4(); + let (mut pool, shape_id) = pool_with_video(image_id); + let group_id = Uuid::new_v4(); + { + let group = pool.add_shape(group_id); + group.set_shape_type(Type::Group(Group { masked: true })); + } + pool.get_mut(&shape_id).unwrap().parent_id = Some(group_id); + + assert_eq!( + eligibility(&pool, shape_id, ®istry(image_id)), + Err(VideoIneligible::Masked) + ); + } + + #[test] + fn an_unmasked_ancestor_stays_eligible() { + let image_id = Uuid::new_v4(); + let (mut pool, shape_id) = pool_with_video(image_id); + let group_id = Uuid::new_v4(); + { + let group = pool.add_shape(group_id); + group.set_shape_type(Type::Group(Group { masked: false })); + } + pool.get_mut(&shape_id).unwrap().parent_id = Some(group_id); + + assert_eq!( + eligibility(&pool, shape_id, ®istry(image_id)), + Ok(image_id) + ); + } +} diff --git a/render-wasm/src/shapes.rs b/render-wasm/src/shapes.rs index bb3c74f7f0..ab5c62edc9 100644 --- a/render-wasm/src/shapes.rs +++ b/render-wasm/src/shapes.rs @@ -678,6 +678,16 @@ impl Shape { self.fills.iter() } + /// Every image this shape paints, from its fills and from its strokes. + pub fn image_ids(&self) -> impl Iterator + '_ { + let fills = self.fills.iter(); + let strokes = self.strokes.iter().map(|stroke| &stroke.fill); + fills.chain(strokes).filter_map(|fill| match fill { + Fill::Image(image) => Some(image.id()), + _ => None, + }) + } + pub fn set_fills(&mut self, fills: Vec) { self.deferred_batch_fills = None; self.fills = fills; @@ -2034,6 +2044,35 @@ mod tests { ) } + fn image_fill(id: Uuid) -> Fill { + Fill::Image(ImageFill::new(id, 255, 10, 10, false)) + } + + #[test] + fn image_ids_reports_fill_and_stroke_images() { + let fill_image = Uuid::new_v4(); + let stroke_image = Uuid::new_v4(); + + let mut shape = any_shape(); + shape.add_fill(Fill::Solid(SolidColor(Color::TRANSPARENT))); + shape.add_fill(image_fill(fill_image)); + + let mut stroke = Stroke::new_center_stroke(2.0, StrokeStyle::Solid, None, None, None, None); + stroke.fill = image_fill(stroke_image); + shape.add_stroke(stroke); + + let ids: Vec = shape.image_ids().collect(); + assert_eq!(ids, vec![fill_image, stroke_image]); + } + + #[test] + fn image_ids_is_empty_without_image_fills() { + let mut shape = any_shape(); + shape.add_fill(Fill::Solid(SolidColor(Color::TRANSPARENT))); + + assert_eq!(shape.image_ids().count(), 0); + } + #[test] fn layer_blur_and_background_blur_can_coexist() { let mut shape = any_shape(); diff --git a/render-wasm/src/state/shapes_pool.rs b/render-wasm/src/state/shapes_pool.rs index f39825ebaf..3a742141e9 100644 --- a/render-wasm/src/state/shapes_pool.rs +++ b/render-wasm/src/state/shapes_pool.rs @@ -59,6 +59,14 @@ pub struct ShapesPoolImpl { structure: HashMap>, /// Scale content values, keyed by index scale_content: HashMap, + + /// Bumped whenever a shape is added or handed out mutably, so derived + /// indexes can tell a cached answer from a stale one. + revision: u64, + /// Image id -> shapes painting it, with the revision it was built at. + /// Rebuilt on demand; while nothing mutates shapes (video playback, for + /// instance) the lookup stays O(1) instead of walking the whole pool. + image_index: Option<(u64, HashMap>)>, } // Type aliases - no longer need lifetimes! @@ -78,11 +86,19 @@ impl ShapesPoolImpl { modifier_uuids: Vec::new(), structure: HashMap::default(), scale_content: HashMap::default(), + revision: 0, + image_index: None, } } + #[inline] + fn bump_revision(&mut self) { + self.revision = self.revision.wrapping_add(1); + } + pub fn initialize(&mut self, capacity: usize) { performance::begin_measure!("shapes_pool_initialize"); + self.bump_revision(); self.counter = 0; self.uuid_to_idx = HashMap::with_capacity(capacity); @@ -102,6 +118,7 @@ impl ShapesPoolImpl { } pub fn add_shape(&mut self, id: Uuid) -> &mut Shape { + self.bump_revision(); if self.counter >= self.shapes.len() { // We need more space let current_capacity = self.shapes.capacity(); @@ -144,9 +161,42 @@ impl ShapesPoolImpl { pub fn get_mut(&mut self, id: &Uuid) -> Option<&mut Shape> { let idx = *self.uuid_to_idx.get(id)?; + self.bump_revision(); Some(&mut self.shapes[idx]) } + /// Shapes whose fills or strokes paint `image_id`. The index behind it is + /// rebuilt only after a shape changed, so a caller that runs every frame + /// (video frame uploads) pays for the walk once, not once per frame. + pub fn shapes_with_image(&mut self, image_id: Uuid) -> &[Uuid] { + let stale = !matches!(&self.image_index, Some((rev, _)) if *rev == self.revision); + if stale { + self.image_index = Some((self.revision, self.build_image_index())); + } + + self.image_index + .as_ref() + .and_then(|(_, index)| index.get(&image_id)) + .map(Vec::as_slice) + .unwrap_or(&[]) + } + + fn build_image_index(&self) -> HashMap> { + let mut index: HashMap> = HashMap::default(); + for shape in self.shapes.iter() { + if shape.id.is_nil() || shape.deleted() { + continue; + } + for image_id in shape.image_ids() { + let shapes = index.entry(image_id).or_default(); + if !shapes.contains(&shape.id) { + shapes.push(shape.id); + } + } + } + index + } + /// Returns the current transform modifier matrix for the shape, if any. pub fn get_modifier(&self, id: &Uuid) -> Option<&skia::Matrix> { let idx = *self.uuid_to_idx.get(id)?; @@ -241,6 +291,7 @@ impl ShapesPoolImpl { #[allow(dead_code)] pub fn iter_mut(&mut self) -> std::slice::IterMut<'_, Shape> { + self.bump_revision(); self.shapes.iter_mut() } @@ -552,6 +603,98 @@ impl Clone for ShapesPoolImpl { modifier_uuids: self.modifier_uuids.clone(), structure: self.structure.clone(), scale_content: self.scale_content.clone(), + revision: self.revision, + // Derived like modified_shape_cache: dropped on clone and rebuilt + // on demand. + image_index: None, } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::shapes::{Fill, ImageFill}; + + fn image_fill(id: Uuid) -> Fill { + Fill::Image(ImageFill::new(id, 255, 10, 10, false)) + } + + fn pool_with_shape(image_id: Uuid) -> (ShapesPool, Uuid) { + let mut pool = ShapesPool::new(); + let shape_id = Uuid::new_v4(); + pool.add_shape(shape_id).add_fill(image_fill(image_id)); + (pool, shape_id) + } + + #[test] + fn finds_the_shapes_painting_an_image() { + let image_id = Uuid::new_v4(); + let (mut pool, shape_id) = pool_with_shape(image_id); + + assert_eq!(pool.shapes_with_image(image_id), &[shape_id]); + assert!(pool.shapes_with_image(Uuid::new_v4()).is_empty()); + } + + #[test] + fn lists_every_shape_painting_the_same_image() { + let image_id = Uuid::new_v4(); + let (mut pool, first) = pool_with_shape(image_id); + let second = Uuid::new_v4(); + pool.add_shape(second).add_fill(image_fill(image_id)); + + let found = pool.shapes_with_image(image_id).to_vec(); + assert_eq!(found.len(), 2); + assert!(found.contains(&first)); + assert!(found.contains(&second)); + } + + #[test] + fn a_replaced_fill_is_picked_up() { + let image_id = Uuid::new_v4(); + let replacement = Uuid::new_v4(); + let (mut pool, shape_id) = pool_with_shape(image_id); + assert_eq!(pool.shapes_with_image(image_id), &[shape_id]); + + pool.get_mut(&shape_id) + .unwrap() + .set_fills(vec![image_fill(replacement)]); + + assert!(pool.shapes_with_image(image_id).is_empty()); + assert_eq!(pool.shapes_with_image(replacement), &[shape_id]); + } + + #[test] + fn a_cleared_fill_is_picked_up() { + let image_id = Uuid::new_v4(); + let (mut pool, shape_id) = pool_with_shape(image_id); + assert_eq!(pool.shapes_with_image(image_id), &[shape_id]); + + pool.get_mut(&shape_id).unwrap().clear_fills(); + + assert!(pool.shapes_with_image(image_id).is_empty()); + } + + #[test] + fn a_deleted_shape_drops_out() { + let image_id = Uuid::new_v4(); + let (mut pool, shape_id) = pool_with_shape(image_id); + assert_eq!(pool.shapes_with_image(image_id), &[shape_id]); + + pool.get_mut(&shape_id).unwrap().set_deleted(true); + + assert!(pool.shapes_with_image(image_id).is_empty()); + } + + #[test] + fn a_shape_added_after_the_first_lookup_is_found() { + let image_id = Uuid::new_v4(); + let mut pool = ShapesPool::new(); + assert!(pool.shapes_with_image(image_id).is_empty()); + + let shape_id = Uuid::new_v4(); + pool.add_shape(shape_id).add_fill(image_fill(image_id)); + + assert_eq!(pool.shapes_with_image(image_id), &[shape_id]); + } +} diff --git a/render-wasm/src/wasm.rs b/render-wasm/src/wasm.rs index a5d8261f4d..ad5424ace6 100644 --- a/render-wasm/src/wasm.rs +++ b/render-wasm/src/wasm.rs @@ -13,3 +13,4 @@ pub mod text; pub mod text_editor; pub mod transforms; pub mod ui; +pub mod video; diff --git a/render-wasm/src/wasm/fills/image.rs b/render-wasm/src/wasm/fills/image.rs index 52c4d9f6a9..92199156a5 100644 --- a/render-wasm/src/wasm/fills/image.rs +++ b/render-wasm/src/wasm/fills/image.rs @@ -1,8 +1,8 @@ use crate::error::{Error, Result}; use crate::get_gpu_state; +use crate::get_render_state; use crate::get_resources; use crate::mem; -use crate::shapes::Fill; use crate::state::State; use crate::uuid::Uuid; use crate::with_state; @@ -10,26 +10,28 @@ use crate::{shapes::ImageFill, utils::uuid_from_u32_quartet}; use macros::wasm_error; fn touch_shapes_with_image(state: &mut State, image_id: Uuid) { - let ids: Vec = state - .shapes - .iter() - .filter(|shape| { - shape - .fills() - .any(|f| matches!(f, Fill::Image(i) if i.id() == image_id)) - || shape - .strokes - .iter() - .any(|s| matches!(&s.fill, Fill::Image(i) if i.id() == image_id)) - }) - .map(|shape| shape.id) - .collect(); + let ids: Vec = state.shapes.shapes_with_image(image_id).to_vec(); for id in ids { state.touch_shape(id); } } +/// Like `touch_shapes_with_image`, but leaves out the shapes whose video is +/// stamped at compose time: their tiles hold a transparent hole, so a new frame +/// costs a recomposition and no tile work at all. A shape painting the same +/// video that cannot be composited — one carrying a shadow, say — still needs +/// its tiles back. +fn touch_shapes_not_compositing_image(state: &mut State, image_id: Uuid) { + let ids: Vec = state.shapes.shapes_with_image(image_id).to_vec(); + + for id in ids { + if !get_render_state().is_composited_video(&id, &image_id) { + state.touch_shape(id); + } + } +} + const FLAG_KEEP_ASPECT_RATIO: u8 = 1 << 0; const FLAG_HAS_TRANSFORM: u8 = 1 << 1; const IMAGE_IDS_SIZE: usize = 32; @@ -290,7 +292,8 @@ pub extern "C" fn update_image_from_texture() -> Result<()> { ) { eprintln!("update_image_from_texture error: {}", msg); } - touch_shapes_with_image(state, upload.ids.image_id); + + touch_shapes_not_compositing_image(state, upload.ids.image_id); }); mem::free_bytes()?; diff --git a/render-wasm/src/wasm/video.rs b/render-wasm/src/wasm/video.rs new file mode 100644 index 0000000000..c1f8ca211d --- /dev/null +++ b/render-wasm/src/wasm/video.rs @@ -0,0 +1,67 @@ +use crate::get_render_state; +use crate::render::video::{shape_overlay_blockers, VideoIneligible}; +use crate::utils::uuid_from_u32_quartet; +use crate::uuid::Uuid; +use crate::with_state; + +/// Redraws the shapes painting `image_id` once, so the tiles pick up the switch +/// between the poster frame and the composited overlay. +fn resync_image(image_id: Uuid) { + with_state!(state, { + get_render_state().refresh_composited_videos(&state.shapes); + let ids = state.shapes.shapes_with_image(image_id).to_vec(); + for id in ids { + state.touch_shape(id); + } + }); +} + +/// Marks an image as backed by a playing video, so its frames are stamped +/// during composition instead of rastered into the tiles. Called by +/// `app.render-wasm.api.video` when a video starts. +#[no_mangle] +pub extern "C" fn register_video_image(a: u32, b: u32, c: u32, d: u32) { + let image_id = uuid_from_u32_quartet(a, b, c, d); + get_render_state().videos.register(image_id); + resync_image(image_id); +} + +/// Stops treating an image as video. The shapes painting it go back to showing +/// the still image. +#[no_mangle] +pub extern "C" fn unregister_video_image(a: u32, b: u32, c: u32, d: u32) { + let image_id = uuid_from_u32_quartet(a, b, c, d); + get_render_state().videos.unregister(image_id); + resync_image(image_id); +} + +/// Whether a shape could have a video stamped during composition, as a code the +/// bridge turns into the reason shown in the sidebar. `0` means it could; any +/// other value is a `VideoIneligible` discriminant. Answered from the shape's +/// own properties, so it can be asked before a video is attached. +#[no_mangle] +pub extern "C" fn get_video_eligibility(a: u32, b: u32, c: u32, d: u32) -> u32 { + with_state!(state, { + let shape_id = uuid_from_u32_quartet(a, b, c, d); + + let Some(shape) = state.shapes.get_raw(&shape_id) else { + return VideoIneligible::NoVideoFill as u32; + }; + + match shape_overlay_blockers(&state.shapes, shape) { + Ok(()) => 0, + Err(reason) => reason as u32, + } + }) +} + +/// Turns the `video-overlay-wasm/v1` path on or off. With it off the renderer +/// keeps painting video frames through the tiles. +#[no_mangle] +pub extern "C" fn set_video_overlay_enabled(enabled: bool) { + get_render_state().video_overlay_enabled = enabled; + let image_ids: Vec = get_render_state().videos.iter().copied().collect(); + for image_id in image_ids { + resync_image(image_id); + } +}