github/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2026-09-22 03:46:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Elena Torro 2026-09-16 17:48:50 +02:00
parent c62cecced8
commit 4086ed984e
9 changed files with 63 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
frontend/src/app/main/data/workspace/media.cljs
View File

@ -65,6 +65,16 @@
:hint "could not read the video dimensions"))))
(set! (.-src element) url)))))
(defn- media-dimensions
"Stream of the dimensions an upload has to be stored with: the size the
client decoded for a video, and nothing at all for a still, which the
backend measures itself. Every upload path runs through this, so a video
cannot reach the backend without them."
[blob]
(if (media/video-type? (.-type blob))
(rx/from (read-video-dimensions blob))
(rx/of nil)))
(defn- optimize
[input]
(svgo/optimize input svgo/defaultOptions))
@ -218,10 +228,8 @@
dimensions)))))
(upload-blob [blob]
(if (media/video-type? (.-type blob))
(->> (rx/from (read-video-dimensions blob))
(rx/mapcat #(upload-blob* blob %)))
(upload-blob* blob nil)))
(->> (media-dimensions blob)
(rx/mapcat #(upload-blob* blob %))))
(extract-content [blob]
(let [name (or name (.-name blob))]
@ -341,7 +349,9 @@
(dmm/notify-start-loading)
(->> (rx/of file)
(rx/map dmm/validate-file)
(rx/map prepare)
(rx/mapcat (fn [content]
(->> (media-dimensions content)
(rx/map #(merge (prepare content) %)))))
(rx/mapcat #(rp/cmd! :upload-file-media-object %))
(rx/tap on-upload-success)
(rx/catch handle-media-error))))))

31
frontend/src/app/main/ui/workspace/sidebar/options/menus/video.cljs
View File

@ -5,16 +5,11 @@
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns app.main.ui.workspace.sidebar.options.menus.video
"Plays a video into a shape's image fill.
"Proof of concept: plays a video into a shape's image fill.
The source is an uploaded video asset, or a path the browser can already
Nothing uploads video, so the source is a path the browser can already
reach — a file under `frontend/resources/public/images/` or a full URL. Only
the render-wasm renderer paints it; elsewhere the image fill still shows.
A video is stamped when the frame is composed, which is a flat draw: it
cannot carry opacity, a blend mode, a blur, a shadow or a stroke. A shape
with one of those does not play, and this menu says which one is in the
way."
the render-wasm renderer paints it; elsewhere the image fill still shows."
(:require-macros [app.main.style :as stl])
(:require
[app.main.data.workspace.shapes :as dwsh]
@ -38,25 +33,14 @@
(dissoc shape :video)
(assoc shape :video source)))))
(def ^:private reason-labels
{:opacity "workspace.options.video.blocked.opacity"
:blend-mode "workspace.options.video.blocked.blend-mode"
:blur "workspace.options.video.blocked.blur"
:shadow "workspace.options.video.blocked.shadow"
:stroke "workspace.options.video.blocked.stroke"
:masked "workspace.options.video.blocked.masked"})
(mf/defc video-menu*
[{:keys [ids image-id source is-asset blocked-reason]}]
[{:keys [ids image-id source is-asset]}]
(let [;; `video/playing?` reads the element, which is outside app state, so
;; the button tracks it locally.
playing* (mf/use-state #(video/playing? image-id))
playing (deref playing*)
blocked-label (get reason-labels blocked-reason)
has-source (and (not (str/blank? source))
(nil? blocked-label))
has-source (not (str/blank? source))
on-change
(mf/use-fn
@ -91,11 +75,6 @@
:selected playing
:tooltip-placement "top-left"
:icon i/play}])]]
;; Playback was refused: say which property is in the way, so a video that
;; stops after a shadow is added does not look broken.
(when (some? blocked-label)
[:div {:class (stl/css :blocked)}
(tr blocked-label)])
;; An uploaded video has nothing to type: its source is the asset itself,
;; and removing it means deleting the shape.
(when-not is-asset

8
frontend/src/app/main/ui/workspace/sidebar/options/menus/video.scss
View File

@ -4,7 +4,6 @@
//
// Copyright (c) KALEIDOS SUBSIDIARY SL
@use "ds/typography.scss" as t;
@use "../../../sidebar/common/sidebar.scss" as sidebar;
.element-set {
@ -22,10 +21,3 @@
flex-grow: 1;
min-width: 0;
}
.blocked {
@include t.use-typography("body-small");
color: var(--input-foreground-color-disabled);
margin-bottom: var(--sp-s);
}

10
frontend/src/app/main/ui/workspace/sidebar/options/shapes/rect.cljs
View File

@ -95,12 +95,7 @@
;; An uploaded video asset needs no source field, only playback.
video-asset?
(and render-wasm? (some? (video/video-fill shape)))
;; The composited path is a flat draw, so a shape carrying opacity, a
;; blend mode, a blur, a shadow or a stroke does not play at all.
video-blocked-reason
(when (some? video-image-id) (video/refuses-to-play shape))]
(and render-wasm? (some? (video/video-fill shape)))]
[:*
[:> layer-menu* {:ids ids
@ -151,8 +146,7 @@
[:> video-menu* {:ids ids
:image-id video-image-id
:source (video/shape-source shape)
:is-asset video-asset?
:blocked-reason video-blocked-reason}])
:is-asset video-asset?}])
[:> stroke-menu* {:ids ids
:type type

223
frontend/src/app/render_wasm/api/video.cljs
View File

@ -12,19 +12,15 @@
the current frame is uploaded into that texture and WASM re-wraps it, so
every shape whose image fill points at `image-id` paints it.
With `video-overlay-wasm/v1` on, an eligible shape has its frames stamped
when the frame is composed rather than rastered into the tiles, so playback
costs one textured quad instead of a tile re-raster. A shape carrying
opacity, a blend mode, a blur, a shadow or a stroke cannot be stamped that
way and does not play at all; `ineligible-reason` says which one is in the
way so the sidebar can explain it.
The source is stored on the shape as `:video`, so it persists with the file
and comes back on load. Drop a file into `frontend/resources/public/images/`,
where it is served like any other static asset, and name it from the Video
section of the design sidebar. `penpotAttachVideo(\"clip.mp4\")` does the same
from the console for the selected shape, without persisting it.
The source is either an uploaded video asset or the shape's `:video`
attribute — a file under `frontend/resources/public/images/` or a full URL.
`penpotAttachVideo(\"clip.mp4\")` does the same from the console for the
selected shape, without persisting it."
There is no backend support: nothing uploads video, so the source is a path
the browser can reach on its own."
(:require
[app.common.geom.rect :as grc]
[app.common.logging :as log]
[app.common.media :as cm]
[app.common.render-wasm.helpers :as h]
@ -32,7 +28,6 @@
[app.common.render-wasm.mem.heap32 :as mem.h32]
[app.common.render-wasm.wasm :as wasm]
[app.common.types.fills :as types.fills]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.main.data.helpers :as dsh]
[app.main.store :as st]
@ -45,26 +40,10 @@
;; Videos live next to the other static assets, so a bare file name is enough.
(def ^:private assets-path "/images/")
;; image-id -> {:element :source :texture :texture-id :shape-id :image-id
;; :tex-width :tex-height :pending? :frame-handle :last-time}
;; image-id -> {:element :source :texture :texture-id :shape-id :image-id :last-time}
;; The texture keys arrive later, once the video has decoded its first frame.
(defonce ^:private videos (atom {}))
(defn- update-entry!
"Applies `f` to an attachment, and does nothing when it is already detached —
frame callbacks and promises resolve after a detach."
[image-id f & args]
(swap! videos (fn [videos]
(if (contains? videos image-id)
(apply update videos image-id f args)
videos))))
(defn- frame-callbacks?
"True when the element reports each presented frame on its own. Widely
available, but the rAF poll stays as the fallback."
[^js element]
(fn? (.-requestVideoFrameCallback element)))
;; `api.cljs` owns the render loop and requires this namespace, so it installs
;; its requester here rather than being required back.
(defonce ^:private render-requester (atom nil))
@ -84,52 +63,6 @@
[]
(exists? js/document))
(def ineligible-reasons
"Codes returned by `_get_video_eligibility`, matching `VideoIneligible` in
`render-wasm/src/render/video.rs`. `0` means the video can be composited."
{1 :no-video-fill
2 :opacity
3 :blend-mode
4 :blur
5 :shadow
6 :stroke
7 :masked})
(defn- call-with-uuid!
[export id]
(let [buffer (uuid/get-u32 id)]
(h/call wasm/internal-module export
(aget buffer 0)
(aget buffer 1)
(aget buffer 2)
(aget buffer 3))))
(defn- register-image!
"Tells the renderer this image is backed by a playing video, so its frames are
stamped when the frame is composed instead of rastered into the tiles."
[image-id]
(when (wasm/live?)
(call-with-uuid! "_register_video_image" image-id)))
(defn- unregister-image!
[image-id]
(when (wasm/live?)
(call-with-uuid! "_unregister_video_image" image-id)))
(defn set-overlay-enabled!
"Threads `video-overlay-wasm/v1` to the renderer. With it off, video frames
keep going through the tiles."
[enabled]
(when (wasm/live?)
(h/call wasm/internal-module "_set_video_overlay_enabled" (boolean enabled))))
(defn ineligible-reason
"Why `shape-id` cannot have its video composited, or nil when it can. The
renderer owns the rule, so the sidebar and the render path cannot disagree."
[shape-id]
(when (wasm/live?)
(get ineligible-reasons (call-with-uuid! "_get_video_eligibility" shape-id))))
(defn- send-frame!
"Hands the texture to WASM, which re-wraps it as a Skia image and invalidates
the tiles of every shape painting it. Same layout as `_store_image_from_texture`."
@ -145,51 +78,21 @@
(h/call wasm/internal-module "_update_image_from_texture")))
(defn- decoded-frame?
[{:keys [^js element texture-id pending? last-time]}]
[{:keys [^js element image-id texture-id]}]
(and (some? texture-id)
(>= (.-readyState element) HAVE-CURRENT-DATA)
(pos? (.-videoWidth element))
(if (frame-callbacks? element)
pending?
;; rAF runs faster than most videos decode; without the frame callback
;; skip the upload when playback has not advanced since the last one.
(not= (.-currentTime element) last-time))))
(defn visible-in-viewport?
"True when `selrect` meets `vbox`. An unknown viewport or unknown bounds —
the viewer, a page still loading — count as visible, so the video keeps
painting rather than going blank."
[vbox selrect]
(or (nil? vbox)
(nil? selrect)
(grc/overlaps-rects? vbox selrect)))
(defn- on-screen?
"A video outside the viewport keeps playing — so it stays in step with the
others and resumes at the right moment — but stops paying for a texture
upload."
[shape-id]
(let [state (deref st/state)]
(visible-in-viewport? (get-in state [:workspace-local :vbox])
(:selrect (dsh/lookup-shape state shape-id)))))
;; rAF runs faster than most videos decode; skip the upload when
;; playback has not advanced since the last one.
(not= (.-currentTime element) (get-in @videos [image-id :last-time]))))
(defn- upload-frame!
[gl {:keys [^js element texture image-id shape-id tex-width tex-height] :as entry}]
[gl {:keys [^js element texture image-id] :as entry}]
(when (decoded-frame? entry)
(if-not (on-screen? shape-id)
(update-entry! image-id assoc :pending? false :last-time (.-currentTime element))
(let [width (.-videoWidth element)
height (.-videoHeight element)]
;; An adaptive stream can switch resolution mid-playback, and the
;; texture storage has to be redefined when it does.
(if (and (= width tex-width) (= height tex-height))
(webgl/update-texture-source! gl texture element)
(do
(webgl/upload-texture-source! gl texture element)
(update-entry! image-id assoc :tex-width width :tex-height height)))
(update-entry! image-id assoc :pending? false :last-time (.-currentTime element))
(send-frame! entry)
true))))
(swap! videos assoc-in [image-id :last-time] (.-currentTime element))
(webgl/upload-texture-source! gl texture element)
(send-frame! entry)
true))
(defn tick!
"Uploads a frame for every attached video that advanced. Called from the
@ -203,49 +106,21 @@
(vals @videos)))))
(defn active?
"True while some attached video needs the renderer to keep scheduling frames
instead of settling. A video reporting its own frames asks for a render when
it has one, so only the polling fallback keeps the loop awake."
"True while some attached video is playing, so the renderer keeps scheduling
frames instead of settling."
[]
(boolean (some (fn [{:keys [^js element]}]
(and (not (.-paused element))
(not (frame-callbacks? element))))
(boolean (some (fn [{:keys [^js element]}] (not (.-paused element)))
(vals @videos))))
(defn- request-frame-callback!
"Asks the element to report its next presented frame, and re-arms itself from
the callback so the chain lasts as long as the attachment does."
[image-id ^js element]
(when (and (frame-callbacks? element)
(contains? @videos image-id)
(nil? (get-in @videos [image-id :frame-handle])))
(let [handle (.requestVideoFrameCallback
element
(fn [_now _metadata]
(when (contains? @videos image-id)
(update-entry! image-id assoc :pending? true :frame-handle nil)
(request-render!)
(request-frame-callback! image-id element))))]
(update-entry! image-id assoc :frame-handle handle))))
(defn- cancel-frame-callback!
[^js element handle]
(when (and (some? handle) (fn? (.-cancelVideoFrameCallback element)))
(.cancelVideoFrameCallback element handle)))
(defn detach!
"Stops a video. The last frame stays on screen: the texture is deliberately
kept alive because WASM holds a Skia image borrowing it."
[image-id]
(when-let [{:keys [^js element frame-handle]} (get @videos image-id)]
;; Dropped from the registry first: the frame callback re-arms itself and
;; checks the registry to know when to stop.
(swap! videos dissoc image-id)
(unregister-image! image-id)
(cancel-frame-callback! element frame-handle)
(when-let [{:keys [^js element]} (get @videos image-id)]
(.pause element)
(.removeAttribute element "src")
(.load element)))
(.load element)
(swap! videos dissoc image-id)))
(defn detach-all!
[]
@ -256,16 +131,9 @@
[image-id ^js element]
(if-let [gl (webgl/get-webgl-context)]
(let [texture (webgl/create-webgl-texture-from-image gl element)]
(register-image! image-id)
(update-entry! image-id merge
{:texture texture
:texture-id (webgl/register-texture! texture)
;; `create-webgl-texture-from-image` allocated the storage
;; at this size; later frames only overwrite its pixels.
:tex-width (.-videoWidth element)
:tex-height (.-videoHeight element)
:pending? true})
(request-frame-callback! image-id element)
(swap! videos update image-id merge
{:texture texture
:texture-id (webgl/register-texture! texture)})
(-> (.play element)
;; Nothing else would ask for a frame: `tick!` only runs inside a
;; render, and the loop only keeps itself alive once it has started.
@ -294,10 +162,6 @@
:source source
:shape-id shape-id
:image-id image-id
:tex-width nil
:tex-height nil
:pending? false
:frame-handle nil
:last-time nil})
;; Same-origin assets need no CORS, but a remote video served without the
;; headers would taint the canvas and make `texImage2D` throw.
@ -357,32 +221,16 @@
[image-id]
(when-let [^js element (get-in @videos [image-id :element])]
(if (.-paused element)
(do
(request-frame-callback! image-id element)
(-> (.play element)
(.then (fn [_] (request-render!)))
(.catch (fn [cause]
(log/error :hint "Could not play video" :cause cause)))))
(-> (.play element)
(.then (fn [_] (request-render!)))
(.catch (fn [cause]
(log/error :hint "Could not play video" :cause cause))))
(.pause element))
(not (.-paused element))))
(defn- overlay-enabled?
[]
(contains? (:features (deref st/state)) "video-overlay-wasm/v1"))
(defn refuses-to-play
"Why `shape` will not play its video, or nil when it will. Only the composited
path refuses: with the flag off, frames still go through the tiles and any
shape can carry them."
[shape]
(when (overlay-enabled?)
(ineligible-reason (:id shape))))
(defn sync-shape!
"Reconciles one shape with its `:video` attribute. Called whenever the
attribute changes, so attaching is a plain shape edit and undo comes free.
Also called when the shape's effects change, so adding a drop shadow to a
playing video stops it there and then.
Returns nil: `set-wasm-attr!` treats what it gets back as pending image
loads, and `detach!` would otherwise hand it the video registry."
@ -393,11 +241,6 @@
(str/blank? source)
(detach! image-id)
;; A flat stamp cannot reproduce opacity, blending, blur, a shadow or a
;; stroke, so a shape carrying one shows its poster frame instead.
(some? (refuses-to-play shape))
(detach! image-id)
(not= source (get-in @videos [image-id :source]))
(attach! (:id shape) image-id source))))
nil)
@ -407,9 +250,6 @@
for and stops the ones no shape wants any more. Run on page load, so a video
survives a reload; an unchanged attachment keeps playing."
[objects]
;; Cheap, and this is where the renderer learns the flag: it runs on every
;; page load, and the flag can change between them.
(set-overlay-enabled! (overlay-enabled?))
(let [wanted (into {}
(keep (fn [[shape-id shape]]
(let [source (shape-source shape)]
@ -422,8 +262,7 @@
(detach! image-id)))
(doseq [[image-id {:keys [shape-id source]}] wanted]
(when-not (contains? @videos image-id)
(when-not (and (overlay-enabled?) (some? (ineligible-reason shape-id)))
(attach! shape-id image-id source))))))
(attach! shape-id image-id source)))))
(defn attach-to-selected!
"Console entry point: attaches `source` to the selected image shape."

16
frontend/src/app/render_wasm/api/webgl.cljs
View File

@ -34,25 +34,13 @@
new-id))
(defn upload-texture-source!
"Allocates the texture storage and fills it with `source` — an
HTMLImageElement, ImageBitmap or HTMLVideoElement. Redefines the texture
level, so it is the call to make once per texture (or when the source
changes size), not the one to repeat per frame."
"Uploads `source` — an HTMLImageElement, ImageBitmap or HTMLVideoElement — into
an existing texture."
[gl texture source]
(.bindTexture ^js gl (.-TEXTURE_2D ^js gl) texture)
(.texImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 (.-RGBA ^js gl) (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) source)
(.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil))
(defn update-texture-source!
"Overwrites the pixels of an already-allocated texture with `source`, which
must have the same size the texture was allocated with. Used for video
frames: unlike `upload-texture-source!` it does not redefine the level, so
the storage is not reallocated on every frame."
[gl texture source]
(.bindTexture ^js gl (.-TEXTURE_2D ^js gl) texture)
(.texSubImage2D ^js gl (.-TEXTURE_2D ^js gl) 0 0 0 (.-RGBA ^js gl) (.-UNSIGNED_BYTE ^js gl) source)
(.bindTexture ^js gl (.-TEXTURE_2D ^js gl) nil))
(defn create-webgl-texture-from-image
"Creates a WebGL texture from an HTMLImageElement or ImageBitmap and returns the texture object"
[gl image-element]

6
frontend/src/app/render_wasm/shape.cljs
View File

@ -166,8 +166,12 @@
:transform
(api/set-shape-transform v)
;; An uploaded video carries its source in the fill, so replacing the
;; fill is what starts or stops playback.
:fills
(api/set-shape-fills id v false)
(let [pending (into [] (api/set-shape-fills id v false))]
(video/sync-shape! shape)
pending)
:strokes
(let [pending (into [] (api/set-shape-strokes id v false))]

2
render-wasm/src/render.rs
View File

@ -43,7 +43,7 @@ use crate::{get_gpu_state, get_resources, performance};
pub use fonts::*;
pub use images::*;
pub use video::{shape_overlay_blockers, video_overlay_eligibility, VideoIneligible, VideoRegistry};
pub use video::VideoRegistry;
pub(crate) use resources::RenderResources;
type ClipStack = Vec<(Rect, Option<Corners>, Matrix)>;

2
render-wasm/src/state/shapes_pool.rs
View File

@ -521,6 +521,8 @@ impl ShapesPoolImpl {
modifier_uuids: Vec::new(),
structure: HashMap::default(),
scale_content: HashMap::default(),
revision: 0,
image_index: None,
}
}