github/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2026-08-08 05:48:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

✨ Persist binfile manifest and emit workspace audit events (#11106)

Browse Source 
Persist binfile manifest metadata in file_data on import so file
statistics are available at open-workspace time. Emit a new
open-workspace-file audit event enriched with file statistics:
page count, shape count, component count, linked libraries,
design tokens, and whether the file is a shared library.

Closes #11106

AI-assisted-by: mimo-v2.5-pro
This commit is contained in:
Andrey Antukh 2026-08-06 07:57:05 +00:00
parent 2f04fcddbf
commit 5a7e3d5de5
8 changed files with 196 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/src/app/binfile/common.clj
View File

@ -723,6 +723,7 @@
(-> (select-keys file file-attrs)
(assoc :data nil)
(dissoc :team-id)
(dissoc :metadata)
(dissoc :migrations)))
(defn- file->file-data-params

8
backend/src/app/binfile/v3.clj
View File

@ -392,7 +392,7 @@
params {:type "penpot/export-files"
:version 1
:generated-by (str "penpot/" (:full cf/version))
:refer "penpot"
:referer "penpot"
:files (vec (vals files))
:relations rels}]
(write-entry! output "manifest.json" params))))
@ -734,7 +734,7 @@
:plugin-data plugin-data}))
(defn- import-file
[{:keys [::db/conn ::bfc/project-id] :as cfg} {file-id :id file-name :name}]
[{:keys [::db/conn ::bfc/project-id ::manifest] :as cfg} {file-id :id file-name :name}]
(let [file-id' (bfc/lookup-index file-id)
file (read-file cfg file-id)
media (read-file-media cfg file-id)
@ -801,8 +801,10 @@
(assoc :data data)
(assoc :name file-name)
(assoc :project-id project-id)
(assoc :metadata (d/without-nils
{:generated-by (get manifest :generated-by)
:referer (or (get manifest :referer) (get manifest :refer))}))
(dissoc :options))
file (bfc/process-file cfg file)
file (ctf/check-file file)]

2
backend/src/app/config.clj
View File

@ -52,7 +52,7 @@
:redis-uri "redis://redis/0"
:file-data-backend "legacy-db"
:file-data-backend "db"
:objects-storage-backend "fs"
:objects-storage-fs-directory "assets"

20
backend/src/app/features/fdata.clj
View File

@ -12,6 +12,7 @@
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.time :as ct]
[app.common.types.file :as ctf]
[app.common.types.objects-map :as omap]
[app.config :as cf]
[app.db :as db]
@ -159,15 +160,17 @@
:content-type "application/octet-stream"
:file-id file-id
:id id})
metadata {:storage-ref-id (:id sobject)}
metadata (-> (:metadata params)
(assoc :storage-ref-id (:id sobject)))
params (-> params
(assoc :metadata metadata)
(assoc :data nil))]
(upsert-in-database cfg params))
(= backend "db")
(->> (dissoc params :metadata)
(upsert-in-database cfg))
(let [metadata (dissoc (:metadata params) :storage-ref-id)
params (assoc params :metadata metadata)]
(upsert-in-database cfg params))
(= backend "legacy-db")
(cond
@ -213,18 +216,11 @@
[backend]
(or backend (cf/get :file-data-backend)))
(def ^:private schema:metadata
[:map {:title "Metadata"}
[:storage-ref-id {:optional true} ::sm/uuid]])
(def decode-metadata-with-schema
(sm/decoder schema:metadata sm/json-transformer))
(defn decode-metadata
[metadata]
(some-> metadata
(db/decode-json-pgobject)
(decode-metadata-with-schema)))
(ctf/decode-file-metadata)))
(def ^:private schema:update-params
[:map {:closed true}
@ -232,7 +228,7 @@
[:type [:enum "main" "snapshot" "fragment"]]
[:file-id ::sm/uuid]
[:backend {:optional true} [:enum "db" "legacy-db" "storage"]]
[:metadata {:optional true} [:maybe schema:metadata]]
[:metadata {:optional true} ctf/schema:file-metadata]
[:data {:optional true} bytes?]
[:created-at {:optional true} ::ct/inst]
[:modified-at {:optional true} [:maybe ::ct/inst]]

23
backend/test/backend_tests/binfile_test.clj
View File

@ -207,6 +207,29 @@
(t/is (= (count result) 1))
(t/is (every? uuid? result)))))
(t/deftest import-binfile-v3-persists-manifest-metadata
(let [profile (th/create-profile* 1)
file (prepare-simple-file profile)
output (tmp/tempfile :suffix ".zip")]
(v3/export-files!
(-> th/*system*
(assoc ::bfc/ids #{(:id file)})
(assoc ::bfc/embed-assets false)
(assoc ::bfc/include-libraries false))
(io/output-stream output))
(let [result (-> th/*system*
(assoc ::bfc/project-id (:default-project-id profile))
(assoc ::bfc/profile-id (:id profile))
(assoc ::bfc/input output)
(v3/import-files!))
imported (bfc/get-file th/*system* (first result))]
(t/is (= (count result) 1))
(t/is (some? (get-in imported [:metadata :generated-by])))
(t/is (= "penpot" (get-in imported [:metadata :referer]))))))
(t/deftest read-obj-rejects-oversized-buffer
;; N1-07: read-obj! must reject objects exceeding max-object-size
;; before attempting to allocate the buffer

10
common/src/app/common/types/file.cljc
View File

@ -88,6 +88,12 @@
[:plugin-data {:optional true} schema:plugin-data]
[:tokens-lib {:optional true} schema:tokens-lib]])
(def schema:file-metadata
[:map {:title "Metadata"}
[:storage-ref-id {:optional true} ::sm/uuid]
[:generated-by {:optional true} :string]
[:referer {:optional true} :string]])
(def schema:file
"A schema for validate a file data structure; data is optional
because sometimes we want to validate file without the data."
@ -106,6 +112,7 @@
[:data {:optional true} schema:data]
[:version :int]
[:features ::cfeat/features]
[:metadata {:optional true} schema:file-metadata]
[:migrations {:optional true}
[::sm/set {:ordered true} :string]]])
@ -123,6 +130,9 @@
(def check-file-media
(sm/check-fn schema:media))
(def decode-file-metadata
(sm/decoder schema:file-metadata sm/json-transformer))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; INITIALIZATION
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

61
frontend/src/app/main/data/workspace.cljs
View File

@ -17,11 +17,13 @@
[app.common.geom.proportions :as gpp]
[app.common.geom.shapes :as gsh]
[app.common.logging :as log]
[app.common.math :as mth]
[app.common.path-names :as cpn]
[app.common.transit :as t]
[app.common.types.component :as ctc]
[app.common.types.components-list :as ctkl]
[app.common.types.shape :as cts]
[app.common.types.tokens-lib :as ctob]
[app.common.types.variant :as ctv]
[app.common.uuid :as uuid]
[app.config :as cf]
@ -266,6 +268,59 @@
(rx/map (fn [_] (mcp/init))))
(rx/empty))))))
(defn- compute-shape-stats
"Compute shape statistics in a single pass over pages-index.
Returns {:num-shapes N :max-shapes-per-page M}"
[pages-index]
(reduce-kv
(fn [acc _page-id page]
(let [n (count (:objects page))]
(-> acc
(update :num-shapes + n)
(update :max-shapes-per-page max n))))
{:num-shapes 0
:max-shapes-per-page 0}
pages-index))
(defn compute-file-stats
"Compute file statistics. Returns a map of stats without event keys."
[state file-id]
(let [file (dsh/lookup-file state file-id)
file-data (:data file)
libraries (refs/select-libraries (:files state) file-id)
pages-index (:pages-index file-data)
{:keys [num-shapes max-shapes-per-page]} (compute-shape-stats pages-index)
n-pages (count (:pages file-data))
n-components (reduce-kv (fn [n _ c] (if (:deleted c) n (inc n)))
0 (:components file-data))
n-linked-libs (dec (count libraries))
tokens-lib (:tokens-lib file-data)
n-tokens (if (some? tokens-lib)
(count (ctob/get-all-tokens tokens-lib))
0)]
{:num-pages n-pages
:num-shapes num-shapes
:avg-shapes-per-page (if (pos? n-pages)
(mth/round (/ num-shapes n-pages))
0)
:max-shapes-per-page max-shapes-per-page
:num-components n-components
:num-linked-libraries (max 0 n-linked-libs)
:is-library (:is-shared file)
:num-tokens n-tokens}))
(defn- emit-workspace-file-stats
[file-id team-id]
(ptk/reify ::emit-workspace-file-stats
ptk/WatchEvent
(watch [_ state _]
(let [stats (compute-file-stats state file-id)]
(rx/of (ev/event (assoc stats
::ev/name "open-workspace-file"
::ev/origin "workspace"
:file-id file-id
:team-id team-id)))))))
(defn- bundle-fetched
[{:keys [file file-id thumbnails] :as bundle}]
(ptk/reify ::bundle-fetched
@ -420,6 +475,12 @@
(rx/take 1)
(rx/map dwc/set-workspace-visited))
;; Emit audit event with file statistics once all libraries are resolved
(->> stream
(rx/filter (ptk/type? ::all-libraries-resolved))
(rx/take 1)
(rx/map #(emit-workspace-file-stats file-id team-id)))
(when-let [component-id (some-> rparams :component-id uuid/parse)]
(->> stream
(rx/filter (ptk/type? ::workspace-initialized))

87
frontend/test/frontend_tests/data/workspace_stats_test.cljs Normal file
View File

@ -0,0 +1,87 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC Sucursal en España SL
(ns frontend-tests.data.workspace-stats-test
(:require
[app.common.test-helpers.files :as cthf]
[app.common.test-helpers.ids-map :as cthi]
[app.common.types.tokens-lib :as ctob]
[app.main.data.workspace :as dw]
[cljs.test :as t :include-macros true]
[frontend-tests.helpers.state :as ths]))
(t/use-fixtures :each
{:before cthi/reset-idmap!})
;; ---------------------------------------------------------------------------
;; Test compute-file-stats with various edge cases
;; ---------------------------------------------------------------------------
(t/deftest compute-file-stats-empty-file
(t/testing "empty file with no pages"
(let [file (cthf/sample-file :file1 :page-label :page1)
store (ths/setup-store file)
state @store
file-id (:id file)
stats (dw/compute-file-stats state file-id)]
(t/is (= (:num-pages stats) 1))
(t/is (>= (:num-shapes stats) 0))
(t/is (>= (:avg-shapes-per-page stats) 0))
(t/is (>= (:max-shapes-per-page stats) 0))
(t/is (>= (:num-components stats) 0))
(t/is (>= (:num-linked-libraries stats) 0))
(t/is (boolean? (:is-library stats)))
(t/is (>= (:num-tokens stats) 0)))))
(t/deftest compute-file-stats-with-shapes
(t/testing "file with shapes"
(let [file (-> (cthf/sample-file :file1 :page-label :page1)
(cthf/add-sample-shape :shape1)
(cthf/add-sample-shape :shape2))
store (ths/setup-store file)
state @store
file-id (:id file)
stats (dw/compute-file-stats state file-id)]
(t/is (= (:num-pages stats) 1))
(t/is (>= (:num-shapes stats) 2))
(t/is (>= (:avg-shapes-per-page stats) 2))
(t/is (>= (:max-shapes-per-page stats) 2)))))
(t/deftest compute-file-stats-no-tokens
(t/testing "file with no tokens lib"
(let [file (cthf/sample-file :file1 :page-label :page1)
store (ths/setup-store file)
state @store
file-id (:id file)
stats (dw/compute-file-stats state file-id)]
(t/is (= (:num-tokens stats) 0)))))
(t/deftest compute-file-stats-with-tokens
(t/testing "file with tokens"
(let [tokens-lib (-> (ctob/make-tokens-lib)
(ctob/add-set {:name "global"
:description "Global tokens"
:tokens [{:name "color.primary"
:type :color
:value "#000000"}]}))
file (-> (cthf/sample-file :file1 :page-label :page1)
(assoc-in [:data :tokens-lib] tokens-lib))
store (ths/setup-store file)
state @store
file-id (:id file)
stats (dw/compute-file-stats state file-id)]
(t/is (= (:num-tokens stats) 1)))))
(t/deftest compute-file-stats-multiple-pages
(t/testing "file with multiple pages"
(let [file (-> (cthf/sample-file :file1 :page-label :page1)
(cthf/add-sample-page :page2)
(cthf/add-sample-page :page3))
store (ths/setup-store file)
state @store
file-id (:id file)
stats (dw/compute-file-stats state file-id)]
(t/is (= (:num-pages stats) 3)))))