From 5a7e3d5de575720edf0e36de50dd4ba1c70225a0 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 6 Aug 2026 07:57:05 +0000 Subject: [PATCH] :sparkles: Persist binfile manifest and emit workspace audit events (#11106) Persist binfile manifest metadata in file_data on import so file statistics are available at open-workspace time. Emit a new open-workspace-file audit event enriched with file statistics: page count, shape count, component count, linked libraries, design tokens, and whether the file is a shared library. Closes #11106 AI-assisted-by: mimo-v2.5-pro --- backend/src/app/binfile/common.clj | 1 + backend/src/app/binfile/v3.clj | 8 +- backend/src/app/config.clj | 2 +- backend/src/app/features/fdata.clj | 20 ++--- backend/test/backend_tests/binfile_test.clj | 23 +++++ common/src/app/common/types/file.cljc | 10 +++ frontend/src/app/main/data/workspace.cljs | 61 +++++++++++++ .../data/workspace_stats_test.cljs | 87 +++++++++++++++++++ 8 files changed, 196 insertions(+), 16 deletions(-) create mode 100644 frontend/test/frontend_tests/data/workspace_stats_test.cljs diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index f984a98550..3e4402be92 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -723,6 +723,7 @@ (-> (select-keys file file-attrs) (assoc :data nil) (dissoc :team-id) + (dissoc :metadata) (dissoc :migrations))) (defn- file->file-data-params diff --git a/backend/src/app/binfile/v3.clj b/backend/src/app/binfile/v3.clj index 952cb69e8f..eab49e1eb3 100644 --- a/backend/src/app/binfile/v3.clj +++ b/backend/src/app/binfile/v3.clj @@ -392,7 +392,7 @@ params {:type "penpot/export-files" :version 1 :generated-by (str "penpot/" (:full cf/version)) - :refer "penpot" + :referer "penpot" :files (vec (vals files)) :relations rels}] (write-entry! output "manifest.json" params)))) @@ -734,7 +734,7 @@ :plugin-data plugin-data})) (defn- import-file - [{:keys [::db/conn ::bfc/project-id] :as cfg} {file-id :id file-name :name}] + [{:keys [::db/conn ::bfc/project-id ::manifest] :as cfg} {file-id :id file-name :name}] (let [file-id' (bfc/lookup-index file-id) file (read-file cfg file-id) media (read-file-media cfg file-id) @@ -801,8 +801,10 @@ (assoc :data data) (assoc :name file-name) (assoc :project-id project-id) + (assoc :metadata (d/without-nils + {:generated-by (get manifest :generated-by) + :referer (or (get manifest :referer) (get manifest :refer))})) (dissoc :options)) - file (bfc/process-file cfg file) file (ctf/check-file file)] diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index bebd5db826..f02136b1ca 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -52,7 +52,7 @@ :redis-uri "redis://redis/0" - :file-data-backend "legacy-db" + :file-data-backend "db" :objects-storage-backend "fs" :objects-storage-fs-directory "assets" diff --git a/backend/src/app/features/fdata.clj b/backend/src/app/features/fdata.clj index 412ca223cf..8e7ff9d978 100644 --- a/backend/src/app/features/fdata.clj +++ b/backend/src/app/features/fdata.clj @@ -12,6 +12,7 @@ [app.common.logging :as l] [app.common.schema :as sm] [app.common.time :as ct] + [app.common.types.file :as ctf] [app.common.types.objects-map :as omap] [app.config :as cf] [app.db :as db] @@ -159,15 +160,17 @@ :content-type "application/octet-stream" :file-id file-id :id id}) - metadata {:storage-ref-id (:id sobject)} + metadata (-> (:metadata params) + (assoc :storage-ref-id (:id sobject))) params (-> params (assoc :metadata metadata) (assoc :data nil))] (upsert-in-database cfg params)) (= backend "db") - (->> (dissoc params :metadata) - (upsert-in-database cfg)) + (let [metadata (dissoc (:metadata params) :storage-ref-id) + params (assoc params :metadata metadata)] + (upsert-in-database cfg params)) (= backend "legacy-db") (cond @@ -213,18 +216,11 @@ [backend] (or backend (cf/get :file-data-backend))) -(def ^:private schema:metadata - [:map {:title "Metadata"} - [:storage-ref-id {:optional true} ::sm/uuid]]) - -(def decode-metadata-with-schema - (sm/decoder schema:metadata sm/json-transformer)) - (defn decode-metadata [metadata] (some-> metadata (db/decode-json-pgobject) - (decode-metadata-with-schema))) + (ctf/decode-file-metadata))) (def ^:private schema:update-params [:map {:closed true} @@ -232,7 +228,7 @@ [:type [:enum "main" "snapshot" "fragment"]] [:file-id ::sm/uuid] [:backend {:optional true} [:enum "db" "legacy-db" "storage"]] - [:metadata {:optional true} [:maybe schema:metadata]] + [:metadata {:optional true} ctf/schema:file-metadata] [:data {:optional true} bytes?] [:created-at {:optional true} ::ct/inst] [:modified-at {:optional true} [:maybe ::ct/inst]] diff --git a/backend/test/backend_tests/binfile_test.clj b/backend/test/backend_tests/binfile_test.clj index 05f1525c5e..c0e45d5429 100644 --- a/backend/test/backend_tests/binfile_test.clj +++ b/backend/test/backend_tests/binfile_test.clj @@ -207,6 +207,29 @@ (t/is (= (count result) 1)) (t/is (every? uuid? result))))) +(t/deftest import-binfile-v3-persists-manifest-metadata + (let [profile (th/create-profile* 1) + file (prepare-simple-file profile) + output (tmp/tempfile :suffix ".zip")] + + (v3/export-files! + (-> th/*system* + (assoc ::bfc/ids #{(:id file)}) + (assoc ::bfc/embed-assets false) + (assoc ::bfc/include-libraries false)) + (io/output-stream output)) + + (let [result (-> th/*system* + (assoc ::bfc/project-id (:default-project-id profile)) + (assoc ::bfc/profile-id (:id profile)) + (assoc ::bfc/input output) + (v3/import-files!)) + imported (bfc/get-file th/*system* (first result))] + + (t/is (= (count result) 1)) + (t/is (some? (get-in imported [:metadata :generated-by]))) + (t/is (= "penpot" (get-in imported [:metadata :referer])))))) + (t/deftest read-obj-rejects-oversized-buffer ;; N1-07: read-obj! must reject objects exceeding max-object-size ;; before attempting to allocate the buffer diff --git a/common/src/app/common/types/file.cljc b/common/src/app/common/types/file.cljc index fe91dca8ad..d7e9feb676 100644 --- a/common/src/app/common/types/file.cljc +++ b/common/src/app/common/types/file.cljc @@ -88,6 +88,12 @@ [:plugin-data {:optional true} schema:plugin-data] [:tokens-lib {:optional true} schema:tokens-lib]]) +(def schema:file-metadata + [:map {:title "Metadata"} + [:storage-ref-id {:optional true} ::sm/uuid] + [:generated-by {:optional true} :string] + [:referer {:optional true} :string]]) + (def schema:file "A schema for validate a file data structure; data is optional because sometimes we want to validate file without the data." @@ -106,6 +112,7 @@ [:data {:optional true} schema:data] [:version :int] [:features ::cfeat/features] + [:metadata {:optional true} schema:file-metadata] [:migrations {:optional true} [::sm/set {:ordered true} :string]]]) @@ -123,6 +130,9 @@ (def check-file-media (sm/check-fn schema:media)) +(def decode-file-metadata + (sm/decoder schema:file-metadata sm/json-transformer)) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; INITIALIZATION ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/frontend/src/app/main/data/workspace.cljs b/frontend/src/app/main/data/workspace.cljs index cedfad1d96..cf7f8618bc 100644 --- a/frontend/src/app/main/data/workspace.cljs +++ b/frontend/src/app/main/data/workspace.cljs @@ -17,11 +17,13 @@ [app.common.geom.proportions :as gpp] [app.common.geom.shapes :as gsh] [app.common.logging :as log] + [app.common.math :as mth] [app.common.path-names :as cpn] [app.common.transit :as t] [app.common.types.component :as ctc] [app.common.types.components-list :as ctkl] [app.common.types.shape :as cts] + [app.common.types.tokens-lib :as ctob] [app.common.types.variant :as ctv] [app.common.uuid :as uuid] [app.config :as cf] @@ -266,6 +268,59 @@ (rx/map (fn [_] (mcp/init)))) (rx/empty)))))) +(defn- compute-shape-stats + "Compute shape statistics in a single pass over pages-index. + Returns {:num-shapes N :max-shapes-per-page M}" + [pages-index] + (reduce-kv + (fn [acc _page-id page] + (let [n (count (:objects page))] + (-> acc + (update :num-shapes + n) + (update :max-shapes-per-page max n)))) + {:num-shapes 0 + :max-shapes-per-page 0} + pages-index)) + +(defn compute-file-stats + "Compute file statistics. Returns a map of stats without event keys." + [state file-id] + (let [file (dsh/lookup-file state file-id) + file-data (:data file) + libraries (refs/select-libraries (:files state) file-id) + pages-index (:pages-index file-data) + {:keys [num-shapes max-shapes-per-page]} (compute-shape-stats pages-index) + n-pages (count (:pages file-data)) + n-components (reduce-kv (fn [n _ c] (if (:deleted c) n (inc n))) + 0 (:components file-data)) + n-linked-libs (dec (count libraries)) + tokens-lib (:tokens-lib file-data) + n-tokens (if (some? tokens-lib) + (count (ctob/get-all-tokens tokens-lib)) + 0)] + {:num-pages n-pages + :num-shapes num-shapes + :avg-shapes-per-page (if (pos? n-pages) + (mth/round (/ num-shapes n-pages)) + 0) + :max-shapes-per-page max-shapes-per-page + :num-components n-components + :num-linked-libraries (max 0 n-linked-libs) + :is-library (:is-shared file) + :num-tokens n-tokens})) + +(defn- emit-workspace-file-stats + [file-id team-id] + (ptk/reify ::emit-workspace-file-stats + ptk/WatchEvent + (watch [_ state _] + (let [stats (compute-file-stats state file-id)] + (rx/of (ev/event (assoc stats + ::ev/name "open-workspace-file" + ::ev/origin "workspace" + :file-id file-id + :team-id team-id))))))) + (defn- bundle-fetched [{:keys [file file-id thumbnails] :as bundle}] (ptk/reify ::bundle-fetched @@ -420,6 +475,12 @@ (rx/take 1) (rx/map dwc/set-workspace-visited)) + ;; Emit audit event with file statistics once all libraries are resolved + (->> stream + (rx/filter (ptk/type? ::all-libraries-resolved)) + (rx/take 1) + (rx/map #(emit-workspace-file-stats file-id team-id))) + (when-let [component-id (some-> rparams :component-id uuid/parse)] (->> stream (rx/filter (ptk/type? ::workspace-initialized)) diff --git a/frontend/test/frontend_tests/data/workspace_stats_test.cljs b/frontend/test/frontend_tests/data/workspace_stats_test.cljs new file mode 100644 index 0000000000..c4ac483e5f --- /dev/null +++ b/frontend/test/frontend_tests/data/workspace_stats_test.cljs @@ -0,0 +1,87 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns frontend-tests.data.workspace-stats-test + (:require + [app.common.test-helpers.files :as cthf] + [app.common.test-helpers.ids-map :as cthi] + [app.common.types.tokens-lib :as ctob] + [app.main.data.workspace :as dw] + [cljs.test :as t :include-macros true] + [frontend-tests.helpers.state :as ths])) + +(t/use-fixtures :each + {:before cthi/reset-idmap!}) + +;; --------------------------------------------------------------------------- +;; Test compute-file-stats with various edge cases +;; --------------------------------------------------------------------------- + +(t/deftest compute-file-stats-empty-file + (t/testing "empty file with no pages" + (let [file (cthf/sample-file :file1 :page-label :page1) + store (ths/setup-store file) + state @store + file-id (:id file) + stats (dw/compute-file-stats state file-id)] + (t/is (= (:num-pages stats) 1)) + (t/is (>= (:num-shapes stats) 0)) + (t/is (>= (:avg-shapes-per-page stats) 0)) + (t/is (>= (:max-shapes-per-page stats) 0)) + (t/is (>= (:num-components stats) 0)) + (t/is (>= (:num-linked-libraries stats) 0)) + (t/is (boolean? (:is-library stats))) + (t/is (>= (:num-tokens stats) 0))))) + +(t/deftest compute-file-stats-with-shapes + (t/testing "file with shapes" + (let [file (-> (cthf/sample-file :file1 :page-label :page1) + (cthf/add-sample-shape :shape1) + (cthf/add-sample-shape :shape2)) + store (ths/setup-store file) + state @store + file-id (:id file) + stats (dw/compute-file-stats state file-id)] + (t/is (= (:num-pages stats) 1)) + (t/is (>= (:num-shapes stats) 2)) + (t/is (>= (:avg-shapes-per-page stats) 2)) + (t/is (>= (:max-shapes-per-page stats) 2))))) + +(t/deftest compute-file-stats-no-tokens + (t/testing "file with no tokens lib" + (let [file (cthf/sample-file :file1 :page-label :page1) + store (ths/setup-store file) + state @store + file-id (:id file) + stats (dw/compute-file-stats state file-id)] + (t/is (= (:num-tokens stats) 0))))) + +(t/deftest compute-file-stats-with-tokens + (t/testing "file with tokens" + (let [tokens-lib (-> (ctob/make-tokens-lib) + (ctob/add-set {:name "global" + :description "Global tokens" + :tokens [{:name "color.primary" + :type :color + :value "#000000"}]})) + file (-> (cthf/sample-file :file1 :page-label :page1) + (assoc-in [:data :tokens-lib] tokens-lib)) + store (ths/setup-store file) + state @store + file-id (:id file) + stats (dw/compute-file-stats state file-id)] + (t/is (= (:num-tokens stats) 1))))) + +(t/deftest compute-file-stats-multiple-pages + (t/testing "file with multiple pages" + (let [file (-> (cthf/sample-file :file1 :page-label :page1) + (cthf/add-sample-page :page2) + (cthf/add-sample-page :page3)) + store (ths/setup-store file) + state @store + file-id (:id file) + stats (dw/compute-file-stats state file-id)] + (t/is (= (:num-pages stats) 3)))))