github/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2026-09-25 21:36:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

✨ Bound binfile import db timeout and zip entry scanning (#11584)

Browse Source 
Binfile import ran the whole import in a single transaction with
idle_in_transaction_session_timeout disabled (= 0), so a stalled
import could retain a connection pool slot indefinitely. Set a
finite 20 minutes ceiling via SET LOCAL instead (a compile-time
constant interpolated into the SQL; PostgreSQL does not accept bind
parameters on SET).

The v3 importer also located each file data by rescanning the full
zip entry collection once per manifest file and once per page,
making the cost close to quadratic on large files. Replace the
per-file regex matchers with a single classification pass that
groups entries by their raw path shape; consumers now lookup their
entries per file and page. As a deliberate tightening, the .json
suffix is matched literally: the previous regexes left the dot
unescaped, so crafted paths like files/<f>/tokensXjson or
objects/x-json matched by accident and are now ignored.

Closes #11579

AI-assisted-by: omen-alpha
This commit is contained in:
Andrey Antukh 2026-09-23 13:03:56 +02:00 committed by GitHub
parent 142f3d9de8
commit 7e1d5f3e88
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 439 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/src/app/binfile/common.clj
View File

@ -671,10 +671,16 @@
data data
library-ids))) library-ids)))
(defn disable-database-timeouts! (def ^:const import-transaction-timeout-ms
"Ceiling for binfile import transactions (20 minutes). Interpolated
directly into SQL: compile-time constant, never user input."
(* 20 60 1000))
(defn configure-database-timeouts!
[cfg] [cfg]
(let [conn (db/get-connection cfg)] (let [conn (db/get-connection cfg)]
(db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"]) (db/exec-one! conn [(str "SET LOCAL idle_in_transaction_session_timeout = "
import-transaction-timeout-ms)])
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]))) (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])))
(defn process-file (defn process-file

2
backend/src/app/binfile/v1.clj
View File

@ -454,7 +454,7 @@
(defn- read-import-v1 (defn- read-import-v1
[{:keys [::db/conn ::bfc/project-id ::bfc/profile-id ::bfc/input] :as cfg}] [{:keys [::db/conn ::bfc/project-id ::bfc/profile-id ::bfc/input] :as cfg}]
(bfc/disable-database-timeouts! cfg) (bfc/configure-database-timeouts! cfg)
(pu/with-open [input (zstd-input-stream input) (pu/with-open [input (zstd-input-stream input)
input (io/data-input-stream input)] input (io/data-input-stream input)]

188
backend/src/app/binfile/v3.clj
View File

@ -574,89 +574,85 @@
(let [manifest (json/read reader :key-fn json/read-kebab-key)] (let [manifest (json/read reader :key-fn json/read-kebab-key)]
(decode-manifest manifest))))) (decode-manifest manifest)))))
(defn- match-media-entry-fn (def ^:private file-object-entry-categories
[file-id] "Zip path segments that hold per-file object entries on the
(let [pattern (str "^files/" file-id "/media/([^/]+).json$") `files/<file-id>/<category>/<object-id>.json` shape."
pattern (re-pattern pattern)] #{"media" "colors" "components" "typographies"})
(fn [entry]
(when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
{:entry entry
:id (parse-uuid id)}))))
(defn- match-color-entry-fn (defn- index-entry-name
[file-id] "Classify a single zip entry by the raw shape of its path and
(let [pattern (str "^files/" file-id "/colors/([^/]+).json$") accumulate it on the index.
pattern (re-pattern pattern)]
(fn [entry]
(when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
{:entry entry
:id (parse-uuid id)}))))
(defn- match-component-entry-fn It replaces the per-file regex matchers with a single
[file-id] classification pass over all entries. The `.json` suffix is matched
(let [pattern (str "^files/" file-id "/components/([^/]+).json$") literally: the previous regexes left the dot unescaped, so crafted
pattern (re-pattern pattern)] paths like `files/<file-id>/tokensXjson` or `objects/x-json` matched
(fn [entry] by accident; requiring the literal suffix ignores them (legitimate
(when-let [[_ id] (re-matches pattern (zip-entry-name entry))] exports always write a literal `.json` suffix). Unknown paths are
{:entry entry ignored."
:id (parse-uuid id)})))) [index ^String name entry]
(if-not (and name (str/ends-with? name ".json"))
index
(let [base (subs name 0 (- (count name) 5))
segs (str/split base "/")
seg-n (count segs)
seg-1 (nth segs 0 nil)
seg-2 (nth segs 1 nil)
seg-3 (nth segs 2 nil)]
(defn- match-typography-entry-fn (if-not (and (pos? seg-n) (every? #(pos? (count %)) segs))
[file-id] index
(let [pattern (str "^files/" file-id "/typographies/([^/]+).json$") (cond
pattern (re-pattern pattern)] ;; objects/<object-id>.json
(fn [entry] (and (= seg-n 2) (= seg-1 "objects"))
(when-let [[_ id] (re-matches pattern (zip-entry-name entry))] (update index :objects bfc/conj-vec
{:entry entry {:entry entry :id (parse-uuid seg-2)})
:id (parse-uuid id)}))))
(defn- match-tokens-lib-entry-fn ;; files/<file-id>/tokens.json
[file-id] (and (= seg-n 3) (= seg-1 "files") (= seg-3 "tokens"))
(let [pattern (str "^files/" file-id "/tokens.json$") (update-in index [:tokens seg-2] bfc/conj-vec {:entry entry})
pattern (re-pattern pattern)]
(fn [entry]
(when-let [[_] (re-matches pattern (zip-entry-name entry))]
{:entry entry}))))
(defn- match-thumbnail-entry-fn ;; files/<file-id>/thumbnails/<tag>/<page-id>/<frame-id>.json
[file-id] (and (= seg-n 6) (= seg-1 "files") (= seg-3 "thumbnails"))
(let [pattern (str "^files/" file-id "/thumbnails/([^/]+)/([^/]+)/([^/]+).json$") (update-in index [:thumbnails seg-2] bfc/conj-vec
pattern (re-pattern pattern)] {:entry entry
(fn [entry] :tag (nth segs 3)
(when-let [[_ tag page-id frame-id] (re-matches pattern (zip-entry-name entry))] :page-id (parse-uuid (nth segs 4))
{:entry entry :frame-id (parse-uuid (nth segs 5))
:tag tag :file-id seg-2})
:page-id (parse-uuid page-id)
:frame-id (parse-uuid frame-id)
:file-id file-id}))))
(defn- match-page-entry-fn ;; files/<file-id>/pages/<page-id>.json
[file-id] (and (= seg-n 4) (= seg-1 "files") (= seg-3 "pages"))
(let [pattern (str "^files/" file-id "/pages/([^/]+).json$") (update-in index [:pages seg-2] bfc/conj-vec
pattern (re-pattern pattern)] {:entry entry :id (parse-uuid (nth segs 3))})
(fn [entry]
(when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
{:entry entry
:id (parse-uuid id)}))))
(defn- match-shape-entry-fn ;; files/<file-id>/pages/<page-id>/<shape-id>.json
[file-id page-id] (and (= seg-n 5) (= seg-1 "files") (= seg-3 "pages"))
(let [pattern (str "^files/" file-id "/pages/" page-id "/([^/]+).json$") (update-in index [:shapes seg-2 (nth segs 3)] bfc/conj-vec
pattern (re-pattern pattern)] {:entry entry
(fn [entry] :page-id (nth segs 3)
(when-let [[_ id] (re-matches pattern (zip-entry-name entry))] :id (parse-uuid (nth segs 4))})
{:entry entry
:page-id page-id
:id (parse-uuid id)}))))
(defn- match-storage-entry-fn ;; files/<file-id>/<category>/<object-id>.json
[] (and (= seg-n 4)
(let [pattern "^objects/([^/]+).json$" (= seg-1 "files")
pattern (re-pattern pattern)] (contains? file-object-entry-categories seg-3))
(fn [entry] (update-in index [(keyword seg-3) seg-2] bfc/conj-vec
(when-let [[_ id] (re-matches pattern (zip-entry-name entry))] {:entry entry :id (parse-uuid (nth segs 3))})
{:entry entry
:id (parse-uuid id)})))) :else
index)))))
(defn- index-entries
"Classify all the provided zip entries in a single pass and group
them by their path shape, so import consumers can lookup their
entries per file (and per page) instead of rescanning the whole
entry collection for every file and page."
[entries]
(reduce (fn [index entry]
(index-entry-name index (zip-entry-name entry) entry))
{}
entries))
(defn- read-entry (defn- read-entry
([cfg ^ZipFile input entry] ([cfg ^ZipFile input entry]
@ -686,8 +682,8 @@
(validate-plugin-data))))) (validate-plugin-data)))))
(defn- read-file-media (defn- read-file-media
[{:keys [::bfc/input ::entries] :as cfg} file-id] [{:keys [::bfc/input ::entries-index] :as cfg} file-id]
(->> (keep (match-media-entry-fn file-id) entries) (->> (get-in entries-index [:media (str file-id)])
(reduce (fn [result {:keys [id entry]}] (reduce (fn [result {:keys [id entry]}]
(let [object (->> (read-entry cfg input entry) (let [object (->> (read-entry cfg input entry)
(decode-media) (decode-media)
@ -706,8 +702,8 @@
(not-empty))) (not-empty)))
(defn- read-file-colors (defn- read-file-colors
[{:keys [::bfc/input ::entries] :as cfg} file-id] [{:keys [::bfc/input ::entries-index] :as cfg} file-id]
(->> (keep (match-color-entry-fn file-id) entries) (->> (get-in entries-index [:colors (str file-id)])
(reduce (fn [result {:keys [id entry]}] (reduce (fn [result {:keys [id entry]}]
(let [object (->> (read-entry cfg input entry) (let [object (->> (read-entry cfg input entry)
(decode-color) (decode-color)
@ -720,7 +716,7 @@
(not-empty))) (not-empty)))
(defn- read-file-components (defn- read-file-components
[{:keys [::bfc/input ::entries] :as cfg} file-id] [{:keys [::bfc/input ::entries-index] :as cfg} file-id]
(let [clean-component-post-decode (let [clean-component-post-decode
(fn [component] (fn [component]
(d/update-when component :objects (d/update-when component :objects
@ -738,7 +734,7 @@
objects objects
objects))))] objects))))]
(->> (keep (match-component-entry-fn file-id) entries) (->> (get-in entries-index [:components (str file-id)])
(reduce (fn [result {:keys [id entry]}] (reduce (fn [result {:keys [id entry]}]
(let [object (->> (read-entry cfg input entry) (let [object (->> (read-entry cfg input entry)
(clean-component-pre-decode) (clean-component-pre-decode)
@ -752,8 +748,8 @@
(not-empty)))) (not-empty))))
(defn- read-file-typographies (defn- read-file-typographies
[{:keys [::bfc/input ::entries] :as cfg} file-id] [{:keys [::bfc/input ::entries-index] :as cfg} file-id]
(->> (keep (match-typography-entry-fn file-id) entries) (->> (get-in entries-index [:typographies (str file-id)])
(reduce (fn [result {:keys [id entry]}] (reduce (fn [result {:keys [id entry]}]
(let [object (->> (read-entry cfg input entry) (let [object (->> (read-entry cfg input entry)
(decode-typography) (decode-typography)
@ -766,16 +762,16 @@
(not-empty))) (not-empty)))
(defn- read-file-tokens-lib (defn- read-file-tokens-lib
[{:keys [::bfc/input ::entries] :as cfg} file-id] [{:keys [::bfc/input ::entries-index] :as cfg} file-id]
(when-let [entry (d/seek (match-tokens-lib-entry-fn file-id) entries)] (when-let [{:keys [entry]} (first (get-in entries-index [:tokens (str file-id)]))]
(events/tap :progress {:section :tokens-lib :file-id file-id}) (events/tap :progress {:section :tokens-lib :file-id file-id})
(->> (read-entry cfg input entry nil) (->> (read-entry cfg input entry nil)
(decode-tokens-lib) (decode-tokens-lib)
(validate-tokens-lib)))) (validate-tokens-lib))))
(defn- read-file-shapes (defn- read-file-shapes
[{:keys [::bfc/input ::entries] :as cfg} file-id page-id] [{:keys [::bfc/input ::entries-index] :as cfg} file-id page-id]
(->> (keep (match-shape-entry-fn file-id page-id) entries) (->> (get-in entries-index [:shapes (str file-id) (str page-id)])
(reduce (fn [result {:keys [id entry]}] (reduce (fn [result {:keys [id entry]}]
(let [object (->> (read-entry cfg input entry) (let [object (->> (read-entry cfg input entry)
(bfl/clean-shape-pre-decode) (bfl/clean-shape-pre-decode)
@ -788,8 +784,8 @@
(not-empty))) (not-empty)))
(defn- read-file-pages (defn- read-file-pages
[{:keys [::bfc/input ::entries] :as cfg} file-id] [{:keys [::bfc/input ::entries-index] :as cfg} file-id]
(->> (keep (match-page-entry-fn file-id) entries) (->> (get-in entries-index [:pages (str file-id)])
(keep (fn [{:keys [id entry]}] (keep (fn [{:keys [id entry]}]
(let [page (->> (read-entry cfg input entry) (let [page (->> (read-entry cfg input entry)
(decode-page)) (decode-page))
@ -804,8 +800,8 @@
(d/ordered-map)))) (d/ordered-map))))
(defn- read-file-thumbnails (defn- read-file-thumbnails
[{:keys [::bfc/input ::entries] :as cfg} file-id] [{:keys [::bfc/input ::entries-index] :as cfg} file-id]
(->> (keep (match-thumbnail-entry-fn file-id) entries) (->> (get-in entries-index [:thumbnails (str file-id)])
(reduce (fn [result {:keys [page-id frame-id tag entry]}] (reduce (fn [result {:keys [page-id frame-id tag entry]}]
(let [object (->> (read-entry cfg input entry) (let [object (->> (read-entry cfg input entry)
(decode-file-thumbnail) (decode-file-thumbnail)
@ -934,7 +930,7 @@
(bfc/upsert-file-library-sync! conn (assoc rel-params :synced-at timestamp))))))) (bfc/upsert-file-library-sync! conn (assoc rel-params :synced-at timestamp)))))))
(defn- import-storage-objects (defn- import-storage-objects
[{:keys [::bfc/input ::entries ::bfc/timestamp] :as cfg}] [{:keys [::bfc/input ::entries-index ::bfc/timestamp] :as cfg}]
(events/tap :progress {:section :storage-objects}) (events/tap :progress {:section :storage-objects})
;; IMPORTANT: we strongly do not reuse the main connection that can ;; IMPORTANT: we strongly do not reuse the main connection that can
@ -945,7 +941,7 @@
;; what the storage subsystem registers in other parallel ;; what the storage subsystem registers in other parallel
;; transaction ;; transaction
(let [storage (sto/resolve cfg) (let [storage (sto/resolve cfg)
entries (keep (match-storage-entry-fn) entries)] entries (:objects entries-index)]
(doseq [{:keys [id entry]} entries] (doseq [{:keys [id entry]} entries]
(let [object (-> (read-entry cfg input entry) (let [object (-> (read-entry cfg input entry)
@ -1100,7 +1096,7 @@
(defn- import-files* (defn- import-files*
[{:keys [::manifest] :as cfg}] [{:keys [::manifest] :as cfg}]
(bfc/disable-database-timeouts! cfg) (bfc/configure-database-timeouts! cfg)
(vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id) (vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id)
@ -1168,7 +1164,7 @@
:hint "unable to perform in-place update with binfile containing more than 1 file" :hint "unable to perform in-place update with binfile containing more than 1 file"
:manifest manifest)) :manifest manifest))
(bfc/disable-database-timeouts! cfg) (bfc/configure-database-timeouts! cfg)
(let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true) (let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true)
file (first (get manifest :files)) file (first (get manifest :files))
@ -1212,7 +1208,7 @@
:found (count entries))) :found (count entries)))
cfg (-> cfg cfg (-> cfg
(assoc ::entries entries) (assoc ::entries-index (index-entries entries))
(assoc ::manifest manifest) (assoc ::manifest manifest)
(assoc ::bfc/timestamp timestamp))] (assoc ::bfc/timestamp timestamp))]

425
backend/test/backend_tests/binfile_test.clj
View File

@ -17,6 +17,7 @@
[app.common.thumbnails :as thc] [app.common.thumbnails :as thc]
[app.common.time :as ct] [app.common.time :as ct]
[app.common.types.shape :as cts] [app.common.types.shape :as cts]
[app.common.types.tokens-lib :as ctob]
[app.common.uuid :as uuid] [app.common.uuid :as uuid]
[app.config :as cf] [app.config :as cf]
[app.db :as db] [app.db :as db]
@ -61,46 +62,47 @@
(:result out))) (:result out)))
(defn- prepare-simple-file (defn- prepare-simple-file
[profile] ([profile] (prepare-simple-file profile 1))
(let [page-id-1 (uuid/custom 1 1) ([profile idx]
page-id-2 (uuid/custom 1 2) (let [page-id-1 (uuid/custom 1 1)
shape-id (uuid/custom 2 1) page-id-2 (uuid/custom 1 2)
file (th/create-file* 1 {:profile-id (:id profile) shape-id (uuid/custom 2 1)
:project-id (:default-project-id profile) file (th/create-file* idx {:profile-id (:id profile)
:is-shared false})] :project-id (:default-project-id profile)
(update-file! :is-shared false})]
:file-id (:id file) (update-file!
:profile-id (:id profile) :file-id (:id file)
:revn 0 :profile-id (:id profile)
:vern 0 :revn 0
:changes :vern 0
[{:type :add-page :changes
:name "test 1" [{:type :add-page
:id page-id-1} :name "test 1"
{:type :add-page :id page-id-1}
:name "test 2" {:type :add-page
:id page-id-2}]) :name "test 2"
:id page-id-2}])
(update-file! (update-file!
:file-id (:id file) :file-id (:id file)
:profile-id (:id profile) :profile-id (:id profile)
:revn 0 :revn 0
:vern 0 :vern 0
:changes :changes
[{:type :add-obj [{:type :add-obj
:page-id page-id-1 :page-id page-id-1
:id shape-id :id shape-id
:parent-id uuid/zero :parent-id uuid/zero
:frame-id uuid/zero :frame-id uuid/zero
:components-v2 true :components-v2 true
:obj (cts/setup-shape :obj (cts/setup-shape
{:id shape-id {:id shape-id
:name "image" :name "image"
:frame-id uuid/zero :frame-id uuid/zero
:parent-id uuid/zero :parent-id uuid/zero
:type :rect})}]) :type :rect})}])
(dissoc file :data))) (dissoc file :data))))
(def ^:private svg-raw-page-id (uuid/custom 1 1)) (def ^:private svg-raw-page-id (uuid/custom 1 1))
(def ^:private svg-raw-root-id (uuid/custom 3 1)) (def ^:private svg-raw-root-id (uuid/custom 3 1))
@ -109,58 +111,59 @@
(defn- prepare-svg-raw-file (defn- prepare-svg-raw-file
"A file containing an svg-raw subtree (an svg-raw parent with an "A file containing an svg-raw subtree (an svg-raw parent with an
svg-raw child), which is what importing an SVG produces." svg-raw child), which is what importing an SVG produces."
[profile] ([profile] (prepare-svg-raw-file profile 1))
(let [page-id svg-raw-page-id ([profile idx]
root-id svg-raw-root-id (let [page-id svg-raw-page-id
child-id svg-raw-child-id root-id svg-raw-root-id
child-id svg-raw-child-id
file (th/create-file* 1 {:profile-id (:id profile) file (th/create-file* idx {:profile-id (:id profile)
:project-id (:default-project-id profile) :project-id (:default-project-id profile)
:is-shared false})] :is-shared false})]
(update-file! (update-file!
:file-id (:id file) :file-id (:id file)
:profile-id (:id profile) :profile-id (:id profile)
:revn 0 :revn 0
:vern 0 :vern 0
:changes :changes
[{:type :add-page [{:type :add-page
:name "page 1" :name "page 1"
:id page-id}]) :id page-id}])
(update-file! (update-file!
:file-id (:id file) :file-id (:id file)
:profile-id (:id profile) :profile-id (:id profile)
:revn 0 :revn 0
:vern 0 :vern 0
:changes :changes
[{:type :add-obj [{:type :add-obj
:page-id page-id :page-id page-id
:id root-id :id root-id
:parent-id uuid/zero :parent-id uuid/zero
:frame-id uuid/zero :frame-id uuid/zero
:components-v2 true :components-v2 true
:obj (cts/setup-shape :obj (cts/setup-shape
{:id root-id {:id root-id
:name "svg-root" :name "svg-root"
:frame-id uuid/zero :frame-id uuid/zero
:parent-id uuid/zero :parent-id uuid/zero
:type :svg-raw :type :svg-raw
:content {:tag :svg :attrs {} :content []}})} :content {:tag :svg :attrs {} :content []}})}
{:type :add-obj {:type :add-obj
:page-id page-id :page-id page-id
:id child-id :id child-id
:parent-id root-id :parent-id root-id
:frame-id uuid/zero :frame-id uuid/zero
:components-v2 true :components-v2 true
:obj (cts/setup-shape :obj (cts/setup-shape
{:id child-id {:id child-id
:name "svg-text" :name "svg-text"
:frame-id uuid/zero :frame-id uuid/zero
:parent-id root-id :parent-id root-id
:type :svg-raw :type :svg-raw
:content {:tag :text :attrs {} :content []}})}]) :content {:tag :text :attrs {} :content []}})}])
(dissoc file :data))) (dissoc file :data))))
(t/deftest import-binfile-v3-preserves-svg-raw-children (t/deftest import-binfile-v3-preserves-svg-raw-children
(let [profile (th/create-profile* 1) (let [profile (th/create-profile* 1)
@ -2174,3 +2177,251 @@
(ex-data e)))] (ex-data e)))]
(t/is (= :validation (:type out))) (t/is (= :validation (:type out)))
(t/is (= :max-file-size-reached (:code out)))))) (t/is (= :max-file-size-reached (:code out))))))
(t/deftest import-configures-finite-transaction-idle-timeout
;; The binfile import transaction must set a finite ceiling for
;; idle_in_transaction_session_timeout (20 min) instead of disabling
;; it entirely: a stalled import must not retain a pool connection
;; without any upper bound. The ceiling must be scoped with SET
;; LOCAL: once the transaction ends, the pool session default must
;; be restored for subsequent transactions.
(let [pg-setting-sql ["SELECT setting FROM pg_settings WHERE name = 'idle_in_transaction_session_timeout'"]
in-tx (db/tx-run! th/*system*
(fn [cfg]
(bfc/configure-database-timeouts! cfg)
(:setting (db/exec-one! cfg pg-setting-sql))))
;; SET LOCAL must not leak past the transaction boundary
after (db/tx-run! th/*system*
(fn [cfg]
(:setting (db/exec-one! cfg pg-setting-sql))))]
(t/is (= "1200000" in-tx))
(t/is (= "300000" after))))
(def ^:private index-test-file-id "22222222-2222-2222-2222-222222222222")
(def ^:private index-test-page-id "44444444-4444-4444-4444-444444444444")
(defn- index-entries-of
[names]
(@#'v3/index-entries (map #(java.util.zip.ZipEntry. %) names)))
(t/deftest index-entries-classifies-entry-names
(let [f index-test-file-id
page index-test-page-id
index (index-entries-of
["manifest.json"
(str "files/" f ".json")
(str "files/" f "/plugin-data.json")
(str "files/" f "/tokens.json")
"objects/11111111-1111-1111-1111-111111111111.json"
(str "files/" f "/media/33333333-3333-3333-3333-333333333333.json")
(str "files/" f "/colors/not-a-uuid.json")
(str "files/" f "/components/comp.json")
(str "files/" f "/typographies/t.json")
(str "files/" f "/pages/" page ".json")
(str "files/" f "/pages/" page "/shape.json")
(str "files/" f "/thumbnails/medium/" page "/frame.json")])]
(t/is (= [(parse-uuid "11111111-1111-1111-1111-111111111111")]
(mapv :id (:objects index))))
(t/is (= #{f} (set (keys (:media index)))))
(t/is (= [(parse-uuid "33333333-3333-3333-3333-333333333333")]
(mapv :id (get (:media index) f))))
(t/is (= #{f} (set (keys (:colors index)))))
;; non-uuid ids are preserved as nil, same as the matchers do today
(t/is (= [nil] (mapv :id (get (:colors index) f))))
(t/is (= #{f} (set (keys (:components index)))))
(t/is (= #{f} (set (keys (:typographies index)))))
(t/is (= #{f} (set (keys (:pages index)))))
(t/is (= [(parse-uuid page)] (mapv :id (get (:pages index) f))))
(t/is (= #{f} (set (keys (:shapes index)))))
(t/is (= #{page} (set (keys (get (:shapes index) f)))))
(t/is (= [nil] (mapv :id (get (get (:shapes index) f) page))))
(t/is (= #{f} (set (keys (:thumbnails index)))))
(let [thumb (first (get (:thumbnails index) f))]
(t/is (= "medium" (:tag thumb)))
(t/is (= (parse-uuid page) (:page-id thumb)))
(t/is (= (parse-uuid "frame") (:frame-id thumb)))
(t/is (= f (:file-id thumb))))
(t/is (= #{f} (set (keys (:tokens index)))))))
(t/deftest index-entries-ignores-unknown-entry-names
;; unknown paths, wrong depth, wrong suffix or non-uuid file segments
;; are all ignored, same as today's anchored regexes
(t/is (empty? (index-entries-of ["manifest.json"])))
(t/is (empty? (index-entries-of [(str "files/" index-test-file-id ".json")])))
(t/is (empty? (index-entries-of [(str "files/" index-test-file-id "/plugin-data.json")])))
(t/is (empty? (index-entries-of ["objects/a/b.json"])))
(t/is (empty? (index-entries-of [(str "files/" index-test-file-id "/media/a/b.json")])))
(t/is (empty? (index-entries-of [(str "files/" index-test-file-id "/media/a.txt")])))
(t/is (empty? (index-entries-of [(str "files/" index-test-file-id "/unknown/a.json")])))
(t/is (empty? (index-entries-of [(str "files//media/a.json")]))))
(t/deftest index-entries-requires-literal-json-suffix
;; deliberate tightening vs today's regexes: the dot is unescaped in
;; `([^/]+).json$` / `tokens.json$`, so today `tokensXjson` and
;; `objects/x-json` DO match; the classifier requires a literal
;; `.json` suffix and ignores them
(t/is (empty? (index-entries-of [(str "files/" index-test-file-id "/tokensXjson")])))
(t/is (empty? (index-entries-of ["objects/x-json"]))))
(t/deftest import-binfile-v3-multiple-files-preserves-per-file-content
;; the entries index must attribute every zip entry to the file it
;; belongs to: with several files in the same manifest, pages and
;; shapes of one file must not leak into another. The file also
;; carries library content (color, typography, component and tokens
;; set) so every rewired consumer is exercised on its present path.
(let [profile (th/create-profile* 1)
simple (prepare-simple-file profile)
color-id (uuid/custom 5 1)
typo-id (uuid/custom 5 2)
comp-id (uuid/custom 5 3)
svg (prepare-svg-raw-file profile 2)
output (tmp/tempfile :suffix ".zip")]
(update-file!
:file-id (:id simple)
:profile-id (:id profile)
:revn 0
:vern 0
:changes
[{:type :add-color
:color {:id color-id
:name "import-color"
:color "#FF0000"}}
{:type :add-typography
:typography {:id typo-id
:name "import-typography"
:font-id "source-sans-pro"
:font-family "Source Sans Pro"
:font-variant-id "regular"
:font-size "16"
:font-weight "400"
:font-style "normal"
:line-height "1.4"
:letter-spacing "0"
:text-transform "none"}}
{:type :add-component
:id comp-id
:name "import-component"
:path ""
:main-instance-id (uuid/custom 2 1)
:main-instance-page (uuid/custom 1 1)}
{:type :set-tokens-lib
:tokens-lib (-> (ctob/make-tokens-lib)
(ctob/add-set (ctob/make-token-set :name "ImportSet")))}])
(v3/export-files!
(-> th/*system*
(assoc ::bfc/ids #{(:id simple) (:id svg)})
(assoc ::bfc/embed-assets false)
(assoc ::bfc/include-libraries false))
(io/output-stream output))
;; import returns the imported file ids plus the library link
;; resolution; the test only needs the ids
(let [result (:file-ids (-> th/*system*
(assoc ::bfc/project-id (:default-project-id profile))
(assoc ::bfc/profile-id (:id profile))
(assoc ::bfc/input output)
(v3/import-files!)))
files (map #(bfc/get-file th/*system* %) result)
svg-imported (some #(when (contains? (get-in % [:data :pages-index
svg-raw-page-id
:objects])
svg-raw-root-id)
%)
files)
simple-imported (some #(when (contains? (get-in % [:data :pages-index
(uuid/custom 1 1)
:objects])
(uuid/custom 2 1))
%)
files)]
(t/is (= 2 (count result)))
(t/is (= 2 (count (distinct result))))
(t/is (some? svg-imported))
(t/is (some? simple-imported))
;; the svg-raw file keeps its subtree on its own page (plus the
;; default page created by create-file)
(t/is (= [svg-raw-child-id]
(get-in svg-imported [:data :pages-index svg-raw-page-id
:objects svg-raw-root-id :shapes])))
(t/is (= 2 (count (get-in svg-imported [:data :pages-index]))))
;; the simple file keeps its default page plus its two pages and
;; its shape
(t/is (= 3 (count (get-in simple-imported [:data :pages-index]))))
(t/is (contains? (get-in simple-imported
[:data :pages-index (uuid/custom 1 1) :objects])
(uuid/custom 2 1)))
;; library content is restored on its present path: the
;; consumers' get-in keys must match the classifier buckets
(t/is (some? (get-in simple-imported [:data :colors color-id])))
(t/is (some? (get-in simple-imported [:data :typographies typo-id])))
(t/is (some? (get-in simple-imported [:data :components comp-id])))
(t/is (= ["ImportSet"]
(vec (ctob/get-set-names
(get-in simple-imported [:data :tokens-lib]))))))))
(t/deftest import-binfile-v3-restores-media-objects
;; storage objects, per-file media entries and object thumbnails
;; are classified through the entries index and restored end to end
(let [profile (th/create-profile* 1)
file (prepare-file-with-media profile)
thumb-page-id (uuid/custom 1 1)
thumb-frame-id (uuid/custom 6 1)
thumb-tag "medium"
output (tmp/tempfile :suffix ".zip")]
;; a thumbnail row backed by the same storage object as the media
;; object, so the export produces a thumbnails/ zip entry
(let [mobj (th/db-get :file-media-object {:file-id (:id file)})]
(db/insert! th/*system* :file-tagged-object-thumbnail
{:file-id (:id file)
:tag thumb-tag
:object-id (thc/fmt-object-id {:file-id (:id file)
:page-id thumb-page-id
:frame-id thumb-frame-id
:tag thumb-tag})
:media-id (:media-id mobj)}))
(v3/export-files!
(-> th/*system*
(assoc ::bfc/ids #{(:id file)})
(assoc ::bfc/embed-assets false)
(assoc ::bfc/include-libraries false))
(io/output-stream output))
;; import returns the imported file ids plus the library link
;; resolution; the test only needs the ids
(let [result (:file-ids (-> th/*system*
(assoc ::bfc/project-id (:default-project-id profile))
(assoc ::bfc/profile-id (:id profile))
(assoc ::bfc/input output)
(v3/import-files!)))
mobjs (db/query th/*system* :file-media-object
{:file-id (first result)})
thumbs (db/query th/*system* :file-tagged-object-thumbnail
{:file-id (first result)})]
(t/is (= 1 (count result)))
(t/is (pos? (count mobjs)))
(t/is (every? some? (map :media-id mobjs)))
;; the thumbnail is restored with its object-id rebuilt around
;; the new file id and a media-id that resolves to storage
(t/is (= 1 (count thumbs)))
(let [thumb (first thumbs)]
(t/is (= thumb-tag (:tag thumb)))
(t/is (= (str (first result) "/" thumb-page-id "/" thumb-frame-id "/" thumb-tag)
(:object-id thumb)))
(t/is (some? (:media-id thumb)))))))