diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index 701112a51c..2909e35815 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -671,10 +671,16 @@ data library-ids))) -(defn disable-database-timeouts! +(def ^:const import-transaction-timeout-ms + "Ceiling for binfile import transactions (20 minutes). Interpolated + directly into SQL: compile-time constant, never user input." + (* 20 60 1000)) + +(defn configure-database-timeouts! [cfg] (let [conn (db/get-connection cfg)] - (db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"]) + (db/exec-one! conn [(str "SET LOCAL idle_in_transaction_session_timeout = " + import-transaction-timeout-ms)]) (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]))) (defn process-file diff --git a/backend/src/app/binfile/v1.clj b/backend/src/app/binfile/v1.clj index 23d7b8795c..d5716f1920 100644 --- a/backend/src/app/binfile/v1.clj +++ b/backend/src/app/binfile/v1.clj @@ -454,7 +454,7 @@ (defn- read-import-v1 [{:keys [::db/conn ::bfc/project-id ::bfc/profile-id ::bfc/input] :as cfg}] - (bfc/disable-database-timeouts! cfg) + (bfc/configure-database-timeouts! cfg) (pu/with-open [input (zstd-input-stream input) input (io/data-input-stream input)] diff --git a/backend/src/app/binfile/v3.clj b/backend/src/app/binfile/v3.clj index 7f370ccac9..16492ad59b 100644 --- a/backend/src/app/binfile/v3.clj +++ b/backend/src/app/binfile/v3.clj @@ -574,89 +574,85 @@ (let [manifest (json/read reader :key-fn json/read-kebab-key)] (decode-manifest manifest))))) -(defn- match-media-entry-fn - [file-id] - (let [pattern (str "^files/" file-id "/media/([^/]+).json$") - pattern (re-pattern pattern)] - (fn [entry] - (when-let [[_ id] (re-matches pattern (zip-entry-name entry))] - {:entry entry - :id (parse-uuid id)})))) +(def ^:private file-object-entry-categories + "Zip path segments that hold per-file object entries on the + `files///.json` shape." + #{"media" "colors" "components" "typographies"}) -(defn- match-color-entry-fn - [file-id] - (let [pattern (str "^files/" file-id "/colors/([^/]+).json$") - pattern (re-pattern pattern)] - (fn [entry] - (when-let [[_ id] (re-matches pattern (zip-entry-name entry))] - {:entry entry - :id (parse-uuid id)})))) +(defn- index-entry-name + "Classify a single zip entry by the raw shape of its path and + accumulate it on the index. -(defn- match-component-entry-fn - [file-id] - (let [pattern (str "^files/" file-id "/components/([^/]+).json$") - pattern (re-pattern pattern)] - (fn [entry] - (when-let [[_ id] (re-matches pattern (zip-entry-name entry))] - {:entry entry - :id (parse-uuid id)})))) + It replaces the per-file regex matchers with a single + classification pass over all entries. The `.json` suffix is matched + literally: the previous regexes left the dot unescaped, so crafted + paths like `files//tokensXjson` or `objects/x-json` matched + by accident; requiring the literal suffix ignores them (legitimate + exports always write a literal `.json` suffix). Unknown paths are + ignored." + [index ^String name entry] + (if-not (and name (str/ends-with? name ".json")) + index + (let [base (subs name 0 (- (count name) 5)) + segs (str/split base "/") + seg-n (count segs) + seg-1 (nth segs 0 nil) + seg-2 (nth segs 1 nil) + seg-3 (nth segs 2 nil)] -(defn- match-typography-entry-fn - [file-id] - (let [pattern (str "^files/" file-id "/typographies/([^/]+).json$") - pattern (re-pattern pattern)] - (fn [entry] - (when-let [[_ id] (re-matches pattern (zip-entry-name entry))] - {:entry entry - :id (parse-uuid id)})))) + (if-not (and (pos? seg-n) (every? #(pos? (count %)) segs)) + index + (cond + ;; objects/.json + (and (= seg-n 2) (= seg-1 "objects")) + (update index :objects bfc/conj-vec + {:entry entry :id (parse-uuid seg-2)}) -(defn- match-tokens-lib-entry-fn - [file-id] - (let [pattern (str "^files/" file-id "/tokens.json$") - pattern (re-pattern pattern)] - (fn [entry] - (when-let [[_] (re-matches pattern (zip-entry-name entry))] - {:entry entry})))) + ;; files//tokens.json + (and (= seg-n 3) (= seg-1 "files") (= seg-3 "tokens")) + (update-in index [:tokens seg-2] bfc/conj-vec {:entry entry}) -(defn- match-thumbnail-entry-fn - [file-id] - (let [pattern (str "^files/" file-id "/thumbnails/([^/]+)/([^/]+)/([^/]+).json$") - pattern (re-pattern pattern)] - (fn [entry] - (when-let [[_ tag page-id frame-id] (re-matches pattern (zip-entry-name entry))] - {:entry entry - :tag tag - :page-id (parse-uuid page-id) - :frame-id (parse-uuid frame-id) - :file-id file-id})))) + ;; files//thumbnails///.json + (and (= seg-n 6) (= seg-1 "files") (= seg-3 "thumbnails")) + (update-in index [:thumbnails seg-2] bfc/conj-vec + {:entry entry + :tag (nth segs 3) + :page-id (parse-uuid (nth segs 4)) + :frame-id (parse-uuid (nth segs 5)) + :file-id seg-2}) -(defn- match-page-entry-fn - [file-id] - (let [pattern (str "^files/" file-id "/pages/([^/]+).json$") - pattern (re-pattern pattern)] - (fn [entry] - (when-let [[_ id] (re-matches pattern (zip-entry-name entry))] - {:entry entry - :id (parse-uuid id)})))) + ;; files//pages/.json + (and (= seg-n 4) (= seg-1 "files") (= seg-3 "pages")) + (update-in index [:pages seg-2] bfc/conj-vec + {:entry entry :id (parse-uuid (nth segs 3))}) -(defn- match-shape-entry-fn - [file-id page-id] - (let [pattern (str "^files/" file-id "/pages/" page-id "/([^/]+).json$") - pattern (re-pattern pattern)] - (fn [entry] - (when-let [[_ id] (re-matches pattern (zip-entry-name entry))] - {:entry entry - :page-id page-id - :id (parse-uuid id)})))) + ;; files//pages//.json + (and (= seg-n 5) (= seg-1 "files") (= seg-3 "pages")) + (update-in index [:shapes seg-2 (nth segs 3)] bfc/conj-vec + {:entry entry + :page-id (nth segs 3) + :id (parse-uuid (nth segs 4))}) -(defn- match-storage-entry-fn - [] - (let [pattern "^objects/([^/]+).json$" - pattern (re-pattern pattern)] - (fn [entry] - (when-let [[_ id] (re-matches pattern (zip-entry-name entry))] - {:entry entry - :id (parse-uuid id)})))) + ;; files///.json + (and (= seg-n 4) + (= seg-1 "files") + (contains? file-object-entry-categories seg-3)) + (update-in index [(keyword seg-3) seg-2] bfc/conj-vec + {:entry entry :id (parse-uuid (nth segs 3))}) + + :else + index))))) + +(defn- index-entries + "Classify all the provided zip entries in a single pass and group + them by their path shape, so import consumers can lookup their + entries per file (and per page) instead of rescanning the whole + entry collection for every file and page." + [entries] + (reduce (fn [index entry] + (index-entry-name index (zip-entry-name entry) entry)) + {} + entries)) (defn- read-entry ([cfg ^ZipFile input entry] @@ -686,8 +682,8 @@ (validate-plugin-data))))) (defn- read-file-media - [{:keys [::bfc/input ::entries] :as cfg} file-id] - (->> (keep (match-media-entry-fn file-id) entries) + [{:keys [::bfc/input ::entries-index] :as cfg} file-id] + (->> (get-in entries-index [:media (str file-id)]) (reduce (fn [result {:keys [id entry]}] (let [object (->> (read-entry cfg input entry) (decode-media) @@ -706,8 +702,8 @@ (not-empty))) (defn- read-file-colors - [{:keys [::bfc/input ::entries] :as cfg} file-id] - (->> (keep (match-color-entry-fn file-id) entries) + [{:keys [::bfc/input ::entries-index] :as cfg} file-id] + (->> (get-in entries-index [:colors (str file-id)]) (reduce (fn [result {:keys [id entry]}] (let [object (->> (read-entry cfg input entry) (decode-color) @@ -720,7 +716,7 @@ (not-empty))) (defn- read-file-components - [{:keys [::bfc/input ::entries] :as cfg} file-id] + [{:keys [::bfc/input ::entries-index] :as cfg} file-id] (let [clean-component-post-decode (fn [component] (d/update-when component :objects @@ -738,7 +734,7 @@ objects objects))))] - (->> (keep (match-component-entry-fn file-id) entries) + (->> (get-in entries-index [:components (str file-id)]) (reduce (fn [result {:keys [id entry]}] (let [object (->> (read-entry cfg input entry) (clean-component-pre-decode) @@ -752,8 +748,8 @@ (not-empty)))) (defn- read-file-typographies - [{:keys [::bfc/input ::entries] :as cfg} file-id] - (->> (keep (match-typography-entry-fn file-id) entries) + [{:keys [::bfc/input ::entries-index] :as cfg} file-id] + (->> (get-in entries-index [:typographies (str file-id)]) (reduce (fn [result {:keys [id entry]}] (let [object (->> (read-entry cfg input entry) (decode-typography) @@ -766,16 +762,16 @@ (not-empty))) (defn- read-file-tokens-lib - [{:keys [::bfc/input ::entries] :as cfg} file-id] - (when-let [entry (d/seek (match-tokens-lib-entry-fn file-id) entries)] + [{:keys [::bfc/input ::entries-index] :as cfg} file-id] + (when-let [{:keys [entry]} (first (get-in entries-index [:tokens (str file-id)]))] (events/tap :progress {:section :tokens-lib :file-id file-id}) (->> (read-entry cfg input entry nil) (decode-tokens-lib) (validate-tokens-lib)))) (defn- read-file-shapes - [{:keys [::bfc/input ::entries] :as cfg} file-id page-id] - (->> (keep (match-shape-entry-fn file-id page-id) entries) + [{:keys [::bfc/input ::entries-index] :as cfg} file-id page-id] + (->> (get-in entries-index [:shapes (str file-id) (str page-id)]) (reduce (fn [result {:keys [id entry]}] (let [object (->> (read-entry cfg input entry) (bfl/clean-shape-pre-decode) @@ -788,8 +784,8 @@ (not-empty))) (defn- read-file-pages - [{:keys [::bfc/input ::entries] :as cfg} file-id] - (->> (keep (match-page-entry-fn file-id) entries) + [{:keys [::bfc/input ::entries-index] :as cfg} file-id] + (->> (get-in entries-index [:pages (str file-id)]) (keep (fn [{:keys [id entry]}] (let [page (->> (read-entry cfg input entry) (decode-page)) @@ -804,8 +800,8 @@ (d/ordered-map)))) (defn- read-file-thumbnails - [{:keys [::bfc/input ::entries] :as cfg} file-id] - (->> (keep (match-thumbnail-entry-fn file-id) entries) + [{:keys [::bfc/input ::entries-index] :as cfg} file-id] + (->> (get-in entries-index [:thumbnails (str file-id)]) (reduce (fn [result {:keys [page-id frame-id tag entry]}] (let [object (->> (read-entry cfg input entry) (decode-file-thumbnail) @@ -934,7 +930,7 @@ (bfc/upsert-file-library-sync! conn (assoc rel-params :synced-at timestamp))))))) (defn- import-storage-objects - [{:keys [::bfc/input ::entries ::bfc/timestamp] :as cfg}] + [{:keys [::bfc/input ::entries-index ::bfc/timestamp] :as cfg}] (events/tap :progress {:section :storage-objects}) ;; IMPORTANT: we strongly do not reuse the main connection that can @@ -945,7 +941,7 @@ ;; what the storage subsystem registers in other parallel ;; transaction (let [storage (sto/resolve cfg) - entries (keep (match-storage-entry-fn) entries)] + entries (:objects entries-index)] (doseq [{:keys [id entry]} entries] (let [object (-> (read-entry cfg input entry) @@ -1100,7 +1096,7 @@ (defn- import-files* [{:keys [::manifest] :as cfg}] - (bfc/disable-database-timeouts! cfg) + (bfc/configure-database-timeouts! cfg) (vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id) @@ -1168,7 +1164,7 @@ :hint "unable to perform in-place update with binfile containing more than 1 file" :manifest manifest)) - (bfc/disable-database-timeouts! cfg) + (bfc/configure-database-timeouts! cfg) (let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true) file (first (get manifest :files)) @@ -1212,7 +1208,7 @@ :found (count entries))) cfg (-> cfg - (assoc ::entries entries) + (assoc ::entries-index (index-entries entries)) (assoc ::manifest manifest) (assoc ::bfc/timestamp timestamp))] diff --git a/backend/test/backend_tests/binfile_test.clj b/backend/test/backend_tests/binfile_test.clj index d0f86fd861..5b3afa8f93 100644 --- a/backend/test/backend_tests/binfile_test.clj +++ b/backend/test/backend_tests/binfile_test.clj @@ -17,6 +17,7 @@ [app.common.thumbnails :as thc] [app.common.time :as ct] [app.common.types.shape :as cts] + [app.common.types.tokens-lib :as ctob] [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] @@ -61,46 +62,47 @@ (:result out))) (defn- prepare-simple-file - [profile] - (let [page-id-1 (uuid/custom 1 1) - page-id-2 (uuid/custom 1 2) - shape-id (uuid/custom 2 1) - file (th/create-file* 1 {:profile-id (:id profile) - :project-id (:default-project-id profile) - :is-shared false})] - (update-file! - :file-id (:id file) - :profile-id (:id profile) - :revn 0 - :vern 0 - :changes - [{:type :add-page - :name "test 1" - :id page-id-1} - {:type :add-page - :name "test 2" - :id page-id-2}]) + ([profile] (prepare-simple-file profile 1)) + ([profile idx] + (let [page-id-1 (uuid/custom 1 1) + page-id-2 (uuid/custom 1 2) + shape-id (uuid/custom 2 1) + file (th/create-file* idx {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false})] + (update-file! + :file-id (:id file) + :profile-id (:id profile) + :revn 0 + :vern 0 + :changes + [{:type :add-page + :name "test 1" + :id page-id-1} + {:type :add-page + :name "test 2" + :id page-id-2}]) - (update-file! - :file-id (:id file) - :profile-id (:id profile) - :revn 0 - :vern 0 - :changes - [{:type :add-obj - :page-id page-id-1 - :id shape-id - :parent-id uuid/zero - :frame-id uuid/zero - :components-v2 true - :obj (cts/setup-shape - {:id shape-id - :name "image" - :frame-id uuid/zero - :parent-id uuid/zero - :type :rect})}]) + (update-file! + :file-id (:id file) + :profile-id (:id profile) + :revn 0 + :vern 0 + :changes + [{:type :add-obj + :page-id page-id-1 + :id shape-id + :parent-id uuid/zero + :frame-id uuid/zero + :components-v2 true + :obj (cts/setup-shape + {:id shape-id + :name "image" + :frame-id uuid/zero + :parent-id uuid/zero + :type :rect})}]) - (dissoc file :data))) + (dissoc file :data)))) (def ^:private svg-raw-page-id (uuid/custom 1 1)) (def ^:private svg-raw-root-id (uuid/custom 3 1)) @@ -109,58 +111,59 @@ (defn- prepare-svg-raw-file "A file containing an svg-raw subtree (an svg-raw parent with an svg-raw child), which is what importing an SVG produces." - [profile] - (let [page-id svg-raw-page-id - root-id svg-raw-root-id - child-id svg-raw-child-id + ([profile] (prepare-svg-raw-file profile 1)) + ([profile idx] + (let [page-id svg-raw-page-id + root-id svg-raw-root-id + child-id svg-raw-child-id - file (th/create-file* 1 {:profile-id (:id profile) - :project-id (:default-project-id profile) - :is-shared false})] - (update-file! - :file-id (:id file) - :profile-id (:id profile) - :revn 0 - :vern 0 - :changes - [{:type :add-page - :name "page 1" - :id page-id}]) + file (th/create-file* idx {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false})] + (update-file! + :file-id (:id file) + :profile-id (:id profile) + :revn 0 + :vern 0 + :changes + [{:type :add-page + :name "page 1" + :id page-id}]) - (update-file! - :file-id (:id file) - :profile-id (:id profile) - :revn 0 - :vern 0 - :changes - [{:type :add-obj - :page-id page-id - :id root-id - :parent-id uuid/zero - :frame-id uuid/zero - :components-v2 true - :obj (cts/setup-shape - {:id root-id - :name "svg-root" - :frame-id uuid/zero - :parent-id uuid/zero - :type :svg-raw - :content {:tag :svg :attrs {} :content []}})} - {:type :add-obj - :page-id page-id - :id child-id - :parent-id root-id - :frame-id uuid/zero - :components-v2 true - :obj (cts/setup-shape - {:id child-id - :name "svg-text" - :frame-id uuid/zero - :parent-id root-id - :type :svg-raw - :content {:tag :text :attrs {} :content []}})}]) + (update-file! + :file-id (:id file) + :profile-id (:id profile) + :revn 0 + :vern 0 + :changes + [{:type :add-obj + :page-id page-id + :id root-id + :parent-id uuid/zero + :frame-id uuid/zero + :components-v2 true + :obj (cts/setup-shape + {:id root-id + :name "svg-root" + :frame-id uuid/zero + :parent-id uuid/zero + :type :svg-raw + :content {:tag :svg :attrs {} :content []}})} + {:type :add-obj + :page-id page-id + :id child-id + :parent-id root-id + :frame-id uuid/zero + :components-v2 true + :obj (cts/setup-shape + {:id child-id + :name "svg-text" + :frame-id uuid/zero + :parent-id root-id + :type :svg-raw + :content {:tag :text :attrs {} :content []}})}]) - (dissoc file :data))) + (dissoc file :data)))) (t/deftest import-binfile-v3-preserves-svg-raw-children (let [profile (th/create-profile* 1) @@ -2174,3 +2177,251 @@ (ex-data e)))] (t/is (= :validation (:type out))) (t/is (= :max-file-size-reached (:code out)))))) + +(t/deftest import-configures-finite-transaction-idle-timeout + ;; The binfile import transaction must set a finite ceiling for + ;; idle_in_transaction_session_timeout (20 min) instead of disabling + ;; it entirely: a stalled import must not retain a pool connection + ;; without any upper bound. The ceiling must be scoped with SET + ;; LOCAL: once the transaction ends, the pool session default must + ;; be restored for subsequent transactions. + (let [pg-setting-sql ["SELECT setting FROM pg_settings WHERE name = 'idle_in_transaction_session_timeout'"] + in-tx (db/tx-run! th/*system* + (fn [cfg] + (bfc/configure-database-timeouts! cfg) + (:setting (db/exec-one! cfg pg-setting-sql)))) + ;; SET LOCAL must not leak past the transaction boundary + after (db/tx-run! th/*system* + (fn [cfg] + (:setting (db/exec-one! cfg pg-setting-sql))))] + (t/is (= "1200000" in-tx)) + (t/is (= "300000" after)))) + +(def ^:private index-test-file-id "22222222-2222-2222-2222-222222222222") +(def ^:private index-test-page-id "44444444-4444-4444-4444-444444444444") + +(defn- index-entries-of + [names] + (@#'v3/index-entries (map #(java.util.zip.ZipEntry. %) names))) + +(t/deftest index-entries-classifies-entry-names + (let [f index-test-file-id + page index-test-page-id + index (index-entries-of + ["manifest.json" + (str "files/" f ".json") + (str "files/" f "/plugin-data.json") + (str "files/" f "/tokens.json") + "objects/11111111-1111-1111-1111-111111111111.json" + (str "files/" f "/media/33333333-3333-3333-3333-333333333333.json") + (str "files/" f "/colors/not-a-uuid.json") + (str "files/" f "/components/comp.json") + (str "files/" f "/typographies/t.json") + (str "files/" f "/pages/" page ".json") + (str "files/" f "/pages/" page "/shape.json") + (str "files/" f "/thumbnails/medium/" page "/frame.json")])] + + (t/is (= [(parse-uuid "11111111-1111-1111-1111-111111111111")] + (mapv :id (:objects index)))) + + (t/is (= #{f} (set (keys (:media index))))) + (t/is (= [(parse-uuid "33333333-3333-3333-3333-333333333333")] + (mapv :id (get (:media index) f)))) + + (t/is (= #{f} (set (keys (:colors index))))) + ;; non-uuid ids are preserved as nil, same as the matchers do today + (t/is (= [nil] (mapv :id (get (:colors index) f)))) + + (t/is (= #{f} (set (keys (:components index))))) + (t/is (= #{f} (set (keys (:typographies index))))) + + (t/is (= #{f} (set (keys (:pages index))))) + (t/is (= [(parse-uuid page)] (mapv :id (get (:pages index) f)))) + + (t/is (= #{f} (set (keys (:shapes index))))) + (t/is (= #{page} (set (keys (get (:shapes index) f))))) + (t/is (= [nil] (mapv :id (get (get (:shapes index) f) page)))) + + (t/is (= #{f} (set (keys (:thumbnails index))))) + (let [thumb (first (get (:thumbnails index) f))] + (t/is (= "medium" (:tag thumb))) + (t/is (= (parse-uuid page) (:page-id thumb))) + (t/is (= (parse-uuid "frame") (:frame-id thumb))) + (t/is (= f (:file-id thumb)))) + + (t/is (= #{f} (set (keys (:tokens index))))))) + +(t/deftest index-entries-ignores-unknown-entry-names + ;; unknown paths, wrong depth, wrong suffix or non-uuid file segments + ;; are all ignored, same as today's anchored regexes + (t/is (empty? (index-entries-of ["manifest.json"]))) + (t/is (empty? (index-entries-of [(str "files/" index-test-file-id ".json")]))) + (t/is (empty? (index-entries-of [(str "files/" index-test-file-id "/plugin-data.json")]))) + (t/is (empty? (index-entries-of ["objects/a/b.json"]))) + (t/is (empty? (index-entries-of [(str "files/" index-test-file-id "/media/a/b.json")]))) + (t/is (empty? (index-entries-of [(str "files/" index-test-file-id "/media/a.txt")]))) + (t/is (empty? (index-entries-of [(str "files/" index-test-file-id "/unknown/a.json")]))) + (t/is (empty? (index-entries-of [(str "files//media/a.json")])))) + +(t/deftest index-entries-requires-literal-json-suffix + ;; deliberate tightening vs today's regexes: the dot is unescaped in + ;; `([^/]+).json$` / `tokens.json$`, so today `tokensXjson` and + ;; `objects/x-json` DO match; the classifier requires a literal + ;; `.json` suffix and ignores them + (t/is (empty? (index-entries-of [(str "files/" index-test-file-id "/tokensXjson")]))) + (t/is (empty? (index-entries-of ["objects/x-json"])))) + +(t/deftest import-binfile-v3-multiple-files-preserves-per-file-content + ;; the entries index must attribute every zip entry to the file it + ;; belongs to: with several files in the same manifest, pages and + ;; shapes of one file must not leak into another. The file also + ;; carries library content (color, typography, component and tokens + ;; set) so every rewired consumer is exercised on its present path. + (let [profile (th/create-profile* 1) + simple (prepare-simple-file profile) + color-id (uuid/custom 5 1) + typo-id (uuid/custom 5 2) + comp-id (uuid/custom 5 3) + svg (prepare-svg-raw-file profile 2) + output (tmp/tempfile :suffix ".zip")] + + (update-file! + :file-id (:id simple) + :profile-id (:id profile) + :revn 0 + :vern 0 + :changes + [{:type :add-color + :color {:id color-id + :name "import-color" + :color "#FF0000"}} + {:type :add-typography + :typography {:id typo-id + :name "import-typography" + :font-id "source-sans-pro" + :font-family "Source Sans Pro" + :font-variant-id "regular" + :font-size "16" + :font-weight "400" + :font-style "normal" + :line-height "1.4" + :letter-spacing "0" + :text-transform "none"}} + {:type :add-component + :id comp-id + :name "import-component" + :path "" + :main-instance-id (uuid/custom 2 1) + :main-instance-page (uuid/custom 1 1)} + {:type :set-tokens-lib + :tokens-lib (-> (ctob/make-tokens-lib) + (ctob/add-set (ctob/make-token-set :name "ImportSet")))}]) + + (v3/export-files! + (-> th/*system* + (assoc ::bfc/ids #{(:id simple) (:id svg)}) + (assoc ::bfc/embed-assets false) + (assoc ::bfc/include-libraries false)) + (io/output-stream output)) + + ;; import returns the imported file ids plus the library link + ;; resolution; the test only needs the ids + (let [result (:file-ids (-> th/*system* + (assoc ::bfc/project-id (:default-project-id profile)) + (assoc ::bfc/profile-id (:id profile)) + (assoc ::bfc/input output) + (v3/import-files!))) + files (map #(bfc/get-file th/*system* %) result) + svg-imported (some #(when (contains? (get-in % [:data :pages-index + svg-raw-page-id + :objects]) + svg-raw-root-id) + %) + files) + simple-imported (some #(when (contains? (get-in % [:data :pages-index + (uuid/custom 1 1) + :objects]) + (uuid/custom 2 1)) + %) + files)] + + (t/is (= 2 (count result))) + (t/is (= 2 (count (distinct result)))) + (t/is (some? svg-imported)) + (t/is (some? simple-imported)) + + ;; the svg-raw file keeps its subtree on its own page (plus the + ;; default page created by create-file) + (t/is (= [svg-raw-child-id] + (get-in svg-imported [:data :pages-index svg-raw-page-id + :objects svg-raw-root-id :shapes]))) + (t/is (= 2 (count (get-in svg-imported [:data :pages-index])))) + + ;; the simple file keeps its default page plus its two pages and + ;; its shape + (t/is (= 3 (count (get-in simple-imported [:data :pages-index])))) + (t/is (contains? (get-in simple-imported + [:data :pages-index (uuid/custom 1 1) :objects]) + (uuid/custom 2 1))) + + ;; library content is restored on its present path: the + ;; consumers' get-in keys must match the classifier buckets + (t/is (some? (get-in simple-imported [:data :colors color-id]))) + (t/is (some? (get-in simple-imported [:data :typographies typo-id]))) + (t/is (some? (get-in simple-imported [:data :components comp-id]))) + (t/is (= ["ImportSet"] + (vec (ctob/get-set-names + (get-in simple-imported [:data :tokens-lib])))))))) + +(t/deftest import-binfile-v3-restores-media-objects + ;; storage objects, per-file media entries and object thumbnails + ;; are classified through the entries index and restored end to end + (let [profile (th/create-profile* 1) + file (prepare-file-with-media profile) + thumb-page-id (uuid/custom 1 1) + thumb-frame-id (uuid/custom 6 1) + thumb-tag "medium" + output (tmp/tempfile :suffix ".zip")] + + ;; a thumbnail row backed by the same storage object as the media + ;; object, so the export produces a thumbnails/ zip entry + (let [mobj (th/db-get :file-media-object {:file-id (:id file)})] + (db/insert! th/*system* :file-tagged-object-thumbnail + {:file-id (:id file) + :tag thumb-tag + :object-id (thc/fmt-object-id {:file-id (:id file) + :page-id thumb-page-id + :frame-id thumb-frame-id + :tag thumb-tag}) + :media-id (:media-id mobj)})) + + (v3/export-files! + (-> th/*system* + (assoc ::bfc/ids #{(:id file)}) + (assoc ::bfc/embed-assets false) + (assoc ::bfc/include-libraries false)) + (io/output-stream output)) + + ;; import returns the imported file ids plus the library link + ;; resolution; the test only needs the ids + (let [result (:file-ids (-> th/*system* + (assoc ::bfc/project-id (:default-project-id profile)) + (assoc ::bfc/profile-id (:id profile)) + (assoc ::bfc/input output) + (v3/import-files!))) + mobjs (db/query th/*system* :file-media-object + {:file-id (first result)}) + thumbs (db/query th/*system* :file-tagged-object-thumbnail + {:file-id (first result)})] + (t/is (= 1 (count result))) + (t/is (pos? (count mobjs))) + (t/is (every? some? (map :media-id mobjs))) + + ;; the thumbnail is restored with its object-id rebuilt around + ;; the new file id and a media-id that resolves to storage + (t/is (= 1 (count thumbs))) + (let [thumb (first thumbs)] + (t/is (= thumb-tag (:tag thumb))) + (t/is (= (str (first result) "/" thumb-page-id "/" thumb-frame-id "/" thumb-tag) + (:object-id thumb))) + (t/is (some? (:media-id thumb)))))))