alonso.torres 11ac2f8989 ✨ Migrate swap slots to override groups and remove them
The migration 0033 stores every swap of a file as an override group.
A swap made in a copy is paired with the child of the near main that
holds the same position, as sync paired it; any other slot is dropped.
All pairings are found on the data as it was, so swaps nested in swaps
pair against the content of the outer one, and libraries pair the same
whether they are migrated or not. A head that refers to its own slot
only inherited it, and drops it even when its near main cannot be
reached. Any other slot that matches no child, or whose near main is in
a library that is not loaded, keeps the slot as its ref, so validation
reports it. An import, which migrates its files before it links their
libraries, leaves those slots and converts them once the libraries are
linked. A migrated head that sits elsewhere than the child it replaces
would place it gets its geometry marked as touched, so the first sync
does not move it. Migrated and new files carry the feature
components/swap-group, so a client that still writes slots cannot open
them.

Validation checks the new form: :misplaced-slot reports any slot,
:misplaced-swap-group the group anywhere but on a nested copy head in a
copy, :ref-not-in-near-main a nested copy head whose ref is not a child
of the near main of its parent, and :duplicate-ref two children of a
copy that refer to the same main shape. Each has a repair.

Nothing but the migrations reads slots any more: the slot branches go
from sync, swap, switch, reset, detach, paste, duplicate and move. The
helpers that read and write a slot on a shape move to files/swap-slots,
and the ones that follow refs to files/comp-processors. The graph
projection drops the FillsSwapSlot edge, the components debugger shows
the component a swapped head takes its content from, and the plugin API
documents componentRefShape for a swapped copy. The swap tests are
named after what they check.

Benchmark (lower quantile, ms; baseline / now / ratio):
  deepest-edit-through-three-levels    35.2 /  34.6 / 0.98
  edit-through-a-chain-of-dependents   68.9 /  63.5 / 0.92
  library-update-into-copies           83.1 /  76.4 / 0.92
  main-edit-into-10-copies              1.8 /   1.6 / 0.87
  main-edit-into-100-copies            15.4 /  15.1 / 0.98
  main-edit-into-1000-copies          153.1 / 148.8 / 0.97
  variant-switch-into-copies           32.1 /  26.4 / 0.82

AI-assisted-by: claude-opus-5-5
2026-11-10 20:21:15 +01:00

1497 lines
57 KiB
Clojure

;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns app.binfile.v3
"A ZIP based binary file exportation"
(:refer-clojure :exclude [read])
(:require
[app.binfile.cleaner :as bfl]
[app.binfile.common :as bfc]
[app.binfile.migrations :as bfm]
[app.common.data :as d]
[app.common.data.macros :as dm]
[app.common.exceptions :as ex]
[app.common.features :as cfeat]
[app.common.files.migrations :as-alias fmg]
[app.common.files.tokens :as cfo]
[app.common.json :as json]
[app.common.logging :as l]
[app.common.media :as cmedia]
[app.common.schema :as sm]
[app.common.thumbnails :as cth]
[app.common.time :as ct]
[app.common.types.color :as ctcl]
[app.common.types.component :as ctc]
[app.common.types.file :as ctf]
[app.common.types.page :as ctp]
[app.common.types.plugins :as ctpg]
[app.common.types.shape :as cts]
[app.common.types.tokens-lib :as ctob]
[app.common.types.tokens-status :as ctos]
[app.common.types.typography :as cty]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.db.sql :as-alias sql]
[app.storage :as sto]
[app.storage.impl :as sto.impl]
[app.util.events :as events]
[clojure.java.io :as jio]
[cuerdas.core :as str]
[datoteka.fs :as fs]
[datoteka.io :as io])
(:import
java.io.File
java.io.FilterInputStream
java.io.InputStream
java.io.OutputStreamWriter
java.lang.AutoCloseable
java.util.zip.ZipEntry
java.util.zip.ZipFile
java.util.zip.ZipOutputStream))
;; --- SCHEMA
(def ^:private schema:manifest
[:map {:title "Manifest"}
[:version ::sm/int]
[:type :string]
[:referer {:optional true} :string]
[:generated-by {:optional true} :string]
[:files
[:vector
[:map
[:id ::sm/uuid]
[:name :string]
[:features ::cfeat/features]]]]
[:relations {:optional true}
[:vector
[:tuple ::sm/uuid ::sm/uuid]]]
;; TODO: rename to :links
[:external-libraries {:optional true}
[:vector
[:map
[:id ::sm/uuid]
[:name :string]
[:slug :string]
[:used-by {:optional true} [:vector ::sm/uuid]]]]]])
(def ^:private schema:storage-object
[:map {:title "StorageObject"}
[:id ::sm/uuid]
[:size ::sm/int]
[:content-type bfc/schema:content-type]
[:bucket [::sm/one-of {:format :string} sto/valid-buckets]]
[:hash {:optional true} :string]])
(def ^:private schema:file-thumbnail
[:map {:title "FileThumbnail"}
[:file-id ::sm/uuid]
[:page-id ::sm/uuid]
[:frame-id ::sm/uuid]
[:tag :string]
[:media-id ::sm/uuid]])
(def ^:private schema:file
(sm/merge
ctf/schema:file
[:map
[:options {:optional true} ctf/schema:options]
[:tokens-source {:optional true} ::sm/uuid]]))
;; --- HELPERS
(defn- default-now
[o]
(or o (ct/now)))
;; --- ENCODERS
(def encode-file
(sm/encoder schema:file sm/json-transformer))
(def encode-page
(sm/encoder ctp/schema:page sm/json-transformer))
(def encode-shape
(sm/encoder cts/schema:shape sm/json-transformer))
(def encode-media
(sm/encoder ctf/schema:media sm/json-transformer))
(def encode-component
(sm/encoder ctc/schema:component sm/json-transformer))
(def encode-color
(sm/encoder ctcl/schema:library-color sm/json-transformer))
(def encode-typography
(sm/encoder cty/schema:typography sm/json-transformer))
(def encode-tokens-lib
(sm/encoder ctob/schema:tokens-lib sm/json-transformer))
(def encode-tokens-status
(sm/encoder ctos/schema:tokens-status sm/json-transformer))
(def encode-plugin-data
(sm/encoder ctpg/schema:plugin-data sm/json-transformer))
(def encode-storage-object
(sm/encoder schema:storage-object sm/json-transformer))
(def encode-file-thumbnail
(sm/encoder schema:file-thumbnail sm/json-transformer))
;; --- DECODERS
(def decode-manifest
(sm/decoder schema:manifest sm/json-transformer))
(def decode-media
(sm/decoder ctf/schema:media sm/json-transformer))
(def decode-component
(sm/decoder ctc/schema:component sm/json-transformer))
(def decode-color
(sm/decoder ctcl/schema:library-color sm/json-transformer))
(def decode-file
(sm/decoder schema:file sm/json-transformer))
(def decode-page
(sm/decoder ctp/schema:page sm/json-transformer))
(def decode-shape
(sm/decoder cts/schema:shape sm/json-transformer))
(def decode-typography
(sm/decoder cty/schema:typography sm/json-transformer))
(def decode-tokens-lib
(sm/decoder ctob/schema:tokens-lib sm/json-transformer))
(def decode-tokens-status
(sm/decoder ctos/schema:tokens-status sm/json-transformer))
(def decode-plugin-data
(sm/decoder ctpg/schema:plugin-data sm/json-transformer))
(def decode-storage-object
(sm/decoder schema:storage-object sm/json-transformer))
(def decode-file-thumbnail
(sm/decoder schema:file-thumbnail sm/json-transformer))
;; --- VALIDATORS
(def validate-manifest
(sm/check-fn schema:manifest))
(def validate-file
(sm/check-fn ctf/schema:file))
(def validate-page
(sm/check-fn ctp/schema:page))
(def validate-shape
(sm/check-fn cts/schema:shape))
(def validate-media
(sm/check-fn ctf/schema:media))
(def validate-color
(sm/check-fn ctcl/schema:library-color))
(def validate-component
(sm/check-fn ctc/schema:component))
(def validate-typography
(sm/check-fn cty/schema:typography))
(def validate-tokens-lib
(sm/check-fn ctob/schema:tokens-lib))
(def validate-tokens-status
(sm/check-fn ctos/schema:tokens-status))
(def validate-plugin-data
(sm/check-fn ctpg/schema:plugin-data))
(def validate-storage-object
(sm/check-fn schema:storage-object))
(def validate-file-thumbnail
(sm/check-fn schema:file-thumbnail))
;; --- PROGRESS
(defn- tap-progress
"Emit a progress tap of the core, with the counters of the outer scopes
in course when the cfg has any.
The outer counters live in `::outer-counters` of the cfg while a scope
is being processed (today only `:files`, tomorrow `:teams` and
`:projects` too), so every tap inside carries them as `:outer-counters`:
the stages inside do not repeat them, and whoever reads the taps (the
jobs adapter) builds its report from one tap alone instead of
remembering the last position it saw. A tap that happens outside any
scope carries no outer counters, which is the truth about it."
[cfg data]
(events/tap :progress
(cond-> data
(some? (::outer-counters cfg))
(assoc :outer-counters (::outer-counters cfg)))))
(defn- with-outer-counter
"Return cfg with the counter of `scope` set to `position`, keeping the
counters of the other outer scopes already in course, so entering a
nested scope never wipes the ones above it."
[cfg scope position]
(assoc-in cfg [::outer-counters scope] position))
;; --- EXPORT IMPL
(defn- write-entry!
[^ZipOutputStream output ^String path data]
(.putNextEntry output (ZipEntry. path))
(let [writer (OutputStreamWriter. output "UTF-8")]
(json/write writer data :indent true :key-fn json/write-camel-key)
(.flush writer))
(.closeEntry output))
(defn- get-file
[{:keys [::bfc/export-type] :as cfg} file-id]
(let [detach? (= export-type :detach-libraries)
embed? (= export-type :merge-libraries)]
(db/tx-run! cfg (fn [cfg]
(bfc/set-transaction-idle-timeout cfg)
(cond-> (bfc/get-file cfg file-id
{:realize? true
:include-deleted? true
:lock-for-update? true})
detach?
(-> (ctf/detach-external-references file-id)
(dissoc :libraries))
embed?
(update :data #(bfc/embed-assets cfg % file-id))
(not (or detach? embed?))
(update :data #(bfc/absorb-unlinked-tokens-source
cfg % (bfc/get-libraries cfg [file-id])))
:always
(bfc/clean-file-features))))))
(defn- export-storage-objects
[{:keys [::output] :as cfg}]
(let [storage (sto/resolve cfg)
ids (-> bfc/*state* deref :storage-objects not-empty)
total (count ids)]
(doseq [[index id] (d/enumerate ids)]
(let [sobject (sto/get-object storage id)
smeta (meta sobject)
ext (cmedia/mtype->extension (:content-type smeta))
path (str "objects/" id ".json")
params (-> (meta sobject)
(assoc :id (:id sobject))
(assoc :size (:size sobject))
(encode-storage-object))]
(write-entry! output path params)
(tap-progress cfg {:section :storage-object :id id
:current (inc index) :total total})
(with-open [input (sto/get-object-data storage sobject)]
(.putNextEntry ^ZipOutputStream output (ZipEntry. (str "objects/" id ext)))
(io/copy input output :size (:size sobject))
(.closeEntry ^ZipOutputStream output))))))
(defn- export-file
[{:keys [::file-id ::output] :as cfg}]
(let [file (get-file cfg file-id)
media (->> (bfc/get-file-media cfg file)
(map (fn [media]
(dissoc media :file-id))))
data (:data file)
typographies (:typographies data)
components (:components data)
colors (:colors data)
tokens-lib (:tokens-lib data)
tokens-status (:tokens-status data)
tokens-source (:tokens-source data)
pages (:pages data)
pages-index (:pages-index data)
thumbnails (bfc/get-file-object-thumbnails cfg file-id)]
(tap-progress cfg {:section :file :id file-id :name (:name file)})
(vswap! bfc/*state* update :files assoc file-id
{:id file-id
:name (:name file)
:features (:features file)})
(let [file (cond-> (select-keys file bfc/file-attrs)
(:options data)
(assoc :options (:options data))
(:tokens-source data)
(assoc :tokens-source (:tokens-source data)))
file (-> file
(dissoc :data)
(dissoc :deleted-at)
(encode-file))
path (str "files/" file-id ".json")]
(write-entry! output path file))
(doseq [[index page-id] (d/enumerate pages)]
(let [path (str "files/" file-id "/pages/" page-id ".json")
page (get pages-index page-id)
objects (:objects page)
page (-> page
(dissoc :objects)
(assoc :index index))
page (encode-page page)]
(write-entry! output path page)
(tap-progress cfg {:section :page :id page-id :name (:name page) :file-id file-id
:current (inc index) :total (count pages)})
(doseq [[shape-id shape] objects]
(let [path (str "files/" file-id "/pages/" page-id "/" shape-id ".json")
shape (assoc shape :page-id page-id)
shape (encode-shape shape)]
(write-entry! output path shape)))))
(vswap! bfc/*state* bfc/collect-storage-objects media)
(vswap! bfc/*state* bfc/collect-storage-objects thumbnails)
(doseq [[index {:keys [id] :as item}] (d/enumerate media)]
(let [path (str "files/" file-id "/media/" id ".json")
encoded (encode-media item)]
(tap-progress cfg {:section :media :id id :file-id file-id
:current (inc index) :total (count media)})
(write-entry! output path encoded)))
(doseq [[index thumbnail] (d/enumerate thumbnails)]
(let [data (cth/parse-object-id (:object-id thumbnail))
path (str "files/" file-id "/thumbnails/" (:tag data) "/" (:page-id data)
"/" (:frame-id data) ".json")
data (-> data
(assoc :media-id (:media-id thumbnail))
(encode-file-thumbnail))]
(tap-progress cfg {:section :thumbnails :id (:object-id thumbnail) :file-id file-id
:current (inc index) :total (count thumbnails)})
(write-entry! output path data)))
(doseq [[index [id component]] (d/enumerate components)]
(let [path (str "files/" file-id "/components/" id ".json")
component (encode-component component)]
(tap-progress cfg {:section :component :id id :file-id file-id
:current (inc index) :total (count components)})
(write-entry! output path component)))
(doseq [[index [id color]] (d/enumerate colors)]
(let [path (str "files/" file-id "/colors/" id ".json")
color (-> (encode-color color)
(dissoc :file-id))
color (cond-> color
(and (contains? color :path)
(str/empty? (:path color)))
(dissoc :path))]
(tap-progress cfg {:section :color :id id :file-id file-id
:current (inc index) :total (count colors)})
(write-entry! output path color)))
(doseq [[index [id object]] (d/enumerate typographies)]
(let [path (str "files/" file-id "/typographies/" id ".json")
typography (encode-typography object)]
(tap-progress cfg {:section :typography :id id :file-id file-id
:current (inc index) :total (count typographies)})
(write-entry! output path typography)))
(when (and tokens-lib
(not (ctob/empty-lib? tokens-lib)))
(let [path (str "files/" file-id "/tokens.json")
encoded-tokens (encode-tokens-lib tokens-lib)]
(tap-progress cfg {:section :tokens-lib :file-id file-id})
(write-entry! output path encoded-tokens)))
(when tokens-status
(let [path (str "files/" file-id "/tokens-status.json")
library (if (and (some? tokens-source)
(not= tokens-source (:id data)))
(get-file cfg tokens-source)
file)
effective-tokens-lib (when library
(dm/get-in library [:data :tokens-lib]))
encoded-status (-> (encode-tokens-status tokens-status)
(cfo/ids->names effective-tokens-lib))]
(tap-progress cfg {:section :tokens-status :file-id file-id})
(write-entry! output path encoded-status)))))
(defn- export-files
[{:keys [::bfc/file-ids ::bfc/export-type ::output] :as cfg}]
(let [original-ids file-ids
file-ids (into file-ids (when (= export-type :include-libraries)
(bfc/get-libraries cfg file-ids)))
rels (if (= export-type :include-libraries)
(->> (bfc/get-files-rels cfg file-ids)
(mapv (juxt :file-id :library-file-id)))
[])
;; Compute external libraries: referenced by original files but
;; not included in the export set. Only relevant for :link-later.
external-libs
(when (= export-type :link-later)
(let [original-rels (bfc/get-files-rels cfg original-ids)
lib-ids (into #{} (map :library-file-id) original-rels)]
(when (seq lib-ids)
(let [lib-names (bfc/get-files-names cfg lib-ids)]
(->> lib-names
(mapv (fn [{:keys [id name]}]
(let [slug (bfc/slugify-name name)]
(when-not (str/blank? slug)
{:id id
:name name
:slug slug
:used-by (->> original-rels
(filter #(= (:library-file-id %) id))
(mapv :file-id))}))))
(filterv some?))))))]
(vswap! bfc/*state* assoc :files (d/ordered-map))
;; Write all the exporting files
(doseq [[index file-id] (d/enumerate file-ids)]
(-> cfg
(assoc ::file-id file-id)
(with-outer-counter :files {:current (inc index)
:total (count file-ids)})
(export-file)))
;; Write manifest file
(let [files (:files @bfc/*state*)
params (cond-> {:type "penpot/export-files"
:version 1
:generated-by (str "penpot/" (:full cf/version))
:referer "penpot"
:files (vec (vals files))
:relations rels}
(seq external-libs)
(assoc :external-libraries external-libs))]
(write-entry! output "manifest.json" params))))
;; --- IMPORT IMPL
(defn- read-zip-entries
[^ZipFile input]
(into #{} (iterator-seq (.entries input))))
(defn- get-zip-entry*
[^ZipFile input ^String path]
(.getEntry input path))
(defn- get-zip-entry
[input path]
(let [entry (get-zip-entry* input path)]
(when-not entry
(ex/raise :type :validation
:code :inconsistent-penpot-file
:hint "the penpot file seems corrupt, missing underlying zip entry"
:path path))
entry))
(defn- get-zip-entry-size
[^ZipEntry entry]
(.getSize entry))
(defn- zip-entry-name
[^ZipEntry entry]
(.getName entry))
(defn- zip-entry-stream
^InputStream
[^ZipFile input ^ZipEntry entry]
(.getInputStream input entry))
(defn- size-limiting-stream
"Wraps an InputStream to enforce a maximum number of decompressed bytes.
Raises :validation :max-file-size-reached when the limit is exceeded.
`counter` holds the bytes accounted so far: pass a fresh atom for a
per-entry cap, or the shared job-wide atom for the cumulative budget.
`entry-name` identifies the entry being read and is included in the
raised ex-data for triage."
^InputStream
[^InputStream input ^long max-size counter entry-name]
(let [on-read (fn [n]
(when (pos? n)
(when (> (swap! counter + (long n)) max-size)
(ex/raise :type :validation
:code :max-file-size-reached
:hint (str "stream exceeded max size on entry " entry-name ": " max-size)
:path entry-name
:max max-size)))
n)]
(proxy [FilterInputStream] [input]
(read
([]
(let [b (.read input)]
(when (pos? b) (on-read 1))
b))
([^bytes buf]
(on-read (.read input buf 0 (alength buf))))
([^bytes buf off]
(on-read (.read input buf (int off) (- (alength buf) (int off)))))
([^bytes buf off len]
(on-read (.read input buf (int off) (int len)))))
(skip [n]
;; Skipped bytes were already decompressed, so they count against
;; the budget like read bytes do.
(let [skipped (.skip input (long n))]
(when (pos? skipped)
(on-read skipped))
skipped)))))
(defn- setup-limits
"Resolve the binfile import limits once per job from cfg, falling back
to the namespace defaults when the keys are absent. Returns cfg with
`::max-text-entry-size` (per JSON/text entry cap), `::max-text-total-size` (cumulative
JSON/text budget), `::accumulated-total-text-size` (shared atom holding the cumulative
bytes read so far), `::max-binary-entry-size` (storage blob cap) and
`::max-zip-entries` (zip entry count cap)."
[cfg]
(assoc cfg
::max-text-entry-size (or (::bfc/import-max-text-entry-size cfg) bfc/default-max-text-entry-size)
::max-text-total-size (or (::bfc/import-max-text-total-size cfg) bfc/default-max-text-total-size)
::accumulated-total-text-size (atom 0)
::max-binary-entry-size (or (::bfc/import-max-binary-entry-size cfg) bfc/default-max-binary-entry-size)
::max-zip-entries (or (::bfc/import-max-zip-entries cfg) bfc/default-max-zip-entries)))
(defn- zip-entry-reader
"Opens a UTF-8 reader over a zip entry, enforcing the per-entry
decompressed-size cap on the actual bytes streamed through it and
accounting the same bytes against the shared job-wide budget.
A cheap pre-check on the entry-declared size rejects obvious bombs without
opening the stream; the streaming counters remain the authoritative guard
because the declared size is attacker-controlled metadata."
[cfg ^ZipFile input ^ZipEntry entry]
(let [entry-name (zip-entry-name entry)
declared (get-zip-entry-size entry)
max-size (::max-text-entry-size cfg)]
(when (and (not (neg? declared)) (> declared (long max-size)))
(ex/raise :type :validation
:code :max-file-size-reached
:hint (str "zip entry exceeds maximum size: " entry-name)
:path entry-name
:max max-size
:found declared))
(-> (zip-entry-stream input entry)
(size-limiting-stream max-size (atom 0) entry-name)
(size-limiting-stream (::max-text-total-size cfg) (::accumulated-total-text-size cfg) entry-name)
(io/reader :encoding "UTF-8"))))
(defn- zip-entry-storage-content
"Wraps a ZipFile and ZipEntry into a penpot storage compatible
object and avoid creating temporal objects"
[input entry & {:keys [max-size]}]
(let [stream-fn (fn []
(cond-> (zip-entry-stream input entry)
max-size (size-limiting-stream max-size (atom 0) (zip-entry-name entry))))
hash (delay (->> (stream-fn)
(sto.impl/calculate-hash)))]
(reify
sto.impl/IContentObject
(get-size [_]
(get-zip-entry-size entry))
sto.impl/IContentHash
(get-hash [_]
(deref hash))
jio/IOFactory
(make-reader [this opts]
(jio/make-reader this opts))
(make-writer [_ _]
(throw (UnsupportedOperationException. "not implemented")))
(make-input-stream [_ _]
(stream-fn))
(make-output-stream [_ _]
(throw (UnsupportedOperationException. "not implemented"))))))
(defn- read-manifest
[cfg ^ZipFile input]
(let [entry (get-zip-entry input "manifest.json")]
(with-open [^AutoCloseable reader (zip-entry-reader cfg input entry)]
(let [manifest (json/read reader :key-fn json/read-kebab-key)]
(decode-manifest manifest)))))
(def ^:private file-object-entry-categories
"Zip path segments that hold per-file object entries on the
`files/<file-id>/<category>/<object-id>.json` shape."
#{"media" "colors" "components" "typographies"})
(defn- index-entry-name
"Classify a single zip entry by the raw shape of its path and
accumulate it on the index.
It replaces the per-file regex matchers with a single
classification pass over all entries. The `.json` suffix is matched
literally: the previous regexes left the dot unescaped, so crafted
paths like `files/<file-id>/tokensXjson` or `objects/x-json` matched
by accident; requiring the literal suffix ignores them (legitimate
exports always write a literal `.json` suffix). Unknown paths are
ignored."
[index ^String name entry]
(if-not (and name (str/ends-with? name ".json"))
index
(let [base (subs name 0 (- (count name) 5))
segs (str/split base "/")
seg-n (count segs)
seg-1 (nth segs 0 nil)
seg-2 (nth segs 1 nil)
seg-3 (nth segs 2 nil)]
(if-not (and (pos? seg-n) (every? #(pos? (count %)) segs))
index
(cond
;; objects/<object-id>.json
(and (= seg-n 2) (= seg-1 "objects"))
(update index :objects bfc/conj-vec
{:entry entry :id (parse-uuid seg-2)})
;; files/<file-id>/tokens.json
(and (= seg-n 3) (= seg-1 "files") (= seg-3 "tokens"))
(update-in index [:tokens seg-2] bfc/conj-vec {:entry entry})
;; files/<file-id>/tokens-status.json
(and (= seg-n 3) (= seg-1 "files") (= seg-3 "tokens-status"))
(update-in index [:tokens-status seg-2] bfc/conj-vec {:entry entry})
;; files/<file-id>/thumbnails/<tag>/<page-id>/<frame-id>.json
(and (= seg-n 6) (= seg-1 "files") (= seg-3 "thumbnails"))
(update-in index [:thumbnails seg-2] bfc/conj-vec
{:entry entry
:tag (nth segs 3)
:page-id (parse-uuid (nth segs 4))
:frame-id (parse-uuid (nth segs 5))
:file-id seg-2})
;; files/<file-id>/pages/<page-id>.json
(and (= seg-n 4) (= seg-1 "files") (= seg-3 "pages"))
(update-in index [:pages seg-2] bfc/conj-vec
{:entry entry :id (parse-uuid (nth segs 3))})
;; files/<file-id>/pages/<page-id>/<shape-id>.json
(and (= seg-n 5) (= seg-1 "files") (= seg-3 "pages"))
(update-in index [:shapes seg-2 (nth segs 3)] bfc/conj-vec
{:entry entry
:page-id (nth segs 3)
:id (parse-uuid (nth segs 4))})
;; files/<file-id>/<category>/<object-id>.json
(and (= seg-n 4)
(= seg-1 "files")
(contains? file-object-entry-categories seg-3))
(update-in index [(keyword seg-3) seg-2] bfc/conj-vec
{:entry entry :id (parse-uuid (nth segs 3))})
:else
index)))))
(defn- index-entries
"Classify all the provided zip entries in a single pass and group
them by their path shape, so import consumers can lookup their
entries per file (and per page) instead of rescanning the whole
entry collection for every file and page."
[entries]
(reduce (fn [index entry]
(index-entry-name index (zip-entry-name entry) entry))
{}
entries))
(defn- read-entry
([cfg ^ZipFile input entry]
(read-entry cfg input entry json/read-kebab-key))
([cfg ^ZipFile input entry key-fn]
(with-open [^AutoCloseable reader (zip-entry-reader cfg input entry)]
(json/read reader :key-fn key-fn))))
(defn- read-file
[{:keys [::bfc/input ::bfc/timestamp] :as cfg} file-id]
(let [path (str "files/" file-id ".json")
entry (get-zip-entry input path)]
(-> (read-entry cfg input entry)
(decode-file)
(update :revn d/nilv 1)
(update :created-at d/nilv timestamp)
(update :modified-at d/nilv timestamp)
(validate-file))))
(defn- read-file-plugin-data
[{:keys [::bfc/input] :as cfg} file-id]
(let [path (str "files/" file-id "/plugin-data.json")
entry (get-zip-entry* input path)]
(when entry
(-> (read-entry cfg input entry)
(decode-plugin-data)
(validate-plugin-data)))))
(defn- read-file-media
[{:keys [::bfc/input ::entries-index] :as cfg} file-id]
(->> (get-in entries-index [:media (str file-id)])
(reduce (fn [result {:keys [id entry]}]
(let [object (->> (read-entry cfg input entry)
(decode-media)
(validate-media))
object (-> object
(assoc :file-id file-id)
(update :created-at default-now)
;; FIXME: this is set default to true for
;; setting a value, this prop is no longer
;; relevant;
(assoc :is-local true))]
(if (= id (:id object))
(conj result object)
result)))
[])
(not-empty)))
(defn- read-file-colors
[{:keys [::bfc/input ::entries-index] :as cfg} file-id]
(let [entries (get-in entries-index [:colors (str file-id)])]
(->> (d/enumerate entries)
(reduce (fn [result [index {:keys [id entry]}]]
(let [object (->> (read-entry cfg input entry)
(decode-color)
(validate-color))]
(tap-progress cfg {:section :color :id id :file-id file-id
:current (inc index) :total (count entries)})
(if (= id (:id object))
(assoc result id object)
result)))
{})
(not-empty))))
(defn- read-file-components
[{:keys [::bfc/input ::entries-index] :as cfg} file-id]
(let [clean-component-post-decode
(fn [component]
(d/update-when component :objects
(fn [objects]
(reduce-kv (fn [objects id shape]
(assoc objects id (bfl/clean-shape-post-decode shape)))
objects
objects))))
clean-component-pre-decode
(fn [component]
(d/update-when component :objects
(fn [objects]
(reduce-kv (fn [objects id shape]
(assoc objects id (bfl/clean-shape-pre-decode shape)))
objects
objects))))
entries
(get-in entries-index [:components (str file-id)])]
(->> (d/enumerate entries)
(reduce (fn [result [index {:keys [id entry]}]]
(let [object (->> (read-entry cfg input entry)
(clean-component-pre-decode)
(decode-component)
(clean-component-post-decode))]
(tap-progress cfg {:section :component :id id :file-id file-id
:current (inc index) :total (count entries)})
(if (= id (:id object))
(assoc result id object)
result)))
{})
(not-empty))))
(defn- read-file-typographies
[{:keys [::bfc/input ::entries-index] :as cfg} file-id]
(let [entries (get-in entries-index [:typographies (str file-id)])]
(->> (d/enumerate entries)
(reduce (fn [result [index {:keys [id entry]}]]
(let [object (->> (read-entry cfg input entry)
(decode-typography)
(validate-typography))]
(tap-progress cfg {:section :typography :id id :file-id file-id
:current (inc index) :total (count entries)})
(if (= id (:id object))
(assoc result id object)
result)))
{})
(not-empty))))
(defn- read-file-tokens-lib*
[{:keys [::bfc/input ::entries-index] :as cfg} file-id]
(when-let [{:keys [entry]} (first (get-in entries-index [:tokens (str file-id)]))]
(tap-progress cfg {:section :tokens-lib :file-id file-id})
(->> (read-entry cfg input entry nil)
(decode-tokens-lib)
(validate-tokens-lib))))
(defn- read-file-tokens-lib
"Read the tokens lib of a file once per import. Decoding generates new
set ids, so the tokens lib of a library must be the same instance when
it is imported and when it resolves the tokens status of a file that
uses it as tokens source."
[cfg file-id]
(let [cache (:tokens-libs @bfc/*state*)]
(if (contains? cache file-id)
(get cache file-id)
(let [tokens-lib (read-file-tokens-lib* cfg file-id)]
(vswap! bfc/*state* assoc-in [:tokens-libs file-id] tokens-lib)
tokens-lib))))
(defn- read-file-tokens-status
[{:keys [::bfc/input ::entries-index] :as cfg} file-id tokens-lib]
(when-let [{:keys [entry]} (first (get-in entries-index [:tokens-status (str file-id)]))]
(tap-progress cfg {:section :tokens-status :file-id file-id})
(->> (read-entry cfg input entry nil)
(cfo/names->ids tokens-lib)
(decode-tokens-status)
(validate-tokens-status))))
(defn- read-file-shapes
[{:keys [::bfc/input ::entries-index] :as cfg} file-id page-id]
(->> (get-in entries-index [:shapes (str file-id) (str page-id)])
(reduce (fn [result {:keys [id entry]}]
(let [object (->> (read-entry cfg input entry)
(bfl/clean-shape-pre-decode)
(decode-shape)
(bfl/clean-shape-post-decode))]
(if (= id (:id object))
(assoc result id object)
result)))
{})
(not-empty)))
(defn- read-file-pages
[{:keys [::bfc/input ::entries-index] :as cfg} file-id]
(let [entries (get-in entries-index [:pages (str file-id)])]
(->> (d/enumerate entries)
(keep (fn [[index {:keys [id entry]}]]
(let [page (->> (read-entry cfg input entry)
(decode-page))
page (dissoc page :options)]
(tap-progress cfg {:section :page :id id :file-id file-id
:current (inc index) :total (count entries)})
(when (= id (:id page))
(let [objects (read-file-shapes cfg file-id id)]
(assoc page :objects objects))))))
(sort-by :index)
(reduce (fn [result {:keys [id] :as page}]
(assoc result id (dissoc page :index)))
(d/ordered-map)))))
(defn- read-file-thumbnails
[{:keys [::bfc/input ::entries-index] :as cfg} file-id]
(->> (get-in entries-index [:thumbnails (str file-id)])
(reduce (fn [result {:keys [page-id frame-id tag entry]}]
(let [object (->> (read-entry cfg input entry)
(decode-file-thumbnail)
(validate-file-thumbnail))]
(if (and (= frame-id (:frame-id object))
(= page-id (:page-id object))
(= tag (:tag object)))
(conj result object)
result)))
[])
(not-empty)))
(defn- read-file-data
[cfg file-id tokens-source]
(let [colors (read-file-colors cfg file-id)
typographies (read-file-typographies cfg file-id)
tokens-lib (read-file-tokens-lib cfg file-id)
effective-tokens-lib (if (and (some? tokens-source)
(not= tokens-source file-id))
(read-file-tokens-lib cfg tokens-source)
tokens-lib)
tokens-status (read-file-tokens-status cfg file-id effective-tokens-lib)
components (read-file-components cfg file-id)
plugin-data (read-file-plugin-data cfg file-id)
pages (read-file-pages cfg file-id)]
{:pages (-> pages keys vec)
:pages-index (into {} pages)
:colors colors
:typographies typographies
:tokens-lib tokens-lib
:tokens-status tokens-status
:components components
:plugin-data plugin-data}))
(defn- import-file
[{:keys [::db/conn ::bfc/project-id ::manifest] :as cfg}
{file-id :id file-name :name}
position]
(let [cfg (with-outer-counter cfg :files position)
file-id' (bfc/lookup-index file-id)
file (read-file cfg file-id)
media (read-file-media cfg file-id)
thumbnails (read-file-thumbnails cfg file-id)]
(l/dbg :hint "processing file"
:id (str file-id')
:prev-id (str file-id)
:features (str/join "," (:features file))
:version (:version file)
::l/sync? true)
(tap-progress cfg {:section :file :file-id file-id})
(vswap! bfc/*state* update :index bfc/update-index media :id)
(doseq [[index item] (d/enumerate media)]
(let [params (-> item
(update :id bfc/lookup-index)
(assoc :file-id file-id')
(d/update-when :media-id bfc/lookup-index)
(d/update-when :thumbnail-id bfc/lookup-index))]
(tap-progress cfg {:section :media :id (:id params) :file-id file-id
:current (inc index) :total (count media)})
(l/dbg :hint "inserting media object"
:file-id (str file-id')
:id (str (:id params))
:media-id (str (:media-id params))
:thumbnail-id (str (:thumbnail-id params))
:old-id (str (:id item))
::l/sync? true)
(db/insert! conn :file-media-object params
::db/on-conflict-do-nothing? (::bfc/overwrite cfg))))
(doseq [[index item] (d/enumerate thumbnails)]
(let [media-id (bfc/lookup-index (:media-id item))
object-id (-> (assoc item :file-id file-id')
(cth/fmt-object-id))
params {:file-id file-id'
:object-id object-id
:tag (:tag item)
:media-id media-id}]
(l/dbg :hint "inserting object thumbnail"
:file-id (str file-id')
:media-id (str media-id)
::l/sync? true)
(tap-progress cfg {:section :thumbnail :file-id file-id :object-id object-id
:current (inc index) :total (count thumbnails)})
(db/insert! conn :file-tagged-object-thumbnail params
::db/on-conflict-do-nothing? true)))
(let [data (-> (read-file-data cfg file-id (:tokens-source file))
(d/without-nils)
(assoc :id file-id')
(cond-> (:options file)
(assoc :options (:options file)))
(cond-> (:tokens-source file)
(assoc :tokens-source (bfc/lookup-index (:tokens-source file)))))
file (-> (select-keys file bfc/file-attrs)
(assoc :id file-id')
(assoc :data data)
(assoc :name file-name)
(assoc :project-id project-id)
(assoc :metadata (d/without-nils
{:generated-by (get manifest :generated-by)
:referer (or (get manifest :referer) (get manifest :refer))}))
(dissoc :options)
(dissoc :tokens-source)
(bfc/check-components-v2))
file (bfc/process-imported-file cfg file)
file (ctf/check-file file)]
(bfm/register-pending-migrations! cfg file)
(bfc/save-file! cfg file)
file-id')))
(defn- import-file-relations
[{:keys [::db/conn ::manifest ::bfc/timestamp] :as cfg}]
(let [relations (:relations manifest)
file-ids (into #{} (map :id) (:files manifest))]
(doseq [[index [file-id libr-id]] (d/enumerate relations)]
(tap-progress cfg {:section :relations
:current (inc index) :total (count relations)})
(if (and (not= file-id libr-id)
(contains? file-ids file-id)
(contains? file-ids libr-id))
(let [file-id (bfc/lookup-index file-id)
libr-id (bfc/lookup-index libr-id)]
(l/dbg :hint "create file library link"
:file-id (str file-id)
:lib-id (str libr-id)
::l/sync? true)
(let [rel-params {:file-id file-id
:library-file-id libr-id}]
(db/insert! conn :file-library-rel rel-params)
(bfc/upsert-file-library-sync! conn (assoc rel-params :synced-at timestamp))))
(l/warn :hint "ignoring invalid file library relation"
:file-id (str file-id)
:lib-id (str libr-id))))))
(defn- import-storage-objects
[{:keys [::bfc/input ::entries-index ::bfc/timestamp] :as cfg}]
;; IMPORTANT: we strongly do not reuse the main connection that can
;; run inside a transaction because the storage upload process can
;; fail in the middle of uploading and leave garbage on the underlying
;; backend, if we participate in the main transaction and it aborts
;; we will lose all registry of the pending to reconcile blobs
;; what the storage subsystem registers in other parallel
;; transaction
(let [storage (sto/resolve cfg)
entries (:objects entries-index)
total (count entries)]
(doseq [[index {:keys [id entry]}] (d/enumerate entries)]
(tap-progress cfg {:section :storage-objects
:current (inc index) :total total})
(let [object (-> (read-entry cfg input entry)
(decode-storage-object)
(d/update-when :content-type bfc/normalize-content-type)
(update :bucket d/nilv sto/default-bucket)
(validate-storage-object))
ext (cmedia/mtype->extension (:content-type object))
path (str "objects/" id ext)
content (zip-entry-storage-content input
(get-zip-entry input path)
:max-size (::max-binary-entry-size cfg))]
(when (not= (:size object) (sto/get-size content))
(ex/raise :type :validation
:code :inconsistent-penpot-file
:hint "found corrupted storage object: size does not match"
:path path
:expected-size (:size object)
:found-size (sto/get-size content)))
(let [max (::max-binary-entry-size cfg)]
(when (> (sto/get-size content) max)
(ex/raise :type :validation
:code :max-file-size-reached
:hint (str "storage object exceeds maximum size: " (sto/get-size content))
:path path
:max max
:found (sto/get-size content))))
(when-let [hash (get object :hash)]
(when (not= hash (sto/get-hash content))
(ex/raise :type :validation
:code :inconsistent-penpot-file
:hint "found corrupted storage object: hash does not match"
:path path
:expected-hash (:hash object)
:found-hash (sto/get-hash content))))
(let [clean (when (bfc/svg-object? object)
(let [limit (::max-binary-entry-size cfg)
raw (with-open [istream (cond-> (zip-entry-stream input (get-zip-entry input path))
limit (size-limiting-stream limit (atom 0) path))]
(io/read istream))]
(bfc/sanitize-imported-svg object raw)))
content (if-let [{:keys [bytes hash]} clean]
(-> (sto/content bytes)
(sto/wrap-with-hash hash))
content)
params (-> object
(dissoc :id :size)
(assoc ::sto/content content)
(assoc ::sto/deduplicate? true)
(assoc ::sto/touched-at timestamp))
sobject (sto/put-object! storage params)]
(l/dbg :hint "persisted storage object"
:id (str (:id sobject))
:prev-id (str id)
:bucket (:bucket params)
::l/sync? true)
(vswap! bfc/*state* update :index assoc id (:id sobject)))))))
(defn- add-to-file
"Add a resolved library entry to a file in the file-grouped resolution.
`key` is :done (auto-linked) or :pending (needs resolution)."
[acc file-id file-name key entry]
(update acc file-id (fn [file]
(let [file (or file {:id file-id
:name file-name
:done []
:pending []})]
(update file key conj entry)))))
(defn- compute-link-decisions
"Returns a map of {old-lib-id -> {:library-id ... :library ...}} for external
libraries that should be auto-linked (single candidate AND importer has edit
permission). Libraries with zero or multiple candidates, or where the importer
lacks permission, are excluded — their refs should remain dangling."
[{:keys [::db/conn ::manifest ::bfc/team-id ::bfc/profile-id] :as cfg}]
(reduce
(fn [acc ext-lib]
(let [slug (:slug ext-lib)]
(if (nil? slug)
acc
(let [matching (into [] (bfc/find-shared-files-by-slug cfg team-id slug))]
(if (not= 1 (count matching))
acc
(let [library (first matching)
perms (bfc/get-file-permissions conn profile-id (:id library))]
(if (:can-edit perms)
(assoc acc (:id ext-lib) {:library-id (:id library)
:library library})
acc)))))))
{}
(:external-libraries manifest)))
(defn- resolve-and-link-libraries
"For each external library in the manifest, resolve candidates by slug.
Auto-links single matches (creating DB rows) and builds a file-grouped
resolution map keyed by imported file-id (new UUID)."
[{:keys [::db/conn ::manifest ::bfc/team-id ::bfc/timestamp] :as cfg} files-info]
(assert (uuid? team-id) "team-id should be provided")
(let [file-ids (keys files-info)
decisions (compute-link-decisions cfg)]
(reduce
(fn [acc ext-lib]
(assert (contains? ext-lib :id) "expected `:id` on ext-lib")
(assert (contains? ext-lib :name) "expected `:name` on ext-lib")
(assert (contains? ext-lib :used-by) "expected `:used-by` on ext-lib")
(assert (contains? ext-lib :slug) "expected `:slug` on ext-lib")
(let [used-by (into #{} (map bfc/lookup-index) (:used-by ext-lib))]
(cond
;; No slug → skip
(nil? (:slug ext-lib))
acc
;; Has decision → auto-link (single match + can-edit)
(contains? decisions (:id ext-lib))
(let [{:keys [library-id]} (get decisions (:id ext-lib))
used-by (filter used-by file-ids)]
(doseq [file-id used-by]
(let [rel-params {:file-id file-id :library-file-id library-id}]
(db/insert! conn :file-library-rel rel-params
{::db/on-conflict-do-nothing? true})
(bfc/upsert-file-library-sync! conn (assoc rel-params :synced-at timestamp))))
(let [entry {:id (:id ext-lib)
:name (:name ext-lib)
:linked-to library-id}]
(reduce (fn [acc file-id]
(add-to-file acc file-id (get files-info file-id) :done entry))
acc used-by)))
;; Has candidates but no decision → multi-match or no permission → pending
:else
(let [matching-libraries (into [] (bfc/find-shared-files-by-slug cfg team-id (:slug ext-lib)))]
(if (empty? matching-libraries)
acc
(let [candidates (mapv (fn [lib]
(let [project-id (:project-id lib)
project (bfc/get-project cfg project-id)
project-name (:name project)]
{:id (:id lib)
:name (:name lib)
:project-id project-id
:project-name project-name}))
matching-libraries)
entry {:id (:id ext-lib)
:name (:name ext-lib)
:candidates candidates}]
(reduce (fn [acc file-id]
(add-to-file acc file-id (get files-info file-id) :pending entry))
acc used-by)))))))
{}
(:external-libraries manifest))))
(defn- import-files*
[{:keys [::manifest] :as cfg}]
(bfc/configure-import-transaction cfg)
(vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id)
(import-storage-objects cfg)
;; Pre-resolve external libraries and add their id mappings to the index
;; BEFORE importing files. This allows relink-refs (inside process-file)
;; to correctly remap :component-file references to the destination library.
;; Only remap when a link will actually be created (single match + can-edit).
(let [decisions (compute-link-decisions cfg)]
(doseq [[old-lib-id {:keys [library-id]}] decisions]
(l/trc :hint "pre-resolving external library"
:old-id (str old-lib-id)
:new-id (str library-id))
(vswap! bfc/*state* update :index assoc old-lib-id library-id)))
(let [files (get manifest :files)
total (count files)
file-ids (reduce (fn [result [index file]]
(let [name' (get file :name)
file (assoc file :name name')]
(conj result (import-file cfg file
{:current (inc index)
:total total}))))
[]
(d/enumerate files))
;; Build map of file-id to file-name for resolution
files-info (into {} (map (fn [file-id manifest-file]
[file-id (:name manifest-file)])
file-ids
files))]
(import-file-relations cfg)
(let [resolution (resolve-and-link-libraries cfg files-info)]
(bfc/convert-deferred-swaps! cfg)
(bfm/apply-pending-migrations! cfg)
{:file-ids file-ids
:resolution resolution})))
(defn- invalidate-thumbnails
[cfg file-id]
(let [storage (sto/resolve cfg ::db/reuse-conn true)
sql-1
(str "update file_tagged_object_thumbnail "
" set deleted_at = now() "
" where file_id=? returning media_id")
sql-2
(str "update file_thumbnail "
" set deleted_at = now() "
" where file_id=? returning media_id")]
(run! #(sto/touch-object! storage %)
(sequence
(keep :media-id)
(concat
(db/exec! cfg [sql-1 file-id])
(db/exec! cfg [sql-2 file-id]))))))
(defn- import-file-and-overwrite*
[{:keys [::manifest ::bfc/file-id] :as cfg}]
(when (not= 1 (count (:files manifest)))
(ex/raise :type :validation
:code :invalid-condition
:hint "unable to perform in-place update with binfile containing more than 1 file"
:manifest manifest))
(bfc/configure-import-transaction cfg)
(let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true)
file (first (get manifest :files))
cfg (assoc cfg ::bfc/overwrite true)]
(vswap! bfc/*state* update :index assoc (:id file) file-id)
(binding [bfc/*options* cfg
bfc/*reference-file* ref-file]
(import-storage-objects cfg)
(import-file cfg file {:current 1 :total 1})
(invalidate-thumbnails cfg file-id)
(bfm/apply-pending-migrations! cfg)
{:file-ids [file-id]
:resolution {}})))
(defn- import-files
[{:keys [::bfc/timestamp ::bfc/input] :or {timestamp (ct/now)} :as cfg}]
(assert (instance? ZipFile input) "expected zip file")
(assert (ct/inst? timestamp) "expected valid instant")
;; Resolve all import limits once per job (see `setup-limits`); every
;; bounded read below accounts its actual decompressed bytes against the
;; shared job-wide budget, so many entries each under the per-entry cap
;; cannot sum to an unreasonable total.
(let [cfg (setup-limits cfg)
manifest (-> (read-manifest cfg input)
(validate-manifest))
entries (read-zip-entries input)
max (::max-zip-entries cfg)
_ (when (> (count entries) max)
(ex/raise :type :validation
:code :too-many-zip-entries
:hint (str "zip file has too many entries: " (count entries))
:max max
:found (count entries)))
cfg (-> cfg
(assoc ::entries-index (index-entries entries))
(assoc ::manifest manifest)
(assoc ::bfc/timestamp timestamp))]
(when-not (= "penpot/export-files" (:type manifest))
(ex/raise :type :validation
:code :invalid-binfile-v3-manifest
:hint "unexpected type on manifest"
:manifest manifest))
;; Check if all files referenced on manifest are present
(doseq [{file-id :id features :features} (:files manifest)]
(let [path (str "files/" file-id ".json")]
(when-not (get-zip-entry input path)
(ex/raise :type :validation
:code :invalid-binfile-v3
:hint "some files referenced on manifest not found"
:path path
:file-id file-id))
(cfeat/check-supported-features! features)))
(tap-progress cfg {:section :manifest})
(binding [bfc/*state* (volatile! {:media [] :index {}})]
(if (::bfc/file-id cfg)
(db/tx-run! cfg import-file-and-overwrite*)
(db/tx-run! cfg import-files*)))))
;; --- PUBLIC API
(defn export-files!
"Do the exportation of a specified file in custom penpot binary
format. The cfg carries what `bfc/export-cfg` froze: `::bfc/file-ids`
(the files to export), `::bfc/export-type` (how linked libraries are
handled) and `::bfc/output` (where the package is written).
`::bfc/export-type` determines how linked libraries are handled.
Valid values: `:include-libraries` (include linked libraries),
`:merge-libraries` (embed library assets in the file),
`:detach-libraries` (treat assets as basic objects),
`:link-later` (preserve component metadata for relinking on import)."
[{:keys [::bfc/file-ids ::bfc/output] :as cfg}]
(assert
(and (set? file-ids) (every? uuid? file-ids))
"expected a set of uuid's for `::bfc/file-ids` parameter")
(assert
(satisfies? jio/IOFactory output)
"expected instance of jio/IOFactory for `::bfc/output`")
(let [id (uuid/next)
tp (ct/tpoint)
ab (volatile! false)
cs (volatile! nil)]
(try
(l/info :hint "start exportation" :export-id (str id))
(binding [bfc/*state* (volatile! (bfc/initial-state))]
(with-open [^AutoCloseable output (io/output-stream output)]
(with-open [^AutoCloseable output (ZipOutputStream. output)]
(let [cfg (assoc cfg ::output output)]
(export-files cfg)
(export-storage-objects cfg)))))
(catch java.util.zip.ZipException cause
(vreset! cs cause)
(vreset! ab true)
(throw cause))
(catch java.io.IOException _cause
;; Do nothing, EOF means client closes connection abruptly
(vreset! ab true)
nil)
(catch clojure.lang.ExceptionInfo cause
(if (= :interrupt (:type (ex-data cause)))
(vreset! ab true)
(throw cause)))
(catch Throwable cause
(vreset! cs cause)
(vreset! ab true)
(throw cause))
(finally
(l/info :hint "exportation finished" :export-id (str id)
:elapsed (str (inst-ms (tp)) "ms")
:aborted @ab
:cause @cs)))))
(defn import-files!
[{:keys [::bfc/input] :as cfg}]
(assert
(and (uuid? (::bfc/profile-id cfg))
(uuid? (::bfc/project-id cfg)))
"expected valid profile-id and project-id on `cfg`")
(assert
(io/coercible? input)
"expected instance of jio/IOFactory for `input`")
(let [id (uuid/next)
tp (ct/tpoint)
cs (volatile! nil)]
(l/info :hint "import: started" :id (str id))
(try
(with-open [input (ZipFile. ^File (fs/file input))]
(import-files (assoc cfg ::bfc/input input)))
(catch Throwable cause
(vreset! cs cause)
(throw cause))
(finally
(l/info :hint "import: terminated"
:id (str id)
:elapsed (ct/format-duration (tp))
:error? (some? @cs))))))
(defn get-manifest
"Reads and validates the manifest of a `.penpot` file at `path`.
Runs synchronously on the RPC request thread (before the background
import job exists). Limits are resolved from `cfg` like in the import
job itself, so the read is bounded by the same decompressed-size
limits."
[cfg path]
(let [cfg (setup-limits cfg)]
(with-open [^AutoCloseable input (ZipFile. ^File (fs/file path))]
(-> (read-manifest cfg input)
(validate-manifest)))))