mirror of
https://github.com/penpot/penpot.git
synced 2026-05-17 22:13:39 +00:00
e9d177eae3
The current binfile export process uses a streaming technique. The major problem with the streaming approach is the case when an error happens on the middle of generation, because we have no way to notify the user about the error (because the response is already is sent and contents are streaming directly to the user client/browser). This commit replaces the streaming with temporal files and SSE encoded response for emit the export progress events; once the exportation is finished, a temporal uri to the exported artifact is emited to the user via "end" event and the frontend code will automatically trigger the download. Using the SSE approach removes possible transport timeouts on export large files by sending progress data over the open connection. This commit also removes obsolete code related to old binfile formats.
1051 lines
34 KiB
Clojure
1051 lines
34 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS INC
|
|
|
|
(ns app.binfile.v3
|
|
"A ZIP based binary file exportation"
|
|
(:refer-clojure :exclude [read])
|
|
(:require
|
|
[app.binfile.cleaner :as bfl]
|
|
[app.binfile.common :as bfc]
|
|
[app.binfile.migrations :as bfm]
|
|
[app.common.data :as d]
|
|
[app.common.exceptions :as ex]
|
|
[app.common.features :as cfeat]
|
|
[app.common.files.migrations :as-alias fmg]
|
|
[app.common.json :as json]
|
|
[app.common.logging :as l]
|
|
[app.common.media :as cmedia]
|
|
[app.common.schema :as sm]
|
|
[app.common.thumbnails :as cth]
|
|
[app.common.time :as ct]
|
|
[app.common.types.color :as ctcl]
|
|
[app.common.types.component :as ctc]
|
|
[app.common.types.file :as ctf]
|
|
[app.common.types.page :as ctp]
|
|
[app.common.types.plugins :as ctpg]
|
|
[app.common.types.shape :as cts]
|
|
[app.common.types.tokens-lib :as ctob]
|
|
[app.common.types.typography :as cty]
|
|
[app.common.uuid :as uuid]
|
|
[app.config :as cf]
|
|
[app.db :as db]
|
|
[app.db.sql :as-alias sql]
|
|
[app.storage :as sto]
|
|
[app.storage.impl :as sto.impl]
|
|
[app.util.events :as events]
|
|
[clojure.java.io :as jio]
|
|
[cuerdas.core :as str]
|
|
[datoteka.fs :as fs]
|
|
[datoteka.io :as io])
|
|
(:import
|
|
java.io.File
|
|
java.io.InputStream
|
|
java.io.OutputStreamWriter
|
|
java.lang.AutoCloseable
|
|
java.util.zip.ZipEntry
|
|
java.util.zip.ZipFile
|
|
java.util.zip.ZipOutputStream))
|
|
|
|
;; --- SCHEMA
|
|
|
|
(def ^:private schema:manifest
|
|
[:map {:title "Manifest"}
|
|
[:version ::sm/int]
|
|
[:type :string]
|
|
[:referer {:optional true} :string]
|
|
[:generated-by {:optional true} :string]
|
|
|
|
[:files
|
|
[:vector
|
|
[:map
|
|
[:id ::sm/uuid]
|
|
[:name :string]
|
|
[:features ::cfeat/features]]]]
|
|
|
|
[:relations {:optional true}
|
|
[:vector
|
|
[:tuple ::sm/uuid ::sm/uuid]]]])
|
|
|
|
(def ^:private schema:storage-object
|
|
[:map {:title "StorageObject"}
|
|
[:id ::sm/uuid]
|
|
[:size ::sm/int]
|
|
[:content-type :string]
|
|
[:bucket [::sm/one-of {:format :string} sto/valid-buckets]]
|
|
[:hash {:optional true} :string]])
|
|
|
|
(def ^:private schema:file-thumbnail
|
|
[:map {:title "FileThumbnail"}
|
|
[:file-id ::sm/uuid]
|
|
[:page-id ::sm/uuid]
|
|
[:frame-id ::sm/uuid]
|
|
[:tag :string]
|
|
[:media-id ::sm/uuid]])
|
|
|
|
(def ^:private schema:file
|
|
[:merge
|
|
ctf/schema:file
|
|
[:map [:options {:optional true} ctf/schema:options]]])
|
|
|
|
;; --- HELPERS
|
|
|
|
(defn- default-now
|
|
[o]
|
|
(or o (ct/now)))
|
|
|
|
;; --- ENCODERS
|
|
|
|
(def encode-file
|
|
(sm/encoder schema:file sm/json-transformer))
|
|
|
|
(def encode-page
|
|
(sm/encoder ctp/schema:page sm/json-transformer))
|
|
|
|
(def encode-shape
|
|
(sm/encoder cts/schema:shape sm/json-transformer))
|
|
|
|
(def encode-media
|
|
(sm/encoder ctf/schema:media sm/json-transformer))
|
|
|
|
(def encode-component
|
|
(sm/encoder ctc/schema:component sm/json-transformer))
|
|
|
|
(def encode-color
|
|
(sm/encoder ctcl/schema:library-color sm/json-transformer))
|
|
|
|
(def encode-typography
|
|
(sm/encoder cty/schema:typography sm/json-transformer))
|
|
|
|
(def encode-tokens-lib
|
|
(sm/encoder ctob/schema:tokens-lib sm/json-transformer))
|
|
|
|
(def encode-plugin-data
|
|
(sm/encoder ctpg/schema:plugin-data sm/json-transformer))
|
|
|
|
(def encode-storage-object
|
|
(sm/encoder schema:storage-object sm/json-transformer))
|
|
|
|
(def encode-file-thumbnail
|
|
(sm/encoder schema:file-thumbnail sm/json-transformer))
|
|
|
|
;; --- DECODERS
|
|
|
|
(def decode-manifest
|
|
(sm/decoder schema:manifest sm/json-transformer))
|
|
|
|
(def decode-media
|
|
(sm/decoder ctf/schema:media sm/json-transformer))
|
|
|
|
(def decode-component
|
|
(sm/decoder ctc/schema:component sm/json-transformer))
|
|
|
|
(def decode-color
|
|
(sm/decoder ctcl/schema:library-color sm/json-transformer))
|
|
|
|
(def decode-file
|
|
(sm/decoder schema:file sm/json-transformer))
|
|
|
|
(def decode-page
|
|
(sm/decoder ctp/schema:page sm/json-transformer))
|
|
|
|
(def decode-shape
|
|
(sm/decoder cts/schema:shape sm/json-transformer))
|
|
|
|
(def decode-typography
|
|
(sm/decoder cty/schema:typography sm/json-transformer))
|
|
|
|
(def decode-tokens-lib
|
|
(sm/decoder ctob/schema:tokens-lib sm/json-transformer))
|
|
|
|
(def decode-plugin-data
|
|
(sm/decoder ctpg/schema:plugin-data sm/json-transformer))
|
|
|
|
(def decode-storage-object
|
|
(sm/decoder schema:storage-object sm/json-transformer))
|
|
|
|
(def decode-file-thumbnail
|
|
(sm/decoder schema:file-thumbnail sm/json-transformer))
|
|
|
|
;; --- VALIDATORS
|
|
|
|
(def validate-manifest
|
|
(sm/check-fn schema:manifest))
|
|
|
|
(def validate-file
|
|
(sm/check-fn ctf/schema:file))
|
|
|
|
(def validate-page
|
|
(sm/check-fn ctp/schema:page))
|
|
|
|
(def validate-shape
|
|
(sm/check-fn cts/schema:shape))
|
|
|
|
(def validate-media
|
|
(sm/check-fn ctf/schema:media))
|
|
|
|
(def validate-color
|
|
(sm/check-fn ctcl/schema:library-color))
|
|
|
|
(def validate-component
|
|
(sm/check-fn ctc/schema:component))
|
|
|
|
(def validate-typography
|
|
(sm/check-fn cty/schema:typography))
|
|
|
|
(def validate-tokens-lib
|
|
(sm/check-fn ctob/schema:tokens-lib))
|
|
|
|
(def validate-plugin-data
|
|
(sm/check-fn ctpg/schema:plugin-data))
|
|
|
|
(def validate-storage-object
|
|
(sm/check-fn schema:storage-object))
|
|
|
|
(def validate-file-thumbnail
|
|
(sm/check-fn schema:file-thumbnail))
|
|
|
|
;; --- EXPORT IMPL
|
|
|
|
(defn- write-entry!
|
|
[^ZipOutputStream output ^String path data]
|
|
(.putNextEntry output (ZipEntry. path))
|
|
(let [writer (OutputStreamWriter. output "UTF-8")]
|
|
(json/write writer data :indent true :key-fn json/write-camel-key)
|
|
(.flush writer))
|
|
(.closeEntry output))
|
|
|
|
(defn- get-file
|
|
[{:keys [::bfc/embed-assets ::bfc/include-libraries] :as cfg} file-id]
|
|
|
|
(when (and include-libraries embed-assets)
|
|
(throw (IllegalArgumentException.
|
|
"the `include-libraries` and `embed-assets` are mutally excluding options")))
|
|
|
|
(let [detach? (and (not embed-assets) (not include-libraries))]
|
|
(db/tx-run! cfg (fn [cfg]
|
|
(cond-> (bfc/get-file cfg file-id
|
|
{:realize? true
|
|
:include-deleted? true
|
|
:lock-for-update? true})
|
|
detach?
|
|
(-> (ctf/detach-external-references file-id)
|
|
(dissoc :libraries))
|
|
|
|
embed-assets
|
|
(update :data #(bfc/embed-assets cfg % file-id))
|
|
|
|
:always
|
|
(bfc/clean-file-features))))))
|
|
|
|
(defn- export-storage-objects
|
|
[{:keys [::output] :as cfg}]
|
|
(let [storage (sto/resolve cfg)]
|
|
(doseq [id (-> bfc/*state* deref :storage-objects not-empty)]
|
|
(let [sobject (sto/get-object storage id)
|
|
smeta (meta sobject)
|
|
ext (cmedia/mtype->extension (:content-type smeta))
|
|
path (str "objects/" id ".json")
|
|
params (-> (meta sobject)
|
|
(assoc :id (:id sobject))
|
|
(assoc :size (:size sobject))
|
|
(encode-storage-object))]
|
|
|
|
(write-entry! output path params)
|
|
|
|
(events/tap :progress {:section :storage-object :id id})
|
|
|
|
(with-open [input (sto/get-object-data storage sobject)]
|
|
(.putNextEntry ^ZipOutputStream output (ZipEntry. (str "objects/" id ext)))
|
|
(io/copy input output :size (:size sobject))
|
|
(.closeEntry ^ZipOutputStream output))))))
|
|
|
|
(defn- export-file
|
|
[{:keys [::file-id ::output] :as cfg}]
|
|
(let [file (get-file cfg file-id)
|
|
|
|
media (->> (bfc/get-file-media cfg file)
|
|
(map (fn [media]
|
|
(dissoc media :file-id))))
|
|
|
|
data (:data file)
|
|
typographies (:typographies data)
|
|
components (:components data)
|
|
colors (:colors data)
|
|
tokens-lib (:tokens-lib data)
|
|
|
|
pages (:pages data)
|
|
pages-index (:pages-index data)
|
|
|
|
thumbnails (bfc/get-file-object-thumbnails cfg file-id)]
|
|
|
|
(events/tap :progress {:section :file :id file-id})
|
|
|
|
(vswap! bfc/*state* update :files assoc file-id
|
|
{:id file-id
|
|
:name (:name file)
|
|
:features (:features file)})
|
|
|
|
(let [file (cond-> (select-keys file bfc/file-attrs)
|
|
(:options data)
|
|
(assoc :options (:options data)))
|
|
|
|
file (-> file
|
|
(dissoc :data)
|
|
(dissoc :deleted-at)
|
|
(encode-file))
|
|
|
|
path (str "files/" file-id ".json")]
|
|
(write-entry! output path file))
|
|
|
|
(doseq [[index page-id] (d/enumerate pages)]
|
|
(let [path (str "files/" file-id "/pages/" page-id ".json")
|
|
page (get pages-index page-id)
|
|
objects (:objects page)
|
|
page (-> page
|
|
(dissoc :objects)
|
|
(assoc :index index))
|
|
page (encode-page page)]
|
|
|
|
(write-entry! output path page)
|
|
|
|
(doseq [[shape-id shape] objects]
|
|
(let [path (str "files/" file-id "/pages/" page-id "/" shape-id ".json")
|
|
shape (assoc shape :page-id page-id)
|
|
shape (encode-shape shape)]
|
|
(write-entry! output path shape)))))
|
|
|
|
(vswap! bfc/*state* bfc/collect-storage-objects media)
|
|
(vswap! bfc/*state* bfc/collect-storage-objects thumbnails)
|
|
|
|
(doseq [{:keys [id] :as media} media]
|
|
(let [path (str "files/" file-id "/media/" id ".json")
|
|
media (encode-media media)]
|
|
(write-entry! output path media)))
|
|
|
|
(doseq [thumbnail thumbnails]
|
|
(let [data (cth/parse-object-id (:object-id thumbnail))
|
|
path (str "files/" file-id "/thumbnails/" (:tag data) "/" (:page-id data)
|
|
"/" (:frame-id data) ".json")
|
|
data (-> data
|
|
(assoc :media-id (:media-id thumbnail))
|
|
(encode-file-thumbnail))]
|
|
(write-entry! output path data)))
|
|
|
|
(doseq [[id component] components]
|
|
(let [path (str "files/" file-id "/components/" id ".json")
|
|
component (encode-component component)]
|
|
(write-entry! output path component)))
|
|
|
|
(doseq [[id color] colors]
|
|
(let [path (str "files/" file-id "/colors/" id ".json")
|
|
color (-> (encode-color color)
|
|
(dissoc :file-id))
|
|
color (cond-> color
|
|
(and (contains? color :path)
|
|
(str/empty? (:path color)))
|
|
(dissoc :path))]
|
|
(write-entry! output path color)))
|
|
|
|
(doseq [[id object] typographies]
|
|
(let [path (str "files/" file-id "/typographies/" id ".json")
|
|
typography (encode-typography object)]
|
|
(write-entry! output path typography)))
|
|
|
|
(when (and tokens-lib
|
|
(not (ctob/empty-lib? tokens-lib)))
|
|
(let [path (str "files/" file-id "/tokens.json")
|
|
encoded-tokens (encode-tokens-lib tokens-lib)]
|
|
(write-entry! output path encoded-tokens)))))
|
|
|
|
(defn- export-files
|
|
[{:keys [::bfc/ids ::bfc/include-libraries ::output] :as cfg}]
|
|
(let [ids (into ids (when include-libraries (bfc/get-libraries cfg ids)))
|
|
rels (if include-libraries
|
|
(->> (bfc/get-files-rels cfg ids)
|
|
(mapv (juxt :file-id :library-file-id)))
|
|
[])]
|
|
|
|
(vswap! bfc/*state* assoc :files (d/ordered-map))
|
|
|
|
;; Write all the exporting files
|
|
(doseq [[index file-id] (d/enumerate ids)]
|
|
(-> cfg
|
|
(assoc ::file-id file-id)
|
|
(assoc ::file-seqn index)
|
|
(export-file)))
|
|
|
|
;; Write manifest file
|
|
(let [files (:files @bfc/*state*)
|
|
params {:type "penpot/export-files"
|
|
:version 1
|
|
:generated-by (str "penpot/" (:full cf/version))
|
|
:refer "penpot"
|
|
:files (vec (vals files))
|
|
:relations rels}]
|
|
(write-entry! output "manifest.json" params))))
|
|
|
|
;; --- IMPORT IMPL
|
|
|
|
(defn- read-zip-entries
|
|
[^ZipFile input]
|
|
(into #{} (iterator-seq (.entries input))))
|
|
|
|
(defn- get-zip-entry*
|
|
[^ZipFile input ^String path]
|
|
(.getEntry input path))
|
|
|
|
(defn- get-zip-entry
|
|
[input path]
|
|
(let [entry (get-zip-entry* input path)]
|
|
(when-not entry
|
|
(ex/raise :type :validation
|
|
:code :inconsistent-penpot-file
|
|
:hint "the penpot file seems corrupt, missing underlying zip entry"
|
|
:path path))
|
|
entry))
|
|
|
|
(defn- get-zip-entry-size
|
|
[^ZipEntry entry]
|
|
(.getSize entry))
|
|
|
|
(defn- zip-entry-name
|
|
[^ZipEntry entry]
|
|
(.getName entry))
|
|
|
|
(defn- zip-entry-stream
|
|
^InputStream
|
|
[^ZipFile input ^ZipEntry entry]
|
|
(.getInputStream input entry))
|
|
|
|
(defn- zip-entry-reader
|
|
[^ZipFile input ^ZipEntry entry]
|
|
(-> (zip-entry-stream input entry)
|
|
(io/reader :encoding "UTF-8")))
|
|
|
|
(defn- zip-entry-storage-content
|
|
"Wraps a ZipFile and ZipEntry into a penpot storage compatible
|
|
object and avoid creating temporal objects"
|
|
[input entry]
|
|
(let [hash (delay (->> entry
|
|
(zip-entry-stream input)
|
|
(sto.impl/calculate-hash)))]
|
|
(reify
|
|
sto.impl/IContentObject
|
|
(get-size [_]
|
|
(get-zip-entry-size entry))
|
|
|
|
sto.impl/IContentHash
|
|
(get-hash [_]
|
|
(deref hash))
|
|
|
|
jio/IOFactory
|
|
(make-reader [this opts]
|
|
(jio/make-reader this opts))
|
|
(make-writer [_ _]
|
|
(throw (UnsupportedOperationException. "not implemented")))
|
|
|
|
(make-input-stream [_ _]
|
|
(zip-entry-stream input entry))
|
|
(make-output-stream [_ _]
|
|
(throw (UnsupportedOperationException. "not implemented"))))))
|
|
|
|
(defn- read-manifest
|
|
[^ZipFile input]
|
|
(let [entry (get-zip-entry input "manifest.json")]
|
|
(with-open [^AutoCloseable reader (zip-entry-reader input entry)]
|
|
(let [manifest (json/read reader :key-fn json/read-kebab-key)]
|
|
(decode-manifest manifest)))))
|
|
|
|
(defn- match-media-entry-fn
|
|
[file-id]
|
|
(let [pattern (str "^files/" file-id "/media/([^/]+).json$")
|
|
pattern (re-pattern pattern)]
|
|
(fn [entry]
|
|
(when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
|
|
{:entry entry
|
|
:id (parse-uuid id)}))))
|
|
|
|
(defn- match-color-entry-fn
|
|
[file-id]
|
|
(let [pattern (str "^files/" file-id "/colors/([^/]+).json$")
|
|
pattern (re-pattern pattern)]
|
|
(fn [entry]
|
|
(when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
|
|
{:entry entry
|
|
:id (parse-uuid id)}))))
|
|
|
|
(defn- match-component-entry-fn
|
|
[file-id]
|
|
(let [pattern (str "^files/" file-id "/components/([^/]+).json$")
|
|
pattern (re-pattern pattern)]
|
|
(fn [entry]
|
|
(when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
|
|
{:entry entry
|
|
:id (parse-uuid id)}))))
|
|
|
|
(defn- match-typography-entry-fn
|
|
[file-id]
|
|
(let [pattern (str "^files/" file-id "/typographies/([^/]+).json$")
|
|
pattern (re-pattern pattern)]
|
|
(fn [entry]
|
|
(when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
|
|
{:entry entry
|
|
:id (parse-uuid id)}))))
|
|
|
|
(defn- match-tokens-lib-entry-fn
|
|
[file-id]
|
|
(let [pattern (str "^files/" file-id "/tokens.json$")
|
|
pattern (re-pattern pattern)]
|
|
(fn [entry]
|
|
(when-let [[_] (re-matches pattern (zip-entry-name entry))]
|
|
{:entry entry}))))
|
|
|
|
(defn- match-thumbnail-entry-fn
|
|
[file-id]
|
|
(let [pattern (str "^files/" file-id "/thumbnails/([^/]+)/([^/]+)/([^/]+).json$")
|
|
pattern (re-pattern pattern)]
|
|
(fn [entry]
|
|
(when-let [[_ tag page-id frame-id] (re-matches pattern (zip-entry-name entry))]
|
|
{:entry entry
|
|
:tag tag
|
|
:page-id (parse-uuid page-id)
|
|
:frame-id (parse-uuid frame-id)
|
|
:file-id file-id}))))
|
|
|
|
(defn- match-page-entry-fn
|
|
[file-id]
|
|
(let [pattern (str "^files/" file-id "/pages/([^/]+).json$")
|
|
pattern (re-pattern pattern)]
|
|
(fn [entry]
|
|
(when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
|
|
{:entry entry
|
|
:id (parse-uuid id)}))))
|
|
|
|
(defn- match-shape-entry-fn
|
|
[file-id page-id]
|
|
(let [pattern (str "^files/" file-id "/pages/" page-id "/([^/]+).json$")
|
|
pattern (re-pattern pattern)]
|
|
(fn [entry]
|
|
(when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
|
|
{:entry entry
|
|
:page-id page-id
|
|
:id (parse-uuid id)}))))
|
|
|
|
(defn- match-storage-entry-fn
|
|
[]
|
|
(let [pattern "^objects/([^/]+).json$"
|
|
pattern (re-pattern pattern)]
|
|
(fn [entry]
|
|
(when-let [[_ id] (re-matches pattern (zip-entry-name entry))]
|
|
{:entry entry
|
|
:id (parse-uuid id)}))))
|
|
|
|
(defn- read-entry
|
|
[^ZipFile input entry]
|
|
(with-open [^AutoCloseable reader (zip-entry-reader input entry)]
|
|
(json/read reader :key-fn json/read-kebab-key)))
|
|
|
|
(defn- read-plain-entry
|
|
[^ZipFile input entry]
|
|
(with-open [^AutoCloseable reader (zip-entry-reader input entry)]
|
|
(json/read reader)))
|
|
|
|
(defn- read-file
|
|
[{:keys [::bfc/input ::bfc/timestamp]} file-id]
|
|
(let [path (str "files/" file-id ".json")
|
|
entry (get-zip-entry input path)]
|
|
(-> (read-entry input entry)
|
|
(decode-file)
|
|
(update :revn d/nilv 1)
|
|
(update :created-at d/nilv timestamp)
|
|
(update :modified-at d/nilv timestamp)
|
|
(validate-file))))
|
|
|
|
(defn- read-file-plugin-data
|
|
[{:keys [::bfc/input]} file-id]
|
|
(let [path (str "files/" file-id "/plugin-data.json")
|
|
entry (get-zip-entry* input path)]
|
|
(some->> entry
|
|
(read-entry input)
|
|
(decode-plugin-data)
|
|
(validate-plugin-data))))
|
|
|
|
(defn- read-file-media
|
|
[{:keys [::bfc/input ::entries]} file-id]
|
|
(->> (keep (match-media-entry-fn file-id) entries)
|
|
(reduce (fn [result {:keys [id entry]}]
|
|
(let [object (->> (read-entry input entry)
|
|
(decode-media)
|
|
(validate-media))
|
|
object (-> object
|
|
(assoc :file-id file-id)
|
|
(update :created-at default-now)
|
|
;; FIXME: this is set default to true for
|
|
;; setting a value, this prop is no longer
|
|
;; relevant;
|
|
(assoc :is-local true))]
|
|
(if (= id (:id object))
|
|
(conj result object)
|
|
result)))
|
|
[])
|
|
(not-empty)))
|
|
|
|
(defn- read-file-colors
|
|
[{:keys [::bfc/input ::entries]} file-id]
|
|
(->> (keep (match-color-entry-fn file-id) entries)
|
|
(reduce (fn [result {:keys [id entry]}]
|
|
(let [object (->> (read-entry input entry)
|
|
(decode-color)
|
|
(validate-color))]
|
|
(if (= id (:id object))
|
|
(assoc result id object)
|
|
result)))
|
|
{})
|
|
(not-empty)))
|
|
|
|
(defn- read-file-components
|
|
[{:keys [::bfc/input ::entries]} file-id]
|
|
(let [clean-component-post-decode
|
|
(fn [component]
|
|
(d/update-when component :objects
|
|
(fn [objects]
|
|
(reduce-kv (fn [objects id shape]
|
|
(assoc objects id (bfl/clean-shape-post-decode shape)))
|
|
objects
|
|
objects))))
|
|
clean-component-pre-decode
|
|
(fn [component]
|
|
(d/update-when component :objects
|
|
(fn [objects]
|
|
(reduce-kv (fn [objects id shape]
|
|
(assoc objects id (bfl/clean-shape-pre-decode shape)))
|
|
objects
|
|
objects))))]
|
|
|
|
(->> (keep (match-component-entry-fn file-id) entries)
|
|
(reduce (fn [result {:keys [id entry]}]
|
|
(let [object (->> (read-entry input entry)
|
|
(clean-component-pre-decode)
|
|
(decode-component)
|
|
(clean-component-post-decode))]
|
|
(if (= id (:id object))
|
|
(assoc result id object)
|
|
result)))
|
|
{})
|
|
(not-empty))))
|
|
|
|
(defn- read-file-typographies
|
|
[{:keys [::bfc/input ::entries]} file-id]
|
|
(->> (keep (match-typography-entry-fn file-id) entries)
|
|
(reduce (fn [result {:keys [id entry]}]
|
|
(let [object (->> (read-entry input entry)
|
|
(decode-typography)
|
|
(validate-typography))]
|
|
(if (= id (:id object))
|
|
(assoc result id object)
|
|
result)))
|
|
{})
|
|
(not-empty)))
|
|
|
|
(defn- read-file-tokens-lib
|
|
[{:keys [::bfc/input ::entries]} file-id]
|
|
(when-let [entry (d/seek (match-tokens-lib-entry-fn file-id) entries)]
|
|
(->> (read-plain-entry input entry)
|
|
(decode-tokens-lib)
|
|
(validate-tokens-lib))))
|
|
|
|
(defn- read-file-shapes
|
|
[{:keys [::bfc/input ::entries] :as cfg} file-id page-id]
|
|
(->> (keep (match-shape-entry-fn file-id page-id) entries)
|
|
(reduce (fn [result {:keys [id entry]}]
|
|
(let [object (->> (read-entry input entry)
|
|
(bfl/clean-shape-pre-decode)
|
|
(decode-shape)
|
|
(bfl/clean-shape-post-decode))]
|
|
(if (= id (:id object))
|
|
(assoc result id object)
|
|
result)))
|
|
{})
|
|
(not-empty)))
|
|
|
|
(defn- read-file-pages
|
|
[{:keys [::bfc/input ::entries] :as cfg} file-id]
|
|
(->> (keep (match-page-entry-fn file-id) entries)
|
|
(keep (fn [{:keys [id entry]}]
|
|
(let [page (->> (read-entry input entry)
|
|
(decode-page))
|
|
page (dissoc page :options)]
|
|
(when (= id (:id page))
|
|
(let [objects (read-file-shapes cfg file-id id)]
|
|
(assoc page :objects objects))))))
|
|
(sort-by :index)
|
|
(reduce (fn [result {:keys [id] :as page}]
|
|
(assoc result id (dissoc page :index)))
|
|
(d/ordered-map))))
|
|
|
|
(defn- read-file-thumbnails
|
|
[{:keys [::bfc/input ::entries] :as cfg} file-id]
|
|
(->> (keep (match-thumbnail-entry-fn file-id) entries)
|
|
(reduce (fn [result {:keys [page-id frame-id tag entry]}]
|
|
(let [object (->> (read-entry input entry)
|
|
(decode-file-thumbnail)
|
|
(validate-file-thumbnail))]
|
|
(if (and (= frame-id (:frame-id object))
|
|
(= page-id (:page-id object))
|
|
(= tag (:tag object)))
|
|
(conj result object)
|
|
result)))
|
|
[])
|
|
(not-empty)))
|
|
|
|
(defn- read-file-data
|
|
[cfg file-id]
|
|
(let [colors (read-file-colors cfg file-id)
|
|
typographies (read-file-typographies cfg file-id)
|
|
tokens-lib (read-file-tokens-lib cfg file-id)
|
|
components (read-file-components cfg file-id)
|
|
plugin-data (read-file-plugin-data cfg file-id)
|
|
pages (read-file-pages cfg file-id)]
|
|
{:pages (-> pages keys vec)
|
|
:pages-index (into {} pages)
|
|
:colors colors
|
|
:typographies typographies
|
|
:tokens-lib tokens-lib
|
|
:components components
|
|
:plugin-data plugin-data}))
|
|
|
|
(defn- import-file
|
|
[{:keys [::db/conn ::bfc/project-id] :as cfg} {file-id :id file-name :name}]
|
|
(let [file-id' (bfc/lookup-index file-id)
|
|
file (read-file cfg file-id)
|
|
media (read-file-media cfg file-id)
|
|
thumbnails (read-file-thumbnails cfg file-id)]
|
|
|
|
(l/dbg :hint "processing file"
|
|
:id (str file-id')
|
|
:prev-id (str file-id)
|
|
:features (str/join "," (:features file))
|
|
:version (:version file)
|
|
::l/sync? true)
|
|
|
|
(vswap! bfc/*state* update :index bfc/update-index media :id)
|
|
|
|
(events/tap :progress {:section :media :file-id file-id})
|
|
|
|
(doseq [item media]
|
|
(let [params (-> item
|
|
(update :id bfc/lookup-index)
|
|
(assoc :file-id file-id')
|
|
(d/update-when :media-id bfc/lookup-index)
|
|
(d/update-when :thumbnail-id bfc/lookup-index))]
|
|
|
|
(l/dbg :hint "inserting media object"
|
|
:file-id (str file-id')
|
|
:id (str (:id params))
|
|
:media-id (str (:media-id params))
|
|
:thumbnail-id (str (:thumbnail-id params))
|
|
:old-id (str (:id item))
|
|
::l/sync? true)
|
|
|
|
(db/insert! conn :file-media-object params
|
|
::db/on-conflict-do-nothing? (::bfc/overwrite cfg))))
|
|
|
|
(events/tap :progress {:section :thumbnails :file-id file-id})
|
|
|
|
(doseq [item thumbnails]
|
|
(let [media-id (bfc/lookup-index (:media-id item))
|
|
object-id (-> (assoc item :file-id file-id')
|
|
(cth/fmt-object-id))
|
|
params {:file-id file-id'
|
|
:object-id object-id
|
|
:tag (:tag item)
|
|
:media-id media-id}]
|
|
|
|
(l/dbg :hint "inserting object thumbnail"
|
|
:file-id (str file-id')
|
|
:media-id (str media-id)
|
|
::l/sync? true)
|
|
|
|
(db/insert! conn :file-tagged-object-thumbnail params
|
|
::db/on-conflict-do-nothing? true)))
|
|
|
|
(events/tap :progress {:section :file :file-id file-id})
|
|
|
|
(let [data (-> (read-file-data cfg file-id)
|
|
(d/without-nils)
|
|
(assoc :id file-id')
|
|
(cond-> (:options file)
|
|
(assoc :options (:options file))))
|
|
|
|
file (-> (select-keys file bfc/file-attrs)
|
|
(assoc :id file-id')
|
|
(assoc :data data)
|
|
(assoc :name file-name)
|
|
(assoc :project-id project-id)
|
|
(dissoc :options))
|
|
|
|
file (bfc/process-file cfg file)
|
|
file (ctf/check-file file)]
|
|
|
|
(bfm/register-pending-migrations! cfg file)
|
|
(bfc/save-file! cfg file)
|
|
|
|
file-id')))
|
|
|
|
(defn- import-file-relations
|
|
[{:keys [::db/conn ::manifest ::bfc/timestamp] :as cfg}]
|
|
(events/tap :progress {:section :relations})
|
|
(doseq [[file-id libr-id] (:relations manifest)]
|
|
|
|
(let [file-id (bfc/lookup-index file-id)
|
|
libr-id (bfc/lookup-index libr-id)]
|
|
|
|
(when (and file-id libr-id)
|
|
(l/dbg :hint "create file library link"
|
|
:file-id (str file-id)
|
|
:lib-id (str libr-id)
|
|
::l/sync? true)
|
|
(db/insert! conn :file-library-rel
|
|
{:synced-at timestamp
|
|
:file-id file-id
|
|
:library-file-id libr-id})))))
|
|
|
|
(defn- import-storage-objects
|
|
[{:keys [::bfc/input ::entries ::bfc/timestamp] :as cfg}]
|
|
(events/tap :progress {:section :storage-objects})
|
|
|
|
(let [storage (sto/resolve cfg)
|
|
entries (keep (match-storage-entry-fn) entries)]
|
|
|
|
(doseq [{:keys [id entry]} entries]
|
|
(let [object (->> (read-entry input entry)
|
|
(decode-storage-object)
|
|
(validate-storage-object))
|
|
|
|
ext (cmedia/mtype->extension (:content-type object))
|
|
path (str "objects/" id ext)
|
|
content (->> path
|
|
(get-zip-entry input)
|
|
(zip-entry-storage-content input))]
|
|
|
|
(when (not= (:size object) (sto/get-size content))
|
|
(ex/raise :type :validation
|
|
:code :inconsistent-penpot-file
|
|
:hint "found corrupted storage object: size does not match"
|
|
:path path
|
|
:expected-size (:size object)
|
|
:found-size (sto/get-size content)))
|
|
|
|
(when-let [hash (get object :hash)]
|
|
(when (not= hash (sto/get-hash content))
|
|
(ex/raise :type :validation
|
|
:code :inconsistent-penpot-file
|
|
:hint "found corrupted storage object: hash does not match"
|
|
:path path
|
|
:expected-hash (:hash object)
|
|
:found-hash (sto/get-hash content))))
|
|
|
|
(let [params (-> object
|
|
(dissoc :id :size)
|
|
(assoc ::sto/content content)
|
|
(assoc ::sto/deduplicate? true)
|
|
(assoc ::sto/touched-at timestamp))
|
|
sobject (sto/put-object! storage params)]
|
|
|
|
(l/dbg :hint "persisted storage object"
|
|
:id (str (:id sobject))
|
|
:prev-id (str id)
|
|
:bucket (:bucket params)
|
|
::l/sync? true)
|
|
|
|
(vswap! bfc/*state* update :index assoc id (:id sobject)))))))
|
|
|
|
(defn- import-files*
|
|
[{:keys [::manifest] :as cfg}]
|
|
(bfc/disable-database-timeouts! cfg)
|
|
|
|
(vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id)
|
|
|
|
(import-storage-objects cfg)
|
|
|
|
(let [files (get manifest :files)
|
|
result (reduce (fn [result {:keys [id] :as file}]
|
|
(let [name' (get file :name)
|
|
name' (if (map? name)
|
|
(get name id)
|
|
name')
|
|
file (assoc file :name name')]
|
|
(conj result (import-file cfg file))))
|
|
[]
|
|
files)]
|
|
|
|
(import-file-relations cfg)
|
|
(bfm/apply-pending-migrations! cfg)
|
|
|
|
result))
|
|
|
|
(defn- import-file-and-overwrite*
|
|
[{:keys [::manifest ::bfc/file-id] :as cfg}]
|
|
|
|
(when (not= 1 (count (:files manifest)))
|
|
(ex/raise :type :validation
|
|
:code :invalid-condition
|
|
:hint "unable to perform in-place update with binfile containing more than 1 file"
|
|
:manifest manifest))
|
|
|
|
(bfc/disable-database-timeouts! cfg)
|
|
|
|
(let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true)
|
|
file (first (get manifest :files))
|
|
cfg (assoc cfg ::bfc/overwrite true)]
|
|
|
|
(vswap! bfc/*state* update :index assoc (:id file) file-id)
|
|
|
|
(binding [bfc/*options* cfg
|
|
bfc/*reference-file* ref-file]
|
|
|
|
(import-storage-objects cfg)
|
|
(import-file cfg file)
|
|
|
|
(bfc/invalidate-thumbnails cfg file-id)
|
|
(bfm/apply-pending-migrations! cfg)
|
|
|
|
[file-id])))
|
|
|
|
(defn- import-files
|
|
[{:keys [::bfc/timestamp ::bfc/input] :or {timestamp (ct/now)} :as cfg}]
|
|
|
|
(assert (instance? ZipFile input) "expected zip file")
|
|
(assert (ct/inst? timestamp) "expected valid instant")
|
|
|
|
(let [manifest (-> (read-manifest input)
|
|
(validate-manifest))
|
|
entries (read-zip-entries input)
|
|
cfg (-> cfg
|
|
(assoc ::entries entries)
|
|
(assoc ::manifest manifest)
|
|
(assoc ::bfc/timestamp timestamp))]
|
|
|
|
(when-not (= "penpot/export-files" (:type manifest))
|
|
(ex/raise :type :validation
|
|
:code :invalid-binfile-v3-manifest
|
|
:hint "unexpected type on manifest"
|
|
:manifest manifest))
|
|
|
|
;; Check if all files referenced on manifest are present
|
|
(doseq [{file-id :id features :features} (:files manifest)]
|
|
(let [path (str "files/" file-id ".json")]
|
|
|
|
(when-not (get-zip-entry input path)
|
|
(ex/raise :type :validation
|
|
:code :invalid-binfile-v3
|
|
:hint "some files referenced on manifest not found"
|
|
:path path
|
|
:file-id file-id))
|
|
|
|
(cfeat/check-supported-features! features)))
|
|
|
|
(events/tap :progress {:section :manifest})
|
|
|
|
(binding [bfc/*state* (volatile! {:media [] :index {}})]
|
|
(if (::bfc/file-id cfg)
|
|
(db/tx-run! cfg import-file-and-overwrite*)
|
|
(db/tx-run! cfg import-files*)))))
|
|
|
|
;; --- PUBLIC API
|
|
|
|
(defn export-files!
|
|
"Do the exportation of a specified file in custom penpot binary
|
|
format. There are some options available for customize the output:
|
|
|
|
`::bfc/include-libraries`: additionally to the specified file, all the
|
|
linked libraries also will be included (including transitive
|
|
dependencies).
|
|
|
|
`::bfc/embed-assets`: instead of including the libraries, embed in the
|
|
same file library all assets used from external libraries."
|
|
|
|
[{:keys [::bfc/ids] :as cfg} output]
|
|
|
|
(assert
|
|
(and (set? ids) (every? uuid? ids))
|
|
"expected a set of uuid's for `::bfc/ids` parameter")
|
|
|
|
(assert
|
|
(satisfies? jio/IOFactory output)
|
|
"expected instance of jio/IOFactory for `input`")
|
|
|
|
(let [id (uuid/next)
|
|
tp (ct/tpoint)
|
|
ab (volatile! false)
|
|
cs (volatile! nil)]
|
|
(try
|
|
(l/info :hint "start exportation" :export-id (str id))
|
|
(binding [bfc/*state* (volatile! (bfc/initial-state))]
|
|
(with-open [^AutoCloseable output (io/output-stream output)]
|
|
(with-open [^AutoCloseable output (ZipOutputStream. output)]
|
|
(let [cfg (assoc cfg ::output output)]
|
|
(export-files cfg)
|
|
(export-storage-objects cfg)))))
|
|
|
|
(catch java.util.zip.ZipException cause
|
|
(vreset! cs cause)
|
|
(vreset! ab true)
|
|
(throw cause))
|
|
|
|
(catch java.io.IOException _cause
|
|
;; Do nothing, EOF means client closes connection abruptly
|
|
(vreset! ab true)
|
|
nil)
|
|
|
|
(catch Throwable cause
|
|
(vreset! cs cause)
|
|
(vreset! ab true)
|
|
(throw cause))
|
|
|
|
(finally
|
|
(l/info :hint "exportation finished" :export-id (str id)
|
|
:elapsed (str (inst-ms (tp)) "ms")
|
|
:aborted @ab
|
|
:cause @cs)))))
|
|
|
|
(defn import-files!
|
|
[{:keys [::bfc/input] :as cfg}]
|
|
|
|
(assert
|
|
(and (uuid? (::bfc/profile-id cfg))
|
|
(uuid? (::bfc/project-id cfg)))
|
|
"expected valid profile-id and project-id on `cfg`")
|
|
|
|
(assert
|
|
(io/coercible? input)
|
|
"expected instance of jio/IOFactory for `input`")
|
|
|
|
(let [id (uuid/next)
|
|
tp (ct/tpoint)
|
|
cs (volatile! nil)]
|
|
|
|
(l/info :hint "import: started" :id (str id))
|
|
(try
|
|
(with-open [input (ZipFile. ^File (fs/file input))]
|
|
(import-files (assoc cfg ::bfc/input input)))
|
|
|
|
(catch Throwable cause
|
|
(vreset! cs cause)
|
|
(throw cause))
|
|
|
|
(finally
|
|
(l/info :hint "import: terminated"
|
|
:id (str id)
|
|
:elapsed (ct/format-duration (tp))
|
|
:error? (some? @cs))))))
|
|
|
|
(defn get-manifest
|
|
[path]
|
|
(with-open [^AutoCloseable input (ZipFile. ^File (fs/file path))]
|
|
(-> (read-manifest input)
|
|
(validate-manifest))))
|