mirror of
https://github.com/penpot/penpot.git
synced 2026-11-19 08:30:42 +00:00
1244 lines
44 KiB
Clojure
1244 lines
44 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
|
|
|
(ns app.binfile.common
|
|
"A binfile related file processing common code, used for different
|
|
binfile format implementations and management rpc methods."
|
|
(:require
|
|
[app.common.data :as d]
|
|
[app.common.exceptions :as ex]
|
|
[app.common.features :as cfeat]
|
|
[app.common.files.helpers :as cfh]
|
|
[app.common.files.migrations :as fmg]
|
|
[app.common.files.tokens :as cfo]
|
|
[app.common.files.validate :as fval]
|
|
[app.common.logging :as l]
|
|
[app.common.media :as cm]
|
|
[app.common.schema :as sm]
|
|
[app.common.time :as ct]
|
|
[app.common.types.file :as ctf]
|
|
[app.common.uuid :as uuid]
|
|
[app.common.weak :as weak]
|
|
[app.config :as cf]
|
|
[app.db :as db]
|
|
[app.features.fdata :as fdata]
|
|
[app.features.file-migrations :as fmigr]
|
|
[app.loggers.audit :as-alias audit]
|
|
[app.loggers.webhooks :as-alias webhooks]
|
|
[app.media.svg :as svg]
|
|
[app.storage :as sto]
|
|
[app.util.blob :as blob]
|
|
[app.util.pointer-map :as pmap]
|
|
[app.worker :as-alias wrk]
|
|
[clojure.set :as set]
|
|
[cuerdas.core :as str]
|
|
[datoteka.fs :as fs]
|
|
[datoteka.io :as io]))
|
|
|
|
(set! *warn-on-reflection* true)
|
|
|
|
(def ^:dynamic *state* nil)
|
|
(def ^:dynamic *options* nil)
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;; DEFAULTS
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
;; Threshold in MiB when we pass from using
|
|
;; in-memory byte-array's to use temporal files.
|
|
(def temp-file-threshold
|
|
(* 1024 1024 2))
|
|
|
|
;; Maximum size allowed for a single binary entry during binfile
|
|
;; import: 100MiB. Covers the storage blobs (`objects/` entries in v3,
|
|
;; streams in v1), whose declared size and hash are verified against the
|
|
;; imported bytes. Legitimate media objects fit comfortably below this;
|
|
;; anything larger is rejected instead of being buffered into memory.
|
|
(def ^:const default-max-binary-entry-size
|
|
(* 1024 1024 100))
|
|
|
|
;; Maximum decompressed size allowed for a single JSON/text zip entry
|
|
;; (manifest, files, pages, shapes, colors, components, typographies,
|
|
;; tokens, plugin-data) during binfile import: 20MiB. Legitimate entries
|
|
;; are KB-sized, so this is deliberately much lower than
|
|
;; default-max-binary-entry-size and bounds the DEFLATE amplification of any
|
|
;; single entry.
|
|
(def ^:const default-max-text-entry-size
|
|
(* 1024 1024 20))
|
|
|
|
;; Maximum total decompressed size allowed for all JSON/text zip entries
|
|
;; combined within a single import job: 500MiB. Bounds the case where many
|
|
;; entries, each individually under default-max-text-entry-size, still sum
|
|
;; to an unreasonable total. Set this high because legitimate giant
|
|
;; exports exist: hundreds of thousands of KB-sized entries (pages,
|
|
;; shapes) that never trip the per-entry cap but sum past 400MiB. The
|
|
;; per-entry cap still bounds single-entry DEFLATE amplification, so this
|
|
;; only moves the cumulative ceiling.
|
|
(def ^:const default-max-text-total-size
|
|
(* 1024 1024 500))
|
|
|
|
;; Maximum number of entries allowed in the import zip: 500,000.
|
|
(def ^:const default-max-zip-entries
|
|
(* 500 1000))
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(declare get-resolved-file-libraries)
|
|
(declare update-file!)
|
|
|
|
(def file-attrs
|
|
(sm/keys ctf/schema:file))
|
|
|
|
(defn parse-file-format
|
|
[template]
|
|
(assert (fs/path? template) "expected InputStream for `template`")
|
|
|
|
(with-open [^java.lang.AutoCloseable input (io/input-stream template)]
|
|
(let [buffer (byte-array 4)]
|
|
(io/read-to-buffer input buffer)
|
|
(if (and (= (aget buffer 0) 80)
|
|
(= (aget buffer 1) 75)
|
|
(= (aget buffer 2) 3)
|
|
(= (aget buffer 3) 4))
|
|
:binfile-v3
|
|
:binfile-v1))))
|
|
|
|
(def xf-map-id
|
|
(map :id))
|
|
|
|
(def xf-map-media-id
|
|
(comp
|
|
(mapcat (juxt :media-id
|
|
:thumbnail-id
|
|
:woff1-file-id
|
|
:woff2-file-id
|
|
:ttf-file-id
|
|
:otf-file-id))
|
|
(filter uuid?)))
|
|
|
|
(def into-vec
|
|
(fnil into []))
|
|
|
|
(def conj-vec
|
|
(fnil conj []))
|
|
|
|
(defn initial-state
|
|
[]
|
|
{:storage-objects #{}
|
|
:files #{}})
|
|
|
|
(defn collect-storage-objects
|
|
[state items]
|
|
(update state :storage-objects into xf-map-media-id items))
|
|
|
|
(defn lookup-index
|
|
[id]
|
|
(when id
|
|
(let [val (get-in @*state* [:index id])]
|
|
(l/trc :fn "lookup-index" :id (str id) :result (some-> val str) ::l/sync? true)
|
|
(or val id))))
|
|
|
|
(defn remap-id
|
|
[item key]
|
|
(cond-> item
|
|
(contains? item key)
|
|
(update key lookup-index)))
|
|
|
|
(defn- index-object
|
|
[index obj & attrs]
|
|
(reduce (fn [index attr-fn]
|
|
(let [old-id (attr-fn obj)
|
|
new-id (uuid/next)]
|
|
(assoc index old-id new-id)))
|
|
index
|
|
attrs))
|
|
|
|
(defn update-index
|
|
([coll]
|
|
(update-index {} coll identity))
|
|
([index coll]
|
|
(update-index index coll identity))
|
|
([index coll attr]
|
|
(reduce #(index-object %1 %2 attr) index coll)))
|
|
|
|
(defn- decode-row-features
|
|
[{:keys [features] :as row}]
|
|
(when row
|
|
(cond-> row
|
|
(db/pgarray? features) (assoc :features (db/decode-pgarray features #{})))))
|
|
|
|
(def sql:get-minimal-file
|
|
"SELECT f.id,
|
|
f.revn,
|
|
f.modified_at,
|
|
f.deleted_at
|
|
FROM file AS f
|
|
WHERE f.id = ?")
|
|
|
|
(defn get-minimal-file
|
|
[cfg id & {:as opts}]
|
|
(db/get-with-sql cfg [sql:get-minimal-file id] opts))
|
|
|
|
(def sql:files-with-data
|
|
"SELECT f.id,
|
|
f.project_id,
|
|
f.created_at,
|
|
f.modified_at,
|
|
f.deleted_at,
|
|
f.name,
|
|
f.is_shared,
|
|
f.has_media_trimmed,
|
|
f.revn,
|
|
f.data AS legacy_data,
|
|
f.ignore_sync_until,
|
|
f.comment_thread_seqn,
|
|
f.features,
|
|
f.version,
|
|
f.vern,
|
|
p.team_id,
|
|
coalesce(fd.backend, 'legacy-db') AS backend,
|
|
fd.metadata AS metadata,
|
|
fd.storage_object_id AS storage_object_id,
|
|
fd.data AS data
|
|
FROM file AS f
|
|
LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id)
|
|
INNER JOIN project AS p ON (p.id = f.project_id)")
|
|
|
|
(def sql:get-file
|
|
(str sql:files-with-data " WHERE f.id = ?"))
|
|
|
|
(def sql:get-file-without-data
|
|
"SELECT f.id,
|
|
f.project_id,
|
|
f.created_at,
|
|
f.modified_at,
|
|
f.deleted_at,
|
|
f.name,
|
|
f.is_shared,
|
|
f.has_media_trimmed,
|
|
f.revn,
|
|
f.ignore_sync_until,
|
|
f.comment_thread_seqn,
|
|
f.features,
|
|
f.version,
|
|
f.vern,
|
|
p.team_id
|
|
FROM file AS f
|
|
INNER JOIN project AS p ON (p.id = f.project_id)
|
|
WHERE f.id = ?")
|
|
|
|
(defn- migrate-file
|
|
"Apply the pending migrations to the file.
|
|
|
|
With `persist-after-migration?` the migrated map is also persisted, which is
|
|
only safe from a map read under the `file` row lock (see `get-file*`). The
|
|
returned map is then the reattached one: decoded data where the pages the
|
|
migration did not change keep their pointers and their fragment ids, plus
|
|
the `:storage-object-id` of the row the write left, so a caller that
|
|
persists again touches the current object. Without `persist-after-migration?`
|
|
the realized map is returned, which is what a read-only context can do.
|
|
`get-file*` realizes the result when the caller asks for it."
|
|
[cfg {:keys [id] :as file} {:keys [persist-after-migration?]}]
|
|
(fdata/with-tracked-pointers cfg id
|
|
(let [libs (delay (get-resolved-file-libraries cfg file))
|
|
;; Capture the pointer layout before realizing: the migration
|
|
;; runs on plain data, and unchanged positions must keep their
|
|
;; original fragment ids instead of being re-wrapped with new
|
|
;; ones on persist.
|
|
original (fdata/original-pointers (:data file))
|
|
objects-map-before? (contains? (:features file) "fdata/objects-map")
|
|
realized (-> (fdata/realize cfg file)
|
|
(fmg/migrate-file libs))
|
|
objects-map-added? (and (not objects-map-before?)
|
|
(contains? (:features realized) "fdata/objects-map"))]
|
|
|
|
(if persist-after-migration?
|
|
(let [file (if objects-map-added?
|
|
realized
|
|
(fdata/reattach-unchanged original realized))
|
|
written (update-file! cfg file)]
|
|
;; When the file is migrated, we break the rule of not
|
|
;; performing mutations on get operations and update the file
|
|
;; with all migrations applied. The reattached map is also what
|
|
;; the read returns: the caller realizes it only when it asks.
|
|
(assoc (fmigr/resolve-applied-migrations cfg file)
|
|
:storage-object-id (:storage-object-id written)))
|
|
realized))))
|
|
|
|
(defn- fetch-row
|
|
"Read the file row. With `load-data?` the file data columns come along, but
|
|
they are not resolved nor decoded (see `attach-file-data`). With
|
|
`load-migrations?` the applied migrations are resolved, which is what
|
|
`need-migration?` expects. A skip-locked read answers nil instead of raising
|
|
when the row is gone."
|
|
[{:keys [::db/conn] :as cfg} id
|
|
{:keys [include-deleted?
|
|
skip-locked?
|
|
lock-for-update?
|
|
lock-for-share?
|
|
throw-if-not-exists?
|
|
load-migrations?
|
|
load-data?]}]
|
|
|
|
(let [sql (if load-data?
|
|
sql:get-file
|
|
sql:get-file-without-data)
|
|
sql (cond
|
|
lock-for-update?
|
|
(str sql " FOR UPDATE of f")
|
|
|
|
lock-for-share?
|
|
(str sql " FOR SHARE of f")
|
|
|
|
:else
|
|
sql)
|
|
|
|
sql (if (and skip-locked? (or lock-for-share? lock-for-update?))
|
|
(str sql " SKIP LOCKED")
|
|
sql)
|
|
|
|
file (db/get-with-sql conn [sql id] {::db/throw-if-not-exists false
|
|
::db/remove-deleted (not include-deleted?)})
|
|
file (some-> file
|
|
(d/update-when :features db/decode-pgarray #{})
|
|
(d/update-when :metadata fdata/decode-metadata))
|
|
file (if (and file load-migrations?)
|
|
(fmigr/resolve-applied-migrations cfg file)
|
|
file)]
|
|
|
|
(or file
|
|
(when-not (or skip-locked? (not throw-if-not-exists?))
|
|
(ex/raise :type :not-found
|
|
:code :object-not-found
|
|
:hint "database object not found"
|
|
:table :file
|
|
:file-id id)))))
|
|
|
|
|
|
(defn- attach-file-data
|
|
"Resolve the data of an already read row and decode it when `decode?`. It
|
|
completes the map it receives instead of reading the row again. It does not
|
|
realize: `get-file*` owns that step, once per path."
|
|
[cfg file {:keys [decode?]}]
|
|
(let [file (fdata/resolve-file-data cfg file)]
|
|
(if decode?
|
|
(fdata/decode-file-data cfg file)
|
|
file)))
|
|
|
|
(defn- fetch-file
|
|
"Read the row with its data resolved and decoded, under the requested lock."
|
|
[cfg id options]
|
|
(when-let [file (fetch-row cfg id options)]
|
|
(attach-file-data cfg file options)))
|
|
|
|
(def ^:private
|
|
default-get-file-options
|
|
{:load-migrations? true
|
|
:throw-if-not-exists? true
|
|
:persist-after-migration? true
|
|
:load-data? true
|
|
:migrate? true
|
|
:decode? true})
|
|
|
|
(defn- get-file*
|
|
[{:keys [::db/conn] :as cfg} id options]
|
|
|
|
(assert (db/connection? conn) "expected cfg with valid connection")
|
|
|
|
(let [{:keys [migrate?
|
|
realize?
|
|
decode?
|
|
load-data?
|
|
lock-for-share?
|
|
read-only?]
|
|
:as options}
|
|
(merge default-get-file-options options)
|
|
|
|
read-only?
|
|
(or read-only? (db/read-only? conn))]
|
|
|
|
(when (and realize? (or (not load-data?) (not decode?)))
|
|
(throw (IllegalArgumentException. "invalid options, you only can realize when loading data and decoding")))
|
|
|
|
(when (and lock-for-share? load-data?)
|
|
(throw (IllegalArgumentException. "invalid options, `:lock-for-share?` is only allowed on row-only reads")))
|
|
|
|
(cond
|
|
;; The caller only wants the row: that read is the answer.
|
|
(not load-data?)
|
|
(fetch-row cfg id options)
|
|
|
|
(and migrate? decode? (not read-only?))
|
|
(when-let [file (fetch-row cfg id options)]
|
|
(if (fmg/need-migration? file)
|
|
(when-let [file (fetch-file cfg id (assoc options :lock-for-update? true))]
|
|
(cond-> file
|
|
(fmg/need-migration? file)
|
|
(as-> $ (migrate-file cfg $ options))
|
|
|
|
realize?
|
|
(->> (fdata/realize cfg))))
|
|
|
|
(let [file (attach-file-data cfg file options)]
|
|
(cond-> file
|
|
realize?
|
|
(->> (fdata/realize cfg))))))
|
|
|
|
:else
|
|
(when-let [file (fetch-file cfg id options)]
|
|
(cond-> file
|
|
(and migrate? decode? (fmg/need-migration? file))
|
|
(as-> $ (migrate-file cfg $ (assoc options :persist-after-migration? false)))
|
|
|
|
realize?
|
|
(->> (fdata/realize cfg)))))))
|
|
|
|
(defn get-file
|
|
"Get file, resolve all features and apply migrations.
|
|
|
|
Runs inside a transaction: it joins the caller's when the cfg already
|
|
carries a connection and opens one otherwise, so the migration write
|
|
commits with the read. A read-only read (`:read-only?`) takes no lock and
|
|
writes nothing, so it runs on a plain connection instead of opening a
|
|
transaction of its own; with a cfg that already carries a connection it
|
|
still joins the caller's. Accepts a cfg map only: a bare pool or a
|
|
connection raises `IllegalArgumentException`.
|
|
|
|
Useful when you have plan to apply massive or not surgical
|
|
operations on file, because it removes the overhead of lazy fetching
|
|
and decoding.
|
|
|
|
Options:
|
|
- `:realize?` returns plain data instead of fragment pointers; it
|
|
requires `:load-data?` and `:decode?`.
|
|
- `:decode?` decodes the stored blob (default true).
|
|
- `:load-data?` reads the data columns (default true). A row-only read
|
|
(`false`) is the only mode that accepts `:lock-for-share?`.
|
|
- `:migrate?` applies the pending migrations on read (default true).
|
|
- `:load-migrations?` resolves the applied migrations on a row read
|
|
(default true).
|
|
- `:read-only?` applies the pending migrations in memory only and skips
|
|
the transaction. It does not take a lock.
|
|
- `:lock-for-update?`, `:skip-locked?`, `:include-deleted?` and
|
|
`:throw-if-not-exists?` are the usual read knobs. A lock needs a
|
|
read-write transaction from the caller: PostgreSQL rejects row locks in
|
|
a read-only transaction.
|
|
- `:persist-after-migration?` is internal: `get-file*` turns it off
|
|
for the in-memory migration of a read-only context. Do not pass it."
|
|
[cfg file-id & {:as opts}]
|
|
(when-not (map? cfg)
|
|
(throw (IllegalArgumentException.
|
|
"expected cfg map, not a pool or a connection")))
|
|
(if (:read-only? opts)
|
|
(db/run! cfg get-file* file-id opts)
|
|
(db/tx-run! cfg get-file* file-id opts)))
|
|
|
|
(defn clean-file-features
|
|
[file]
|
|
(update file :features (fn [features]
|
|
(if (set? features)
|
|
(-> features
|
|
(cfeat/migrate-legacy-features)
|
|
(set/difference cfeat/frontend-only-features)
|
|
(set/difference cfeat/backend-only-features))
|
|
#{}))))
|
|
|
|
(defn check-file-exists
|
|
[cfg id & {:keys [include-deleted?]
|
|
:or {include-deleted? false}
|
|
:as options}]
|
|
(db/get-with-sql cfg [sql:get-minimal-file id]
|
|
{:db/remove-deleted (not include-deleted?)}))
|
|
|
|
(def ^:private sql:file-permissions
|
|
"select fpr.is_owner,
|
|
fpr.is_admin,
|
|
fpr.can_edit
|
|
from file_profile_rel as fpr
|
|
inner join file as f on (f.id = fpr.file_id)
|
|
inner join project as p on (p.id = f.project_id)
|
|
where fpr.file_id = ?
|
|
and fpr.profile_id = ?
|
|
and f.deleted_at is null
|
|
and p.deleted_at is null
|
|
and exists (select 1
|
|
from team_profile_rel as tpr
|
|
inner join team as t on (t.id = tpr.team_id)
|
|
where tpr.team_id = p.team_id
|
|
and tpr.profile_id = fpr.profile_id
|
|
and t.deleted_at is null)
|
|
union all
|
|
select tpr.is_owner,
|
|
tpr.is_admin,
|
|
tpr.can_edit
|
|
from team_profile_rel as tpr
|
|
inner join team as t on (t.id = tpr.team_id)
|
|
inner join project as p on (p.team_id = tpr.team_id)
|
|
inner join file as f on (p.id = f.project_id)
|
|
where f.id = ?
|
|
and tpr.profile_id = ?
|
|
and t.deleted_at is null
|
|
and p.deleted_at is null
|
|
and f.deleted_at is null
|
|
union all
|
|
select ppr.is_owner,
|
|
ppr.is_admin,
|
|
ppr.can_edit
|
|
from project_profile_rel as ppr
|
|
inner join file as f on (f.project_id = ppr.project_id)
|
|
inner join project as p on (p.id = ppr.project_id)
|
|
where f.id = ?
|
|
and ppr.profile_id = ?
|
|
and f.deleted_at is null
|
|
and p.deleted_at is null
|
|
and exists (select 1
|
|
from team_profile_rel as tpr
|
|
inner join team as t on (t.id = tpr.team_id)
|
|
where tpr.team_id = p.team_id
|
|
and tpr.profile_id = ppr.profile_id
|
|
and t.deleted_at is null)")
|
|
|
|
(defn- get-file-permissions*
|
|
[conn profile-id file-id]
|
|
(when (and profile-id file-id)
|
|
(db/exec! conn [sql:file-permissions
|
|
file-id profile-id
|
|
file-id profile-id
|
|
file-id profile-id])))
|
|
|
|
(defn get-file-permissions
|
|
([conn profile-id file-id]
|
|
(let [rows (get-file-permissions* conn profile-id file-id)
|
|
is-owner (boolean (some :is-owner rows))
|
|
is-admin (boolean (some :is-admin rows))
|
|
can-edit (boolean (some :can-edit rows))]
|
|
(when (seq rows)
|
|
{:type :membership
|
|
:is-owner is-owner
|
|
:is-admin (or is-owner is-admin)
|
|
:can-edit (or is-owner is-admin can-edit)
|
|
:can-read true
|
|
:is-logged (some? profile-id)})))
|
|
|
|
([conn profile-id file-id share-id]
|
|
(let [perms (get-file-permissions conn profile-id file-id)
|
|
ldata (some-> (db/get* conn :share-link {:id share-id :file-id file-id})
|
|
(dissoc :flags)
|
|
(update :pages db/decode-pgarray #{}))]
|
|
|
|
;; NOTE: in a future when share-link becomes more powerful and
|
|
;; will allow us specify which parts of the app is available, we
|
|
;; will probably need to tweak this function in order to expose
|
|
;; this flags to the frontend.
|
|
(cond
|
|
(some? perms) perms
|
|
(some? ldata) {:type :share-link
|
|
:can-read true
|
|
:pages (:pages ldata)
|
|
:is-logged (some? profile-id)
|
|
:who-comment (:who-comment ldata)
|
|
:who-inspect (:who-inspect ldata)}))))
|
|
|
|
|
|
(defn get-project
|
|
[cfg project-id]
|
|
(db/get cfg :project {:id project-id}))
|
|
|
|
(def ^:private sql:get-files-rels
|
|
"SELECT flr.*,
|
|
fls.synced_at
|
|
FROM file_library_rel AS flr
|
|
JOIN file AS l
|
|
ON flr.library_file_id = l.id
|
|
LEFT JOIN file_library_sync AS fls
|
|
ON fls.file_id = flr.file_id
|
|
AND fls.library_file_id = flr.library_file_id
|
|
WHERE flr.file_id = ANY(?)
|
|
AND l.deleted_at IS NULL
|
|
ORDER BY flr.file_id, flr.library_file_id;")
|
|
|
|
(defn get-files-rels
|
|
"Given a set of file-id's, return all matching relations with the libraries.
|
|
Rows come back in stable `(file-id, library-file-id)` order so export
|
|
and import resolve cycles deterministically."
|
|
[cfg ids]
|
|
|
|
(assert (set? ids) "expected a set of uuids")
|
|
(assert (every? uuid? ids) "expected a set of uuids")
|
|
|
|
(db/run! cfg (fn [{:keys [::db/conn]}]
|
|
(let [ids (db/create-array conn "uuid" ids)]
|
|
(db/exec! conn [sql:get-files-rels ids])))))
|
|
|
|
(def ^:private sql:upsert-file-library-sync
|
|
"INSERT INTO file_library_sync (file_id, library_file_id, synced_at)
|
|
VALUES (?::uuid, ?::uuid, ?::timestamptz)
|
|
ON CONFLICT (file_id, library_file_id)
|
|
DO UPDATE SET synced_at = EXCLUDED.synced_at;")
|
|
|
|
(defn upsert-file-library-sync!
|
|
[conn {:keys [file-id library-file-id synced-at]}]
|
|
(db/exec-one! conn [sql:upsert-file-library-sync file-id library-file-id synced-at]))
|
|
|
|
(def ^:private sql:get-libraries
|
|
"WITH RECURSIVE libs AS (
|
|
SELECT fl.id
|
|
FROM file AS fl
|
|
JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
|
|
WHERE flr.file_id = ANY(?)
|
|
UNION
|
|
SELECT fl.id
|
|
FROM file AS fl
|
|
JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id)
|
|
JOIN libs AS l ON (flr.file_id = l.id)
|
|
)
|
|
SELECT DISTINCT l.id
|
|
FROM libs AS l")
|
|
|
|
(defn get-libraries
|
|
"Get all libraries ids related to provided file ids"
|
|
[cfg ids]
|
|
(db/run! cfg (fn [{:keys [::db/conn]}]
|
|
(let [ids' (db/create-array conn "uuid" ids)]
|
|
(->> (db/exec! conn [sql:get-libraries ids'])
|
|
(into #{} xf-map-id))))))
|
|
|
|
(defn library-link-status
|
|
"Check whether linking `file-id` to `library-id` is safe. Returns
|
|
`:ok` when the link can be created, `:self-link` when both ids are
|
|
the same, and `:circular` when the link would close a dependency
|
|
cycle (i.e. `file-id` is already among the transitive libraries of
|
|
`library-id`). Shared by the UI command (`link-file-to-library`,
|
|
which raises) and the v3 importer (`import-file-relations`, which
|
|
skips and logs), so both stay consistent."
|
|
[cfg file-id library-id]
|
|
(cond
|
|
(= file-id library-id) :self-link
|
|
(contains? (get-libraries cfg [library-id]) file-id) :circular
|
|
:else :ok))
|
|
|
|
(defn get-file-object-thumbnails
|
|
"Return all file object thumbnails for a given file."
|
|
[cfg file-id]
|
|
(->> (db/query cfg :file-tagged-object-thumbnail
|
|
{:file-id file-id
|
|
:deleted-at nil})
|
|
(not-empty)))
|
|
|
|
(def ^:private sql:get-missing-media-references
|
|
"SELECT fmo.*
|
|
FROM file_media_object AS fmo
|
|
WHERE fmo.id = ANY(?::uuid[])
|
|
AND file_id != ?")
|
|
|
|
(defn update-media-references!
|
|
"Given a file and a coll of media-refs, check if all provided
|
|
references are correct or fix them in-place"
|
|
[{:keys [::db/conn] :as cfg} {file-id :id :as file} media-refs]
|
|
(let [missing-index
|
|
(reduce (fn [result {:keys [id] :as fmo}]
|
|
(assoc result id
|
|
(-> fmo
|
|
(assoc :id (uuid/next))
|
|
(assoc :file-id file-id)
|
|
(dissoc :created-at)
|
|
(dissoc :deleted-at))))
|
|
{}
|
|
(db/exec! conn [sql:get-missing-media-references
|
|
(->> (into #{} xf-map-id media-refs)
|
|
(db/create-array conn "uuid"))
|
|
file-id]))
|
|
|
|
lookup-index
|
|
(fn [id]
|
|
(if-let [mobj (get missing-index id)]
|
|
(do
|
|
(l/trc :hint "lookup index"
|
|
:file-id (str file-id)
|
|
:id (str id)
|
|
:result (str (get mobj :id)))
|
|
(get mobj :id))
|
|
|
|
id))
|
|
|
|
update-shapes
|
|
(fn [data {:keys [page-id shape-id]}]
|
|
(d/update-in-when data [:pages-index page-id :objects shape-id] cfh/relink-refs lookup-index))
|
|
|
|
file
|
|
(update file :data #(reduce update-shapes % media-refs))]
|
|
|
|
(doseq [[old-id item] missing-index]
|
|
(l/dbg :hint "create missing references"
|
|
:file-id (str file-id)
|
|
:old-id (str old-id)
|
|
:id (str (:id item)))
|
|
(db/insert! conn :file-media-object item
|
|
{::db/return-keys false}))
|
|
|
|
file))
|
|
|
|
(def sql:get-file-media
|
|
"SELECT * FROM file_media_object WHERE id = ANY(?)")
|
|
|
|
(defn get-file-media*
|
|
[{:keys [::db/conn] :as cfg} {:keys [data id] :as file}]
|
|
(let [used (cfh/collect-used-media data)
|
|
used (db/create-array conn "uuid" used)]
|
|
(->> (db/exec! conn [sql:get-file-media used])
|
|
(mapv (fn [row] (assoc row :file-id id))))))
|
|
|
|
(defn get-file-media
|
|
[cfg file]
|
|
(db/run! cfg get-file-media* file))
|
|
|
|
(def ^:private sql:get-team-files-ids
|
|
"SELECT f.id FROM file AS f
|
|
JOIN project AS p ON (p.id = f.project_id)
|
|
WHERE p.team_id = ?")
|
|
|
|
(defn get-team-files-ids
|
|
"Get a set of file ids for the specified team-id"
|
|
[{:keys [::db/conn]} team-id]
|
|
(->> (db/exec! conn [sql:get-team-files-ids team-id])
|
|
(into #{} xf-map-id)))
|
|
|
|
(def ^:private sql:get-team-projects
|
|
"SELECT p.* FROM project AS p
|
|
WHERE p.team_id = ?
|
|
AND p.deleted_at IS NULL")
|
|
|
|
(defn get-team-projects
|
|
"Get a set of project ids for the team"
|
|
[cfg team-id]
|
|
(->> (db/exec! cfg [sql:get-team-projects team-id])
|
|
(into #{} xf-map-id)))
|
|
|
|
(def ^:private sql:get-project-files
|
|
"SELECT f.id FROM file AS f
|
|
WHERE f.project_id = ?
|
|
AND f.deleted_at IS NULL")
|
|
|
|
(defn get-project-files
|
|
"Get a set of file ids for the project"
|
|
[{:keys [::db/conn]} project-id]
|
|
(->> (db/exec! conn [sql:get-project-files project-id])
|
|
(into #{} xf-map-id)))
|
|
|
|
(defn remap-thumbnail-object-id
|
|
[object-id file-id]
|
|
(str/replace-first object-id #"^(.*?)/" (str file-id "/")))
|
|
|
|
(defn- relink-shapes
|
|
"A function responsible to analyze all file data and
|
|
replace the old :component-file reference with the new
|
|
ones, using the provided file-index."
|
|
[data]
|
|
(cfh/relink-refs data lookup-index))
|
|
|
|
(defn- relink-media
|
|
"A function responsible of process the :media attr of file data and
|
|
remap the old ids with the new ones."
|
|
[media]
|
|
(reduce-kv (fn [res k v]
|
|
(let [id (lookup-index k)]
|
|
(if (uuid? id)
|
|
(-> res
|
|
(assoc id (assoc v :id id))
|
|
(dissoc k))
|
|
res)))
|
|
media
|
|
media))
|
|
|
|
(defn- relink-colors
|
|
"A function responsible of process the :colors attr of file data and
|
|
remap the old ids with the new ones."
|
|
[colors]
|
|
(reduce-kv (fn [res k v]
|
|
(if (:image v)
|
|
(update-in res [k :image :id] lookup-index)
|
|
res))
|
|
colors
|
|
colors))
|
|
|
|
(defn absorb-unlinked-tokens-source
|
|
"Make the file its own tokens source when its tokens source is not one
|
|
of the given library-ids, as happens when the library was deleted. The
|
|
tokens lib of the source is copied while the library still exists, even
|
|
if deleted; otherwise the tokens source is removed and the file gets back
|
|
the tokens status kept aside for its own tokens lib, if any."
|
|
[cfg data library-ids]
|
|
(let [source-id (cfo/get-tokens-source data)]
|
|
(if (and (some? source-id)
|
|
(not (contains? library-ids source-id)))
|
|
(if-let [library (get-file cfg source-id
|
|
:include-deleted? true
|
|
:throw-if-not-exists? false)]
|
|
(cfo/absorb-tokens-lib data (:data library))
|
|
(cfo/detach-tokens-source data))
|
|
data)))
|
|
|
|
(defn embed-assets
|
|
[cfg data file-id]
|
|
(let [library-ids (get-libraries cfg [file-id])
|
|
data (reduce (fn [data library-id]
|
|
(if-let [library (get-file cfg library-id :include-deleted? true)]
|
|
;; The library is read without `:realize?`, so its
|
|
;; `:components` may be a pointer map. Bind the
|
|
;; library's own loader: the absorb walks the
|
|
;; components and would raise without one.
|
|
(cond-> (fdata/with-pointers cfg library-id
|
|
(ctf/absorb-assets data (:data library)))
|
|
(= library-id (cfo/get-tokens-source data))
|
|
(cfo/absorb-tokens-lib (:data library)))
|
|
data))
|
|
data
|
|
library-ids)]
|
|
(absorb-unlinked-tokens-source cfg data library-ids)))
|
|
|
|
(def transaction-idle-timeout-ms
|
|
"Idle-in-transaction guard for the long binfile transactions: 30
|
|
minutes. The import and the export of a file run inside one
|
|
transaction that can go quiet while the ZIP is decoded and the file
|
|
data is realized. The longest operation observed in production is
|
|
under 8 minutes, so this leaves room for bigger files and still
|
|
aborts a connection that is left idle for half an hour."
|
|
1800000)
|
|
|
|
(defn set-transaction-idle-timeout
|
|
"Set `idle_in_transaction_session_timeout` for the current
|
|
transaction. Must be called inside a transaction: `SET LOCAL` is
|
|
scoped to it."
|
|
[cfg]
|
|
(let [conn (db/get-connection cfg)]
|
|
(db/exec-one! conn [(str "SET LOCAL idle_in_transaction_session_timeout = "
|
|
transaction-idle-timeout-ms)])))
|
|
|
|
(defn configure-import-transaction
|
|
[cfg]
|
|
(set-transaction-idle-timeout cfg)
|
|
(let [conn (db/get-connection cfg)]
|
|
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])))
|
|
|
|
(defn process-file
|
|
[cfg {:keys [id] :as file}]
|
|
(let [libs (delay (get-resolved-file-libraries cfg file))]
|
|
(-> file
|
|
(update :data (fn [fdata]
|
|
(-> fdata
|
|
(assoc :id id)
|
|
(dissoc :recent-colors))))
|
|
(update :data (fn [fdata]
|
|
(-> fdata
|
|
(update :pages-index relink-shapes)
|
|
(update :components relink-shapes)
|
|
(update :media relink-media)
|
|
(update :colors relink-colors)
|
|
(d/without-nils))))
|
|
(fmg/migrate-file libs)
|
|
|
|
;; NOTE: this is necessary because when we just creating a new
|
|
;; file from imported artifact or cloned file there are no
|
|
;; migrations registered on the database, so we need to persist
|
|
;; all of them, not only the applied
|
|
(vary-meta dissoc ::fmg/migrated))))
|
|
|
|
(defn- encode-file
|
|
[cfg {:keys [id features] :as file}]
|
|
(let [file (if (and (contains? features "fdata/objects-map")
|
|
(:data file))
|
|
(fdata/enable-objects-map file)
|
|
file)
|
|
|
|
file (if (and (contains? features "fdata/pointer-map")
|
|
(:data file))
|
|
|
|
(binding [pmap/*tracked* (pmap/create-tracked :inherit true)]
|
|
(let [file (fdata/enable-pointer-map file)]
|
|
(fdata/persist-pointers! cfg id)
|
|
file))
|
|
file)]
|
|
|
|
(-> file
|
|
(d/update-when :features into-array)
|
|
(d/update-when :data blob/encode))))
|
|
|
|
(defn- file->params
|
|
[file]
|
|
(-> (select-keys file file-attrs)
|
|
(assoc :data nil)
|
|
(dissoc :team-id)
|
|
(dissoc :metadata)
|
|
(dissoc :migrations)))
|
|
|
|
(defn file->response
|
|
"Strip the internal keys of a file map before it leaves the backend in a
|
|
response. The file data storage pointer is bookkeeping for the write path:
|
|
a `file-data` object is authentication-only, so a leaked id would let any
|
|
logged-in profile fetch the blob from `/assets/by-id`. The metadata key of
|
|
the old pointer is stripped too: a row the migration does not backfill keeps
|
|
it, and the metadata schema is open, so it would travel in the response."
|
|
[file]
|
|
(-> file
|
|
(dissoc :storage-object-id)
|
|
(d/update-when :metadata #(dissoc % :storage-ref-id))))
|
|
|
|
(defn- file->file-data-params
|
|
[{:keys [id] :as file}]
|
|
(let [created-at (or (:created-at file) (ct/now))
|
|
modified-at (or (:modified-at file) created-at)]
|
|
(-> (d/without-nils
|
|
{:id id
|
|
:type "main"
|
|
:file-id id
|
|
:data (:data file)
|
|
:metadata (:metadata file)
|
|
:created-at created-at
|
|
:modified-at modified-at})
|
|
;; The storage object the persisted row pointed at (nil for a new or
|
|
;; imported file). `upsert!` touches it before replacing the row.
|
|
(assoc :storage-object-id (:storage-object-id file)))))
|
|
|
|
(defn insert-file!
|
|
"Insert a new file into the database table. Expectes a not-encoded file.
|
|
Returns nil."
|
|
[{:keys [::db/conn] :as cfg} file & {:as opts}]
|
|
(db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"])
|
|
|
|
(when (:migrations file)
|
|
(fmigr/upsert-migrations! conn file))
|
|
|
|
(let [file (encode-file cfg file)]
|
|
(try
|
|
(db/insert! conn :file
|
|
(file->params file)
|
|
(assoc opts ::db/return-keys false))
|
|
(catch org.postgresql.util.PSQLException cause
|
|
(if (db/duplicate-key-error? cause)
|
|
(ex/raise :type :not-found
|
|
:code :object-not-found
|
|
:hint "file already exists"
|
|
:cause cause)
|
|
(throw cause))))
|
|
|
|
;; An INSERT never replaces a persisted pointer: `duplicate-file` hands
|
|
;; the source file map here, and touching the source object would be a
|
|
;; write on a live object.
|
|
(->> (file->file-data-params (dissoc file :storage-object-id))
|
|
(fdata/upsert! cfg))
|
|
|
|
nil))
|
|
|
|
(defn update-file!
|
|
"Update an existing file on the database. Expects a not-encoded file map,
|
|
as returned by `get-file`, and optionally the fields to change as keyword
|
|
args; they are merged into the file map before persisting.
|
|
|
|
The storage object the persisted row pointed at travels in the map
|
|
(`:storage-object-id`), and `upsert!` touches it before replacing the row.
|
|
That is why the caller must pass the map it read instead of building a
|
|
partial one: nothing is queried here. The params only change mutable fields
|
|
of the file: the pointer and the identity (`:id`, `:file-id`) always come
|
|
from the read map.
|
|
|
|
Returns the map `fdata/upsert!` answers with: `:written` and the
|
|
`:storage-object-id` of the object this write created, so a caller that
|
|
persists again touches the current object instead of the replaced one."
|
|
[{:keys [::db/conn] :as cfg} file & {:as params}]
|
|
|
|
;; A map read with `:load-data? false` has no `:data`; persisting it would
|
|
;; wipe the file content. `ex/raise` instead of `assert`: the guard must
|
|
;; exist in the deployed binary, where `clojure.core/assert` is compiled
|
|
;; out.
|
|
(when-not (contains? file :data)
|
|
(ex/raise :type :internal
|
|
:code :missing-file-data
|
|
:hint "expected a file map read with data"
|
|
:file-id (:id file)))
|
|
|
|
(let [reset? (::reset-migrations? params false)
|
|
;; The pointer and the identity are bookkeeping of the persisted row,
|
|
;; not editable fields: they can only come from the read map. A pointer
|
|
;; in the params would skip the touch and leak the replaced blob; an
|
|
;; `:id` would retarget the UPDATE.
|
|
file (merge file (dissoc params ::reset-migrations?
|
|
:storage-object-id
|
|
:id
|
|
:file-id))]
|
|
|
|
(if reset?
|
|
(fmigr/reset-migrations! conn file)
|
|
(fmigr/upsert-migrations! conn file))
|
|
|
|
(let [file (encode-file cfg file)
|
|
file-params (file->params (dissoc file :id))]
|
|
|
|
(db/update! conn :file file-params
|
|
{:id (:id file)}
|
|
{::db/return-keys false})
|
|
|
|
(fdata/upsert! cfg (file->file-data-params file)))))
|
|
|
|
(defn save-file!
|
|
"Applies all the final validations and perist the file, binfile
|
|
specific, should not be used outside of binfile domain.
|
|
Returns nil"
|
|
[{:keys [::timestamp] :as cfg} file & {:as opts}]
|
|
|
|
(assert (ct/inst? timestamp) "expected valid timestamp")
|
|
|
|
(let [file (-> file
|
|
(assoc :created-at timestamp)
|
|
(assoc :modified-at timestamp)
|
|
(assoc :ignore-sync-until (ct/plus timestamp (ct/duration {:seconds 5})))
|
|
(update :features
|
|
(fn [features]
|
|
(-> (::features cfg #{})
|
|
(set/union features)
|
|
;; We never want to store
|
|
;; frontend-only features on file
|
|
(set/difference cfeat/frontend-only-features)))))]
|
|
|
|
(when (contains? cf/flags :file-schema-validation)
|
|
(fval/validate-file-schema! file))
|
|
|
|
(when (contains? cf/flags :soft-file-schema-validation)
|
|
(let [result (ex/try! (fval/validate-file-schema! file))]
|
|
(when (ex/exception? result)
|
|
(l/error :hint "file schema validation error" :cause result))))
|
|
|
|
(insert-file! cfg file opts)))
|
|
|
|
(def ^:private sql:get-file-libraries
|
|
"WITH RECURSIVE libs AS (
|
|
SELECT fl.*
|
|
FROM file AS fl
|
|
JOIN file_library_rel AS flr
|
|
ON flr.library_file_id = fl.id
|
|
WHERE flr.file_id = ?::uuid
|
|
|
|
UNION
|
|
|
|
SELECT fl.*
|
|
FROM file AS fl
|
|
JOIN file_library_rel AS flr
|
|
ON flr.library_file_id = fl.id
|
|
JOIN libs AS l
|
|
ON flr.file_id = l.id
|
|
)
|
|
SELECT l.id,
|
|
l.features,
|
|
l.project_id,
|
|
p.team_id,
|
|
l.created_at,
|
|
l.modified_at,
|
|
l.deleted_at,
|
|
l.name,
|
|
l.revn,
|
|
l.vern,
|
|
l.is_shared,
|
|
l.version,
|
|
fls.synced_at,
|
|
NOT EXISTS (
|
|
SELECT 1 FROM file_library_rel AS direct
|
|
WHERE direct.file_id = ?::uuid
|
|
AND direct.library_file_id = l.id
|
|
) AS is_indirect
|
|
FROM libs AS l
|
|
JOIN project AS p
|
|
ON p.id = l.project_id
|
|
LEFT JOIN file_library_sync AS fls
|
|
ON fls.file_id = ?::uuid
|
|
AND fls.library_file_id = l.id
|
|
WHERE l.deleted_at IS NULL;")
|
|
|
|
(defn get-file-libraries
|
|
[conn file-id]
|
|
(into []
|
|
(map decode-row-features)
|
|
(db/exec! conn [sql:get-file-libraries file-id file-id file-id])))
|
|
|
|
(defn get-resolved-file-libraries
|
|
"Get all file libraries including itself. Returns an instance of
|
|
LoadableWeakValueMap that allows do not have strong references to
|
|
the loaded libraries and reduce memory pressure on having
|
|
all this libraries at the same time on processing file validation
|
|
or file migration.
|
|
|
|
This still requires at least one library at time to be loaded while
|
|
access to it is performed, but it improves considerable not having
|
|
the need of loading all the libraries at the same time."
|
|
[{:keys [::db/conn] :as cfg} {:keys [id] :as file}]
|
|
(let [library-ids (->> (get-file-libraries conn (:id file))
|
|
(map :id)
|
|
(cons (:id file)))
|
|
;; Fragment rows are file-local and callers bind the loader of
|
|
;; the file they work on, so each library realizes its pointers
|
|
;; here, with its own id, before anything walks its data.
|
|
load-fn (fn [library-id]
|
|
(->> (get-file cfg library-id :migrate? false)
|
|
(fdata/realize-pointers cfg)))]
|
|
(weak/loadable-weak-value-map library-ids load-fn {id file})))
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;; EXTERNAL LIBRARY RESOLUTION HELPERS
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(defn slugify-name
|
|
"Slugify a library name for cross-environment matching.
|
|
Lowercases, replaces non-alphanumeric runs with '-', strips
|
|
leading/trailing '-'."
|
|
[name]
|
|
(str/slug name))
|
|
|
|
(def ^:private sql:get-files-names
|
|
"SELECT id, name FROM file WHERE id = ANY(?)")
|
|
|
|
(defn get-files-names
|
|
"Return [{:id uuid :name string}] for the given file ids."
|
|
[cfg ids]
|
|
(db/run! cfg
|
|
(fn [{:keys [::db/conn]}]
|
|
(let [ids-arr (db/create-array conn "uuid" ids)]
|
|
(db/exec! conn [sql:get-files-names ids-arr])))))
|
|
|
|
(def ^:private sql:get-shared-files-for-team
|
|
"SELECT f.id, f.name, f.project_id
|
|
FROM file AS f
|
|
JOIN project AS p ON (p.id = f.project_id)
|
|
WHERE p.team_id = ?
|
|
AND f.is_shared = true
|
|
AND f.deleted_at IS NULL
|
|
AND p.deleted_at IS NULL")
|
|
|
|
(defn get-shared-files-for-team
|
|
"Return [{:id uuid :name string}] for all shared files in a team."
|
|
[cfg team-id]
|
|
(db/run! cfg
|
|
(fn [{:keys [::db/conn]}]
|
|
(db/exec! conn [sql:get-shared-files-for-team team-id]))))
|
|
|
|
(defn find-shared-files-by-slug
|
|
"Return all shared files in `team-id` whose slugified name equals `slug`."
|
|
[cfg team-id slug]
|
|
(->> (get-shared-files-for-team cfg team-id)
|
|
(filter #(= slug (slugify-name (:name %))))))
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;; SVG IMPORT SANITIZATION
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(def svg-content-type
|
|
"Canonical content type of SVG media objects."
|
|
"image/svg+xml")
|
|
|
|
(defn normalize-content-type
|
|
"Canonicalize a stored content-type: lowercase, trimmed,
|
|
parameters after `;` dropped. Returns nil for missing or unusable
|
|
values so ancient bundle entries without content-type keep passing
|
|
through untouched."
|
|
[ctype]
|
|
(when (string? ctype)
|
|
(let [clean (-> ctype
|
|
(str/split #";" 2)
|
|
(first)
|
|
(str/trim)
|
|
(str/lower))]
|
|
(when-not (str/empty? clean)
|
|
clean))))
|
|
|
|
(defn svg-object?
|
|
"True when the storage `object` claims the canonical SVG content
|
|
type. Expects an already normalized object (see
|
|
`normalize-content-type`): every import path normalizes the
|
|
metadata right after reading it, so the comparison stays an exact
|
|
match in a single place."
|
|
[object]
|
|
(= svg-content-type (:content-type object)))
|
|
|
|
(def schema:content-type
|
|
"Typed content-type for binfile storage objects: a member of
|
|
`cm/storage-object-types`. Values are canonicalized with
|
|
`normalize-content-type` before validation, so legacy spellings
|
|
keep importing while unknown types are rejected."
|
|
[::sm/one-of {:format :string} cm/storage-object-types])
|
|
|
|
(defn check-storage-content-type
|
|
"Check an imported storage `object` (already normalized, see
|
|
`normalize-content-type`) against `cm/storage-object-types`.
|
|
Returns nil when the type is allowed; raises `:type :validation`
|
|
with `:code :media-type-not-allowed` (same as the upload path)
|
|
when it is missing or unknown, so crafted bundles fail closed."
|
|
[object]
|
|
(when-not (contains? cm/storage-object-types (:content-type object))
|
|
(ex/raise :type :validation
|
|
:code :media-type-not-allowed
|
|
:hint "storage object declares an unknown content-type"
|
|
:content-type (:content-type object))))
|
|
|
|
(defn sanitize-imported-svg
|
|
"Sanitize the raw `bytes` of an imported storage `object` when it
|
|
holds an SVG document. Expects an already normalized object (see
|
|
`normalize-content-type`); returns nil when the object is not an
|
|
SVG.
|
|
|
|
Otherwise returns a map with the sanitized `:bytes`, their `:size`
|
|
and their blake2b `:hash`, ready to persist with `sto/put-object!`.
|
|
|
|
Raises a `:validation` exception when the SVG cannot be parsed, the
|
|
same error the upload path reports."
|
|
[object ^bytes raw]
|
|
(when (svg-object? object)
|
|
(let [sanitized (svg/sanitize-svg (String. ^bytes raw "UTF-8"))
|
|
bytes (.getBytes ^String sanitized "UTF-8")]
|
|
{:bytes bytes
|
|
:size (alength ^bytes bytes)
|
|
:hash (sto/calculate-hash (java.io.ByteArrayInputStream. bytes))})))
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;; REQUEST CFG
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(defn resolve-export-type
|
|
"The canonical export type of a request on the legacy RPC: an explicit
|
|
`type` wins, and the deprecated boolean flags are translated. The
|
|
creation command of a job does not need it, because it asks the caller
|
|
for the type instead of guessing it."
|
|
[{:keys [type include-libraries embed-assets]}]
|
|
(cond
|
|
(some? type) type
|
|
(true? include-libraries) :include-libraries
|
|
(true? embed-assets) :merge-libraries
|
|
:else :detach-libraries))
|
|
|
|
(defn export-cfg
|
|
"The domain cfg of an export: the frozen set of files, how their
|
|
libraries are handled, and where the package is written."
|
|
[cfg {:keys [file-ids export-type output]}]
|
|
(-> cfg
|
|
(assoc ::file-ids file-ids)
|
|
(assoc ::export-type export-type)
|
|
(assoc ::output output)))
|
|
|
|
(defn import-cfg
|
|
"The domain cfg of an import: the destination team/profile/project, the
|
|
name, the input and the anti-zip-bomb limits. The caller resolves the
|
|
team, because what allows it to import there differs between the RPC and
|
|
a job; what both must share is fixed here."
|
|
[cfg {:keys [profile-id project-id team name input]}]
|
|
(-> cfg
|
|
(assoc ::features (cfeat/get-team-enabled-features cf/flags team))
|
|
(assoc ::project-id project-id)
|
|
(assoc ::profile-id profile-id)
|
|
(assoc ::team-id (:id team))
|
|
(assoc ::name name)
|
|
(assoc ::input input)
|
|
(assoc ::import-max-binary-entry-size (cf/get :binfile-import-max-binary-entry-size))
|
|
(assoc ::import-max-text-entry-size (cf/get :binfile-import-max-text-entry-size))
|
|
(assoc ::import-max-text-total-size (cf/get :binfile-import-max-text-total-size))
|
|
(assoc ::import-max-zip-entries (cf/get :binfile-import-max-zip-entries))))
|