From a77edc5aa242852d262ad8e54948f18677a9130f Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 22 Jul 2025 13:31:23 +0200 Subject: [PATCH 1/5] :sparkles: Add better uri constructor function --- common/src/app/common/uri.cljc | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/common/src/app/common/uri.cljc b/common/src/app/common/uri.cljc index 4fe6e967a4..e7c258adfc 100644 --- a/common/src/app/common/uri.cljc +++ b/common/src/app/common/uri.cljc @@ -13,12 +13,27 @@ #?(:clj (:import lambdaisland.uri.URI))) -(dm/export u/uri) (dm/export u/join) +(dm/export u/parse) (dm/export u/query-encode) (dm/export un/percent-encode) (dm/export u/uri?) +(defn uri + [o] + (cond + (u/uri? o) + o + + (map? o) + (u/map->URI o) + + (nil? o) + o + + :else + (u/parse o))) + (defn query-string->map [s] (u/query-string->map s)) From 6c7fef29a817fa052d86149925b861e89191c6c8 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 22 Jul 2025 13:33:33 +0200 Subject: [PATCH 2/5] :sparkles: Improve file data type constructor --- common/src/app/common/types/file.cljc | 71 +++++++++++++++++---------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/common/src/app/common/types/file.cljc b/common/src/app/common/types/file.cljc index a814d031be..df75669056 100644 --- a/common/src/app/common/types/file.cljc +++ b/common/src/app/common/types/file.cljc @@ -16,6 +16,7 @@ [app.common.geom.shapes.tree-seq :as gsts] [app.common.logging :as l] [app.common.schema :as sm] + [app.common.time :as dt] [app.common.types.color :as ctc] [app.common.types.component :as ctk] [app.common.types.components-list :as ctkl] @@ -93,12 +94,13 @@ [:map {:title "file"} [:id ::sm/uuid] [:name :string] - [:revn {:optional true} :int] + [:revn :int] [:vern {:optional true} :int] - [:created-at {:optional true} ::sm/inst] - [:modified-at {:optional true} ::sm/inst] + [:created-at ::sm/inst] + [:modified-at ::sm/inst] [:deleted-at {:optional true} ::sm/inst] [:project-id {:optional true} ::sm/uuid] + [:team-id {:optional true} ::sm/uuid] [:is-shared {:optional true} ::sm/boolean] [:data {:optional true} schema:data] [:version :int] @@ -145,35 +147,50 @@ (update :options merge {:components-v2 true :base-font-size BASE-FONT-SIZE}))))) +;; FIXME: we can't handle the "default" migrations for avoid providing +;; them all the time the file is created because we can't import file +;; migrations because of circular import issue; We need to split the +;; list of migrations and impl of migrations in separate namespaces + +;; FIXME: refactor + (defn make-file [{:keys [id project-id name revn is-shared features migrations - ignore-sync-until modified-at deleted-at] - :or {is-shared false revn 0}} + ignore-sync-until created-at modified-at deleted-at] + :as params} - & {:keys [create-page page-id] - :or {create-page true}}] + & {:keys [create-page with-data page-id] + :or {create-page true with-data true}}] - (let [id (or id (uuid/next)) - data (if create-page - (if page-id - (make-file-data id page-id) - (make-file-data id)) - (make-file-data id nil)) + (let [id (or id (uuid/next)) + created-at (or created-at (dt/now)) + modified-at (or modified-at created-at) + features (d/nilv features #{}) - file (d/without-nils - {:id id - :project-id project-id - :name name - :revn revn - :vern 0 - :is-shared is-shared - :version version - :data data - :features features - :migrations migrations - :ignore-sync-until ignore-sync-until - :modified-at modified-at - :deleted-at deleted-at})] + data + (when with-data + (if create-page + (if page-id + (make-file-data id page-id) + (make-file-data id)) + (make-file-data id nil))) + + file + (d/without-nils + {:id id + :project-id project-id + :name name + :revn (d/nilv revn 0) + :vern 0 + :is-shared (d/nilv is-shared false) + :version (:version params version) + :data data + :features features + :migrations migrations + :ignore-sync-until ignore-sync-until + :created-at created-at + :modified-at modified-at + :deleted-at deleted-at})] (check-file file))) From 4bdba6894d85e9e36715251bb3a35ad23cd93ebc Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 22 Jul 2025 13:36:11 +0200 Subject: [PATCH 3/5] :sparkles: Add get-with-sql helper to db module --- backend/src/app/db.clj | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index 4a583256ef..e17bc21b25 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -10,19 +10,20 @@ [app.common.data :as d] [app.common.exceptions :as ex] [app.common.geom.point :as gpt] + [app.common.json :as json] [app.common.logging :as l] [app.common.schema :as sm] [app.common.transit :as t] [app.common.uuid :as uuid] [app.db.sql :as sql] [app.metrics :as mtx] - [app.util.json :as json] [app.util.time :as dt] [clojure.java.io :as io] [clojure.set :as set] [integrant.core :as ig] [next.jdbc :as jdbc] [next.jdbc.date-time :as jdbc-dt] + [next.jdbc.prepare :as jdbc.prepare] [next.jdbc.transaction]) (:import com.zaxxer.hikari.HikariConfig @@ -33,6 +34,7 @@ java.io.InputStream java.io.OutputStream java.sql.Connection + java.sql.PreparedStatement java.sql.Savepoint org.postgresql.PGConnection org.postgresql.geometric.PGpoint @@ -404,6 +406,24 @@ :hint "database object not found")) row)) + +(defn get-with-sql + [ds sql & {:as opts}] + (let [rows (cond->> (exec! ds sql opts) + (::remove-deleted opts true) + (remove is-row-deleted?) + + :always + (not-empty))] + + (when (and (not rows) (::throw-if-not-exists opts true)) + (ex/raise :type :not-found + :code :object-not-found + :hint "database object not found")) + + (first rows))) + + (def ^:private default-plan-opts (-> default-opts (assoc :fetch-size 1000) @@ -599,7 +619,7 @@ val (.getValue o)] (if (or (= typ "json") (= typ "jsonb")) - (json/decode val) + (json/decode val :key-fn keyword) val)))) (defn decode-transit-pgobject @@ -640,7 +660,7 @@ (when data (doto (org.postgresql.util.PGobject.) (.setType "jsonb") - (.setValue (json/encode-str data))))) + (.setValue (json/encode data))))) ;; --- Locks @@ -686,3 +706,8 @@ [cause] (and (sql-exception? cause) (= "40001" (.getSQLState ^java.sql.SQLException cause)))) + +(extend-protocol jdbc.prepare/SettableParameter + clojure.lang.Keyword + (set-parameter [^clojure.lang.Keyword v ^PreparedStatement s ^long i] + (.setObject s i ^String (d/name v)))) From fd62141c04817c2ae86d10ad41b130fe616cf789 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 22 Jul 2025 13:51:21 +0200 Subject: [PATCH 4/5] :sparkles: Disable pointer-map feature (temporary) Because the upcoming refactor changes several aspects of that feature and it not make sense to continue have this active for now, until refactor is merged. --- backend/scripts/repl | 6 +++--- backend/scripts/start-dev | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/scripts/repl b/backend/scripts/repl index 60d6b95e4c..c8daf5f051 100755 --- a/backend/scripts/repl +++ b/backend/scripts/repl @@ -12,7 +12,7 @@ export PENPOT_FLAGS="\ enable-login-with-gitlab \ enable-backend-worker \ enable-backend-asserts \ - enable-feature-fdata-pointer-map \ + disable-feature-fdata-pointer-map \ enable-feature-fdata-objects-map \ enable-audit-log \ enable-transit-readable-response \ @@ -28,11 +28,11 @@ export PENPOT_FLAGS="\ enable-auto-file-snapshot \ enable-webhooks \ enable-access-tokens \ - enable-tiered-file-data-storage \ + disable-tiered-file-data-storage \ enable-file-validation \ enable-file-schema-validation \ enable-subscriptions \ - enable-subscriptions-old"; + disable-subscriptions-old"; # Default deletion delay for devenv export PENPOT_DELETION_DELAY="24h" diff --git a/backend/scripts/start-dev b/backend/scripts/start-dev index a611eb485e..b5050e4e82 100755 --- a/backend/scripts/start-dev +++ b/backend/scripts/start-dev @@ -13,7 +13,7 @@ export PENPOT_FLAGS="\ enable-login-with-ldap \ enable-transit-readable-response \ enable-demo-users \ - enable-feature-fdata-pointer-map \ + disable-feature-fdata-pointer-map \ enable-feature-fdata-objects-map \ disable-secure-session-cookies \ enable-rpc-climit \ @@ -21,11 +21,11 @@ export PENPOT_FLAGS="\ enable-quotes \ enable-file-snapshot \ enable-access-tokens \ - enable-tiered-file-data-storage \ + disable-tiered-file-data-storage \ enable-file-validation \ enable-file-schema-validation \ enable-subscriptions \ - enable-subscriptions-old "; + disable-subscriptions-old"; # Default deletion delay for devenv export PENPOT_DELETION_DELAY="24h" From 37cec8891f44b0c320006193783d8648d3f32d0f Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 22 Jul 2025 14:09:33 +0200 Subject: [PATCH 5/5] :tada: Add inplace binfile import support --- backend/src/app/binfile/common.clj | 168 +++++++++++------- backend/src/app/binfile/v3.clj | 159 ++++++++++------- backend/src/app/features/fdata.clj | 10 +- backend/src/app/features/file_migrations.clj | 24 ++- backend/src/app/rpc/commands/binfile.clj | 21 ++- backend/src/app/rpc/commands/files_create.clj | 9 +- backend/src/app/util/pointer_map.clj | 6 +- common/src/app/common/types/file.cljc | 2 + 8 files changed, 255 insertions(+), 144 deletions(-) diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index 072327792b..b6457714c5 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -15,13 +15,14 @@ [app.common.files.migrations :as fmg] [app.common.files.validate :as fval] [app.common.logging :as l] + [app.common.schema :as sm] [app.common.types.file :as ctf] [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] [app.db.sql :as sql] - [app.features.fdata :as feat.fdata] - [app.features.file-migrations :as feat.fmigr] + [app.features.fdata :as fdata] + [app.features.file-migrations :as fmigr] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] [app.storage :as sto] @@ -32,12 +33,14 @@ [clojure.set :as set] [cuerdas.core :as str] [datoteka.fs :as fs] - [datoteka.io :as io])) + [datoteka.io :as io] + [promesa.exec :as px])) (set! *warn-on-reflection* true) (def ^:dynamic *state* nil) (def ^:dynamic *options* nil) +(def ^:dynamic *reference-file* nil) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; DEFAULTS @@ -53,17 +56,12 @@ (* 1024 1024 100)) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (declare get-resolved-file-libraries) +(declare update-file!) (def file-attrs - #{:id - :name - :migrations - :features - :project-id - :is-shared - :version - :data}) + (sm/keys ctf/schema:file)) (defn parse-file-format [template] @@ -151,22 +149,33 @@ changes (assoc :changes (blob/decode changes)) data (assoc :data (blob/decode data))))) +(def sql:get-minimal-file + "SELECT f.id, + f.revn, + f.modified_at, + f.deleted_at + FROM file AS f + WHERE f.id = ?") + +(defn get-minimal-file + [cfg id & {:as opts}] + (db/get-with-sql cfg [sql:get-minimal-file id] opts)) (defn decode-file "A general purpose file decoding function that resolves all external pointers, run migrations and return plain vanilla file map" [cfg {:keys [id] :as file} & {:keys [migrate?] :or {migrate? true}}] - (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)] + (binding [pmap/*load-fn* (partial fdata/load-pointer cfg id)] (let [file (->> file - (feat.fmigr/resolve-applied-migrations cfg) - (feat.fdata/resolve-file-data cfg)) + (fmigr/resolve-applied-migrations cfg) + (fdata/resolve-file-data cfg)) libs (delay (get-resolved-file-libraries cfg file))] (-> file (update :features db/decode-pgarray #{}) (update :data blob/decode) - (update :data feat.fdata/process-pointers deref) - (update :data feat.fdata/process-objects (partial into {})) + (update :data fdata/process-pointers deref) + (update :data fdata/process-objects (partial into {})) (update :data assoc :id id) (cond-> migrate? (fmg/migrate-file libs)))))) @@ -421,6 +430,27 @@ (db/exec-one! conn ["SET LOCAL idle_in_transaction_session_timeout = 0"]) (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]))) +(defn invalidate-thumbnails + [cfg file-id] + (let [storage (sto/resolve cfg) + + sql-1 + (str "update file_tagged_object_thumbnail " + " set deleted_at = now() " + " where file_id=? returning media_id") + + sql-2 + (str "update file_thumbnail " + " set deleted_at = now() " + " where file_id=? returning media_id")] + + (run! #(sto/touch-object! storage %) + (sequence + (keep :media-id) + (concat + (db/exec! cfg [sql-1 file-id]) + (db/exec! cfg [sql-2 file-id])))))) + (defn process-file [cfg {:keys [id] :as file}] (let [libs (delay (get-resolved-file-libraries cfg file))] @@ -445,77 +475,79 @@ (vary-meta dissoc ::fmg/migrated)))) (defn encode-file - [{:keys [::db/conn] :as cfg} {:keys [id features] :as file}] - (let [file (if (contains? features "fdata/objects-map") - (feat.fdata/enable-objects-map file) + [{:keys [::wrk/executor] :as cfg} {:keys [id features] :as file}] + (let [file (if (and (contains? features "fdata/objects-map") + (:data file)) + (fdata/enable-objects-map file) file) - file (if (contains? features "fdata/pointer-map") - (binding [pmap/*tracked* (pmap/create-tracked)] - (let [file (feat.fdata/enable-pointer-map file)] - (feat.fdata/persist-pointers! cfg id) + file (if (and (contains? features "fdata/pointer-map") + (:data file)) + + (binding [pmap/*tracked* (pmap/create-tracked :inherit true)] + (let [file (fdata/enable-pointer-map file)] + (fdata/persist-pointers! cfg id) file)) file)] (-> file - (update :features db/encode-pgarray conn "text") - (update :data blob/encode)))) + (d/update-when :features into-array) + (d/update-when :data (fn [data] (px/invoke! executor #(blob/encode data))))))) -(defn get-params-from-file +(defn- file->params [file] - (let [params {:has-media-trimmed (:has-media-trimmed file) - :ignore-sync-until (:ignore-sync-until file) - :project-id (:project-id file) - :features (:features file) - :name (:name file) - :is-shared (:is-shared file) - :version (:version file) - :data (:data file) - :id (:id file) - :deleted-at (:deleted-at file) - :created-at (:created-at file) - :modified-at (:modified-at file) - :revn (:revn file) - :vern (:vern file)}] - - (-> (d/without-nils params) - (assoc :data-backend nil) - (assoc :data-ref-id nil)))) + (-> (select-keys file file-attrs) + (dissoc :team-id) + (dissoc :migrations))) (defn insert-file! - "Insert a new file into the database table" + "Insert a new file into the database table. Expectes a not-encoded file. + Returns nil." [{:keys [::db/conn] :as cfg} file & {:as opts}] - (feat.fmigr/upsert-migrations! conn file) - (let [params (-> (encode-file cfg file) - (get-params-from-file))] - (db/insert! conn :file params opts))) + + (when (:migrations file) + (fmigr/upsert-migrations! conn file)) + + (let [file (encode-file cfg file)] + (db/insert! conn :file + (file->params file) + {::db/return-keys false}) + nil)) (defn update-file! - "Update an existing file on the database." - [{:keys [::db/conn ::sto/storage] :as cfg} {:keys [id] :as file} & {:as opts}] - (let [file (encode-file cfg file) - params (-> (get-params-from-file file) - (dissoc :id))] + "Update an existing file on the database. Expects not encoded file." + [{:keys [::db/conn] :as cfg} {:keys [id] :as file} & {:as opts}] - ;; If file was already offloaded, we touch the underlying storage - ;; object for properly trigger storage-gc-touched task - (when (feat.fdata/offloaded? file) - (some->> (:data-ref-id file) (sto/touch-object! storage))) + (if (::reset-migrations opts false) + (fmigr/reset-migrations! conn file) + (fmigr/upsert-migrations! conn file)) - (feat.fmigr/upsert-migrations! conn file) - (db/update! conn :file params {:id id} opts))) + (let [file + (encode-file cfg file) + + params + (file->params (dissoc file :id))] + + (db/update! conn :file params + {:id id} + {::db/return-keys false}) + + nil)) (defn save-file! "Applies all the final validations and perist the file, binfile - specific, should not be used outside of binfile domain" + specific, should not be used outside of binfile domain. + + Returns nil" [{:keys [::timestamp] :as cfg} file & {:as opts}] (assert (dt/instant? timestamp) "expected valid timestamp") - (let [file (-> file (assoc :created-at timestamp) (assoc :modified-at timestamp) - (assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5}))) + (cond-> (not (::overwrite cfg)) + (assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5})))) + (update :revn inc) (update :features (fn [features] (-> (::features cfg #{}) @@ -532,8 +564,9 @@ (when (ex/exception? result) (l/error :hint "file schema validation error" :cause result)))) - (insert-file! cfg file opts))) - + (if (::overwrite cfg) + (update-file! cfg file (assoc opts ::reset-migrations true)) + (insert-file! cfg file opts)))) (def ^:private sql:get-file-libraries "WITH RECURSIVE libs AS ( @@ -558,7 +591,8 @@ l.revn, l.vern, l.synced_at, - l.is_shared + l.is_shared, + l.version FROM libs AS l INNER JOIN project AS p ON (p.id = l.project_id) WHERE l.deleted_at IS NULL OR l.deleted_at > now();") @@ -573,6 +607,8 @@ (map decode-row)) (db/exec! conn [sql:get-file-libraries file-id]))) +;; FIXME: this will use a lot of memory if file uses too many big +;; libraries, we should load required libraries on demand (defn get-resolved-file-libraries "A helper for preload file libraries" [{:keys [::db/conn] :as cfg} file] diff --git a/backend/src/app/binfile/v3.clj b/backend/src/app/binfile/v3.clj index 7208a54f39..4db81f7524 100644 --- a/backend/src/app/binfile/v3.clj +++ b/backend/src/app/binfile/v3.clj @@ -284,10 +284,12 @@ (assoc :options (:options data)) :always - (dissoc :data) + (dissoc :data)) + file (cond-> file :always (encode-file)) + path (str "files/" file-id ".json")] (write-entry! output path file)) @@ -544,15 +546,18 @@ (json/read reader))) (defn- read-file - [{:keys [::bfc/input ::file-id]}] + [{:keys [::bfc/input ::bfc/timestamp]} file-id] (let [path (str "files/" file-id ".json") entry (get-zip-entry input path)] (-> (read-entry input entry) (decode-file) + (update :revn d/nilv 1) + (update :created-at d/nilv timestamp) + (update :modified-at d/nilv timestamp) (validate-file)))) (defn- read-file-plugin-data - [{:keys [::bfc/input ::file-id]}] + [{:keys [::bfc/input]} file-id] (let [path (str "files/" file-id "/plugin-data.json") entry (get-zip-entry* input path)] (some->> entry @@ -561,7 +566,7 @@ (validate-plugin-data)))) (defn- read-file-media - [{:keys [::bfc/input ::file-id ::entries]}] + [{:keys [::bfc/input ::entries]} file-id] (->> (keep (match-media-entry-fn file-id) entries) (reduce (fn [result {:keys [id entry]}] (let [object (->> (read-entry input entry) @@ -581,7 +586,7 @@ (not-empty))) (defn- read-file-colors - [{:keys [::bfc/input ::file-id ::entries]}] + [{:keys [::bfc/input ::entries]} file-id] (->> (keep (match-color-entry-fn file-id) entries) (reduce (fn [result {:keys [id entry]}] (let [object (->> (read-entry input entry) @@ -594,7 +599,7 @@ (not-empty))) (defn- read-file-components - [{:keys [::bfc/input ::file-id ::entries]}] + [{:keys [::bfc/input ::entries]} file-id] (let [clean-component-post-decode (fn [component] (d/update-when component :objects @@ -625,7 +630,7 @@ (not-empty)))) (defn- read-file-typographies - [{:keys [::bfc/input ::file-id ::entries]}] + [{:keys [::bfc/input ::entries]} file-id] (->> (keep (match-typography-entry-fn file-id) entries) (reduce (fn [result {:keys [id entry]}] (let [object (->> (read-entry input entry) @@ -638,14 +643,14 @@ (not-empty))) (defn- read-file-tokens-lib - [{:keys [::bfc/input ::file-id ::entries]}] + [{:keys [::bfc/input ::entries]} file-id] (when-let [entry (d/seek (match-tokens-lib-entry-fn file-id) entries)] (->> (read-plain-entry input entry) (decode-tokens-lib) (validate-tokens-lib)))) (defn- read-file-shapes - [{:keys [::bfc/input ::file-id ::page-id ::entries] :as cfg}] + [{:keys [::bfc/input ::entries] :as cfg} file-id page-id] (->> (keep (match-shape-entry-fn file-id page-id) entries) (reduce (fn [result {:keys [id entry]}] (let [object (->> (read-entry input entry) @@ -659,15 +664,14 @@ (not-empty))) (defn- read-file-pages - [{:keys [::bfc/input ::file-id ::entries] :as cfg}] + [{:keys [::bfc/input ::entries] :as cfg} file-id] (->> (keep (match-page-entry-fn file-id) entries) (keep (fn [{:keys [id entry]}] (let [page (->> (read-entry input entry) (decode-page)) page (dissoc page :options)] (when (= id (:id page)) - (let [objects (-> (assoc cfg ::page-id id) - (read-file-shapes))] + (let [objects (read-file-shapes cfg file-id id)] (assoc page :objects objects)))))) (sort-by :index) (reduce (fn [result {:keys [id] :as page}] @@ -675,7 +679,7 @@ (d/ordered-map)))) (defn- read-file-thumbnails - [{:keys [::bfc/input ::file-id ::entries] :as cfg}] + [{:keys [::bfc/input ::entries] :as cfg} file-id] (->> (keep (match-thumbnail-entry-fn file-id) entries) (reduce (fn [result {:keys [page-id frame-id tag entry]}] (let [object (->> (read-entry input entry) @@ -690,13 +694,13 @@ (not-empty))) (defn- read-file-data - [cfg] - (let [colors (read-file-colors cfg) - typographies (read-file-typographies cfg) - tokens-lib (read-file-tokens-lib cfg) - components (read-file-components cfg) - plugin-data (read-file-plugin-data cfg) - pages (read-file-pages cfg)] + [cfg file-id] + (let [colors (read-file-colors cfg file-id) + typographies (read-file-typographies cfg file-id) + tokens-lib (read-file-tokens-lib cfg file-id) + components (read-file-components cfg file-id) + plugin-data (read-file-plugin-data cfg file-id) + pages (read-file-pages cfg file-id)] {:pages (-> pages keys vec) :pages-index (into {} pages) :colors colors @@ -706,11 +710,11 @@ :plugin-data plugin-data})) (defn- import-file - [{:keys [::bfc/project-id ::file-id ::file-name] :as cfg}] + [{:keys [::bfc/project-id] :as cfg} {file-id :id file-name :name}] (let [file-id' (bfc/lookup-index file-id) - file (read-file cfg) - media (read-file-media cfg) - thumbnails (read-file-thumbnails cfg)] + file (read-file cfg file-id) + media (read-file-media cfg file-id) + thumbnails (read-file-thumbnails cfg file-id)] (l/dbg :hint "processing file" :id (str file-id') @@ -740,7 +744,7 @@ (vswap! bfc/*state* update :index bfc/update-index (map :media-id thumbnails)) (vswap! bfc/*state* update :thumbnails into thumbnails)) - (let [data (-> (read-file-data cfg) + (let [data (-> (read-file-data cfg file-id) (d/without-nils) (assoc :id file-id') (cond-> (:options file) @@ -757,7 +761,7 @@ file (ctf/check-file file)] (bfm/register-pending-migrations! cfg file) - (bfc/save-file! cfg file ::db/return-keys false) + (bfc/save-file! cfg file) file-id'))) @@ -853,7 +857,8 @@ :file-id (str (:file-id params)) ::l/sync? true) - (db/insert! conn :file-media-object params)))) + (db/insert! conn :file-media-object params + ::db/on-conflict-do-nothing? (::bfc/overwrite cfg))))) (defn- import-file-thumbnails [{:keys [::db/conn] :as cfg}] @@ -873,17 +878,77 @@ :media-id (str media-id) ::l/sync? true) - (db/insert! conn :file-tagged-object-thumbnail params)))) + (db/insert! conn :file-tagged-object-thumbnail params + {::db/on-conflict-do-nothing? true})))) + +(defn- import-files* + [{:keys [::manifest] :as cfg}] + (bfc/disable-database-timeouts! cfg) + + (vswap! bfc/*state* update :index bfc/update-index (:files manifest) :id) + + (let [files (get manifest :files) + result (reduce (fn [result {:keys [id] :as file}] + (let [name' (get file :name) + name' (if (map? name) + (get name id) + name') + file (assoc file :name name')] + (conj result (import-file cfg file)))) + [] + files)] + + (import-file-relations cfg) + (import-storage-objects cfg) + (import-file-media cfg) + (import-file-thumbnails cfg) + + (bfm/apply-pending-migrations! cfg) + + result)) + +(defn- import-file-and-overwrite* + [{:keys [::manifest ::bfc/file-id] :as cfg}] + + (when (not= 1 (count (:files manifest))) + (ex/raise :type :validation + :code :invalid-condition + :hint "unable to perform in-place update with binfile containing more than 1 file" + :manifest manifest)) + + (bfc/disable-database-timeouts! cfg) + + (let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true) + file (first (get manifest :files)) + cfg (assoc cfg ::bfc/overwrite true)] + + (vswap! bfc/*state* update :index assoc (:id file) file-id) + + (binding [bfc/*options* cfg + bfc/*reference-file* ref-file] + + (import-file cfg file) + (import-storage-objects cfg) + (import-file-media cfg) + + (bfc/invalidate-thumbnails cfg file-id) + (bfm/apply-pending-migrations! cfg) + + [file-id]))) (defn- import-files - [{:keys [::bfc/timestamp ::bfc/input ::bfc/name] :or {timestamp (dt/now)} :as cfg}] + [{:keys [::bfc/timestamp ::bfc/input] :or {timestamp (dt/now)} :as cfg}] (assert (instance? ZipFile input) "expected zip file") (assert (dt/instant? timestamp) "expected valid instant") (let [manifest (-> (read-manifest input) (validate-manifest)) - entries (read-zip-entries input)] + entries (read-zip-entries input) + cfg (-> cfg + (assoc ::entries entries) + (assoc ::manifest manifest) + (assoc ::bfc/timestamp timestamp))] (when-not (= "penpot/export-files" (:type manifest)) (ex/raise :type :validation @@ -891,7 +956,6 @@ :hint "unexpected type on manifest" :manifest manifest)) - ;; Check if all files referenced on manifest are present (doseq [{file-id :id features :features} (:files manifest)] (let [path (str "files/" file-id ".json")] @@ -907,35 +971,10 @@ (events/tap :progress {:section :manifest}) - (let [index (bfc/update-index (map :id (:files manifest))) - state {:media [] :index index} - cfg (-> cfg - (assoc ::entries entries) - (assoc ::manifest manifest) - (assoc ::bfc/timestamp timestamp))] - - (binding [bfc/*state* (volatile! state)] - (db/tx-run! cfg (fn [cfg] - (bfc/disable-database-timeouts! cfg) - (let [ids (->> (:files manifest) - (reduce (fn [result {:keys [id] :as file}] - (let [name' (get file :name) - name' (if (map? name) - (get name id) - name')] - (conj result (-> cfg - (assoc ::file-id id) - (assoc ::file-name name') - (import-file))))) - []))] - (import-file-relations cfg) - (import-storage-objects cfg) - (import-file-media cfg) - (import-file-thumbnails cfg) - - (bfm/apply-pending-migrations! cfg) - - ids))))))) + (binding [bfc/*state* (volatile! {:media [] :index {}})] + (if (::bfc/file-id cfg) + (db/tx-run! cfg import-file-and-overwrite*) + (db/tx-run! cfg import-files*))))) ;; --- PUBLIC API diff --git a/backend/src/app/features/fdata.clj b/backend/src/app/features/fdata.clj index 17ec7b1ec2..c59cdd0ca4 100644 --- a/backend/src/app/features/fdata.clj +++ b/backend/src/app/features/fdata.clj @@ -18,7 +18,9 @@ [app.storage :as sto] [app.util.blob :as blob] [app.util.objects-map :as omap] - [app.util.pointer-map :as pmap])) + [app.util.pointer-map :as pmap] + [app.worker :as wrk] + [promesa.exec :as px])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; OFFLOAD @@ -81,6 +83,12 @@ (let [data (get-file-data system file)] (assoc file :data data))) +(defn decode-file-data + [{:keys [::wrk/executor]} {:keys [data] :as file}] + (cond-> file + (bytes? data) + (assoc :data (px/invoke! executor #(blob/decode data))))) + (defn load-pointer "A database loader pointer helper" [system file-id id] diff --git a/backend/src/app/features/file_migrations.clj b/backend/src/app/features/file_migrations.clj index 9552d78ba4..ab1d7c828d 100644 --- a/backend/src/app/features/file_migrations.clj +++ b/backend/src/app/features/file_migrations.clj @@ -8,6 +8,7 @@ "Backend specific code for file migrations. Implemented as permanent feature of files." (:require [app.common.data :as d] + [app.common.exceptions :as ex] [app.common.files.migrations :as fmg :refer [xf:map-name]] [app.db :as db] [app.db.sql :as-alias sql])) @@ -26,12 +27,19 @@ (defn upsert-migrations! "Persist or update file migrations. Return the updated/inserted number of rows" - [conn {:keys [id] :as file}] - (let [migrations (or (-> file meta ::fmg/migrated) - (-> file :migrations not-empty) - fmg/available-migrations) + [cfg {:keys [id] :as file}] + (let [conn (db/get-connection cfg) + migrations (or (-> file meta ::fmg/migrated) + (-> file :migrations)) columns [:file-id :name] - rows (mapv (fn [name] [id name]) migrations)] + rows (->> migrations + (mapv (fn [name] [id name])) + (not-empty))] + + (when-not rows + (ex/raise :type :internal + :code :missing-migrations + :hint "no migrations available on file")) (-> (db/insert-many! conn :file-migration columns rows {::db/return-keys false @@ -40,6 +48,6 @@ (defn reset-migrations! "Replace file migrations" - [conn {:keys [id] :as file}] - (db/delete! conn :file-migration {:file-id id}) - (upsert-migrations! conn file)) + [cfg {:keys [id] :as file}] + (db/delete! cfg :file-migration {:file-id id}) + (upsert-migrations! cfg file)) diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj index 0dc301c71e..bb077776eb 100644 --- a/backend/src/app/rpc/commands/binfile.clj +++ b/backend/src/app/rpc/commands/binfile.clj @@ -125,21 +125,35 @@ [:name [:or [:string {:max 250}] [:map-of ::sm/uuid [:string {:max 250}]]]] [:project-id ::sm/uuid] + [:file-id {:optional true} ::sm/uuid] [:version {:optional true} ::sm/int] [:file ::media/upload]]) (sv/defmethod ::import-binfile - "Import a penpot file in a binary format." + "Import a penpot file in a binary format. If `file-id` is provided, + an in-place import will be performed instead of creating a new file. + + The in-place imports are only supported for binfile-v3 and when a + .penpot file only contains one penpot file. + " {::doc/added "1.15" + ::doc/changes ["1.20" "Add file-id param for in-place import" + "1.20" "Set default version to 3"] + ::webhooks/event? true ::sse/stream? true ::sm/params schema:import-binfile} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version file] :as params}] + [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id project-id version file-id file] :as params}] (projects/check-edition-permissions! pool profile-id project-id) - (let [version (or version 1) + (let [version (or version 3) params (-> params (assoc :profile-id profile-id) (assoc :version version)) + + cfg (cond-> cfg + (uuid? file-id) + (assoc ::bfc/file-id file-id)) + manifest (case (int version) 1 nil 3 (bf.v3/get-manifest (:path file)))] @@ -147,5 +161,6 @@ (with-meta (sse/response (partial import-binfile cfg params)) {::audit/props {:file nil + :file-id file-id :generated-by (:generated-by manifest) :referer (:referer manifest)}}))) diff --git a/backend/src/app/rpc/commands/files_create.clj b/backend/src/app/rpc/commands/files_create.clj index a4c9069e07..64ef7338e1 100644 --- a/backend/src/app/rpc/commands/files_create.clj +++ b/backend/src/app/rpc/commands/files_create.clj @@ -29,6 +29,7 @@ [conn {:keys [file-id profile-id role]}] (let [params {:file-id file-id :profile-id profile-id}] + (->> (perms/assign-role-flags params role) (db/insert! conn :file-profile-rel)))) @@ -51,12 +52,12 @@ :is-shared is-shared :features features :ignore-sync-until ignore-sync-until - :modified-at modified-at + :created-at modified-at :deleted-at deleted-at} {:create-page create-page - :page-id page-id}) - file (-> (bfc/insert-file! cfg file) - (bfc/decode-row))] + :page-id page-id})] + + (bfc/insert-file! cfg file) (->> (assoc params :file-id (:id file) :role :owner) (create-file-role! conn)) diff --git a/backend/src/app/util/pointer_map.clj b/backend/src/app/util/pointer_map.clj index ba84d3d4be..eb5fc19b20 100644 --- a/backend/src/app/util/pointer_map.clj +++ b/backend/src/app/util/pointer_map.clj @@ -61,8 +61,10 @@ (declare create) (defn create-tracked - [] - (atom {})) + [& {:keys [inherit]}] + (if inherit + (atom (if *tracked* @*tracked* {})) + (atom {}))) (defprotocol IPointerMap (get-id [_]) diff --git a/common/src/app/common/types/file.cljc b/common/src/app/common/types/file.cljc index df75669056..08b62e686e 100644 --- a/common/src/app/common/types/file.cljc +++ b/common/src/app/common/types/file.cljc @@ -102,6 +102,7 @@ [:project-id {:optional true} ::sm/uuid] [:team-id {:optional true} ::sm/uuid] [:is-shared {:optional true} ::sm/boolean] + [:has-media-trimmed {:optional true} ::sm/boolean] [:data {:optional true} schema:data] [:version :int] [:features ::cfeat/features] @@ -188,6 +189,7 @@ :features features :migrations migrations :ignore-sync-until ignore-sync-until + :has-media-trimmed false :created-at created-at :modified-at modified-at :deleted-at deleted-at})]