diff --git a/backend/deps.edn b/backend/deps.edn index 4a57cc1abe..b513f50eee 100644 --- a/backend/deps.edn +++ b/backend/deps.edn @@ -40,7 +40,6 @@ nrepl/nrepl {:mvn/version "1.8.0"} org.postgresql/postgresql {:mvn/version "42.7.13"} - org.xerial/sqlite-jdbc {:mvn/version "3.53.4.0"} com.zaxxer/HikariCP {:mvn/version "7.1.0"} diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index b6a701b784..c81de59c90 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -24,7 +24,6 @@ [app.common.weak :as weak] [app.config :as cf] [app.db :as db] - [app.db.sql :as sql] [app.features.fdata :as fdata] [app.features.file-migrations :as fmigr] [app.loggers.audit :as-alias audit] @@ -129,18 +128,12 @@ (defn initial-state [] {:storage-objects #{} - :files #{} - :teams #{} - :projects #{}}) + :files #{}}) (defn collect-storage-objects [state items] (update state :storage-objects into xf-map-media-id items)) -(defn collect-summary - [state key items] - (update state key into xf-map-media-id items)) - (defn lookup-index [id] (when id @@ -542,27 +535,6 @@ [cfg project-id] (db/get cfg :project {:id project-id})) -(def ^:private sql:get-teams - "SELECT t.* FROM team WHERE id = ANY(?)") - -(defn get-teams - [cfg ids] - (let [conn (db/get-connection cfg) - ids (db/create-array conn "uuid" ids)] - (->> (db/exec! conn [sql:get-teams ids]) - (map decode-row-features)))) - -(defn get-team - [cfg team-id] - (-> (db/get cfg :team {:id team-id}) - (decode-row-features))) - -(defn get-fonts - [cfg team-id] - (db/query cfg :team-font-variant - {:team-id team-id - :deleted-at nil})) - (def ^:private sql:get-files-rels "SELECT flr.*, fls.synced_at @@ -644,15 +616,6 @@ :deleted-at nil}) (not-empty))) -(defn get-file-thumbnail - "Return the thumbnail for the specified file-id" - [cfg {:keys [id revn]}] - (db/get* cfg :file-thumbnail - {:file-id id - :revn revn - :data nil} - {::sql/columns [:media-id :file-id :revn]})) - (def ^:private sql:get-missing-media-references "SELECT fmo.* FROM file_media_object AS fmo diff --git a/backend/src/app/binfile/v2.clj b/backend/src/app/binfile/v2.clj deleted file mode 100644 index 06bb22b7c8..0000000000 --- a/backend/src/app/binfile/v2.clj +++ /dev/null @@ -1,450 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.binfile.v2 - "A sqlite3 based binary file exportation with support for exportation - of entire team (or multiple teams) at once." - (:refer-clojure :exclude [read]) - (:require - [app.binfile.common :as bfc] - [app.common.data :as d] - [app.common.features :as cfeat] - [app.common.logging :as l] - [app.common.time :as ct] - [app.common.transit :as t] - [app.common.uuid :as uuid] - [app.config :as cf] - [app.db :as db] - [app.db.sql :as sql] - [app.loggers.audit :as-alias audit] - [app.loggers.webhooks :as-alias webhooks] - [app.storage :as sto] - [app.storage.tmp :as tmp] - [app.util.events :as events] - [app.worker :as-alias wrk] - [clojure.set :as set] - [cuerdas.core :as str] - [datoteka.io :as io] - [promesa.util :as pu]) - (:import - java.sql.DriverManager)) - -(set! *warn-on-reflection* true) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; LOW LEVEL API -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -(defn- create-database - ([cfg] - (let [path (tmp/tempfile :prefix "penpot.binfile." :suffix ".sqlite")] - (create-database cfg path))) - ([cfg path] - (let [db (DriverManager/getConnection (str "jdbc:sqlite:" path))] - (assoc cfg ::db db ::path path)))) - -(def ^:private - sql:create-kvdata-table - "CREATE TABLE kvdata ( - tag text NOT NULL, - key text NOT NULL, - val text NOT NULL, - dat blob NULL - )") - -(def ^:private - sql:create-kvdata-index - "CREATE INDEX kvdata__tag_key__idx - ON kvdata (tag, key)") - -(defn- setup-schema! - [{:keys [::db]}] - (db/exec-one! db [sql:create-kvdata-table]) - (db/exec-one! db [sql:create-kvdata-index])) - -(defn- write! - [{:keys [::db]} tag k v & [data]] - (db/insert! db :kvdata - {:tag (d/name tag) - :key (str k) - :val (t/encode-str v {:type :json-verbose}) - :dat data} - {::db/return-keys false})) - -(defn- read-blob - [{:keys [::db]} tag k] - (let [obj (db/get db :kvdata - {:tag (d/name tag) - :key (str k)} - {::sql/columns [:dat]})] - (:dat obj))) - -(defn- read-seq - ([{:keys [::db]} tag] - (->> (db/query db :kvdata - {:tag (d/name tag)} - {::sql/columns [::val]}) - (map :val) - (map t/decode-str))) - ([{:keys [::db]} tag k] - (->> (db/query db :kvdata - {:tag (d/name tag) - :key (str k)} - {::sql/columns [::val]}) - (map :val) - (map t/decode-str)))) - -(defn- read-obj - [{:keys [::db]} tag k] - (let [obj (db/get db :kvdata - {:tag (d/name tag) - :key (str k)} - {::sql/columns [:val]})] - (-> obj :val t/decode-str))) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; IMPORT/EXPORT IMPL -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -(declare ^:private write-project!) -(declare ^:private write-file!) - -(defn- write-team! - [cfg team-id] - - (let [team (bfc/get-team cfg team-id) - fonts (bfc/get-fonts cfg team-id)] - - (events/tap :progress - {:op :export - :section :write-team - :id team-id - :name (:name team)}) - - (l/trc :hint "write" :obj "team" - :id (str team-id) - :fonts (count fonts)) - - (when-let [photo-id (:photo-id team)] - (vswap! bfc/*state* update :storage-objects conj photo-id)) - - (vswap! bfc/*state* update :teams conj team-id) - (vswap! bfc/*state* bfc/collect-storage-objects fonts) - - (write! cfg :team team-id team) - - (doseq [{:keys [id] :as font} fonts] - (vswap! bfc/*state* update :team-font-variants conj id) - (write! cfg :team-font-variant id font)))) - -(defn- write-project! - [cfg project] - (events/tap :progress - {:op :export - :section :write-project - :id (:id project) - :name (:name project)}) - (l/trc :hint "write" :obj "project" :id (str (:id project))) - (write! cfg :project (str (:id project)) project) - (vswap! bfc/*state* update :projects conj (:id project))) - -(defn- write-file! - [cfg file-id] - (let [file (bfc/get-file cfg file-id :realize? true) - thumbs (bfc/get-file-object-thumbnails cfg file-id) - media (bfc/get-file-media cfg file) - rels (bfc/get-files-rels cfg #{file-id})] - - (events/tap :progress - {:op :export - :section :write-file - :id file-id - :name (:name file)}) - - (vswap! bfc/*state* (fn [state] - (-> state - (update :files conj file-id) - (update :file-media-objects into bfc/xf-map-id media) - (bfc/collect-storage-objects thumbs) - (bfc/collect-storage-objects media)))) - - (write! cfg :file file-id file) - (write! cfg :file-rels file-id rels) - - (run! (partial write! cfg :file-media-object file-id) media) - (run! (partial write! cfg :file-object-thumbnail file-id) thumbs) - - (when-let [thumb (bfc/get-file-thumbnail cfg file)] - (vswap! bfc/*state* bfc/collect-storage-objects [thumb]) - (write! cfg :file-thumbnail file-id thumb)) - - (l/trc :hint "write" :obj "file" - :thumbnails (count thumbs) - :rels (count rels) - :media (count media)))) - -(defn- write-storage-object! - [{:keys [::sto/storage] :as cfg} id] - (let [sobj (sto/get-object storage id) - data (with-open [input (sto/get-object-data storage sobj)] - (io/read input))] - - (l/trc :hint "write" :obj "storage-object" :id (str id) :size (:size sobj)) - (write! cfg :storage-object id (meta sobj) data))) - -(defn- read-storage-object! - [{:keys [::sto/storage ::bfc/timestamp] :as cfg} id] - (let [mdata (d/update-when (read-obj cfg :storage-object id) - :content-type bfc/normalize-content-type) - data (read-blob cfg :storage-object id) - ;; NOTE: v2 bundles are legacy and no longer produced; the - ;; sanitizer call stays as defense in depth and is covered by - ;; the shared helper unit tests, with no dedicated v2 - ;; import test. - clean (when (bfc/svg-object? mdata) - (bfc/sanitize-imported-svg mdata data)) - data (or (:bytes clean) data) - hash (or (:hash clean) (sto/calculate-hash data)) - - content (-> (sto/content data) - (sto/wrap-with-hash hash)) - - params (-> mdata - (assoc ::sto/content content) - (assoc ::sto/deduplicate? true) - (assoc ::sto/touched-at timestamp)) - - sobject (sto/put-object! storage params)] - - (vswap! bfc/*state* update :index assoc id (:id sobject)) - - (l/trc :hint "read" :obj "storage-object" - :id (str id) - :new-id (str (:id sobject)) - :size (:size sobject)))) - -(defn read-team! - [{:keys [::db/conn ::bfc/timestamp] :as cfg} team-id] - (l/trc :hint "read" :obj "team" :id (str team-id)) - - (let [team (read-obj cfg :team team-id) - team (-> team - (update :id bfc/lookup-index) - (update :photo-id bfc/lookup-index) - (assoc :created-at timestamp) - (assoc :modified-at timestamp))] - - (events/tap :progress - {:op :import - :section :read-team - :id team-id - :name (:name team)}) - - (db/insert! conn :team - (update team :features db/encode-pgarray conn "text") - ::db/return-keys false) - - (doseq [font (->> (read-seq cfg :team-font-variant) - (filter #(= team-id (:team-id %))))] - (let [font (-> font - (update :id bfc/lookup-index) - (update :team-id bfc/lookup-index) - (update :woff1-file-id bfc/lookup-index) - (update :woff2-file-id bfc/lookup-index) - (update :ttf-file-id bfc/lookup-index) - (update :otf-file-id bfc/lookup-index) - (assoc :created-at timestamp) - (assoc :modified-at timestamp))] - (db/insert! conn :team-font-variant font - ::db/return-keys false))) - - team)) - -(defn read-project! - [{:keys [::db/conn ::bfc/timestamp] :as cfg} project-id] - (l/trc :hint "read" :obj "project" :id (str project-id)) - - (let [project (read-obj cfg :project project-id) - project (-> project - (update :id bfc/lookup-index) - (update :team-id bfc/lookup-index) - (assoc :created-at timestamp) - (assoc :modified-at timestamp))] - - (events/tap :progress - {:op :import - :section :read-project - :id project-id - :name (:name project)}) - - (db/insert! conn :project project - ::db/return-keys false))) - -(defn read-file! - [{:keys [::db/conn ::bfc/timestamp] :as cfg} file-id] - (l/trc :hint "read" :obj "file" :id (str file-id)) - - (let [file (-> (read-obj cfg :file file-id) - (update :id bfc/lookup-index) - (update :project-id bfc/lookup-index)) - file (bfc/process-file cfg file)] - - (events/tap :progress - {:op :import - :section :read-file - :id file-id - :name (:name file)}) - - ;; All features that are enabled and requires explicit migration are - ;; added to the state for a posterior migration step. - (doseq [feature (-> (::bfc/features cfg) - (set/difference cfeat/no-migration-features) - (set/difference (:features file)))] - (vswap! bfc/*state* update :pending-to-migrate (fnil conj []) [feature (:id file)])) - - (bfc/save-file! cfg file ::db/return-keys false)) - - (doseq [thumbnail (read-seq cfg :file-object-thumbnail file-id)] - (let [thumbnail (-> thumbnail - (update :file-id bfc/lookup-index) - (update :media-id bfc/lookup-index)) - file-id (:file-id thumbnail) - - thumbnail (update thumbnail :object-id - #(str/replace-first % #"^(.*?)/" (str file-id "/")))] - - (db/insert! conn :file-tagged-object-thumbnail thumbnail - {::db/return-keys false}))) - - ;; NOTE: v2 packages are only produced and imported through the - ;; developer REPL (`app.srepl.binfile`), never through the product, so - ;; this path keeps inserting the relations as they come: unlike v1 and - ;; v3 it does not skip self-referencing or circular links (see - ;; `bfc/library-link-status`). - (doseq [rel (read-obj cfg :file-rels file-id)] - (let [rel (-> rel - (update :file-id bfc/lookup-index) - (update :library-file-id bfc/lookup-index))] - (db/insert! conn :file-library-rel rel - ::db/return-keys false) - (bfc/upsert-file-library-sync! conn (assoc rel :synced-at timestamp)))) - - (doseq [media (read-seq cfg :file-media-object file-id)] - (let [media (-> media - (update :id bfc/lookup-index) - (update :file-id bfc/lookup-index) - (update :media-id bfc/lookup-index) - (update :thumbnail-id bfc/lookup-index))] - (db/insert! conn :file-media-object media - ::db/return-keys false - ::sql/on-conflict-do-nothing true)))) - -(def ^:private empty-summary - {:teams #{} - :files #{} - :projects #{} - :file-media-objects #{} - :team-font-variants #{} - :storage-objects #{}}) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; PUBLIC API -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -(defn export-team! - [cfg team-id] - (let [id (uuid/next) - tp (ct/tpoint) - cfg (create-database cfg)] - - (l/inf :hint "start" - :operation "export" - :id (str id) - :path (str (::path cfg))) - - (try - (db/tx-run! cfg (fn [cfg] - (setup-schema! cfg) - (binding [bfc/*state* (volatile! empty-summary)] - (write-team! cfg team-id) - - (run! (partial write-project! cfg) - (bfc/get-team-projects cfg team-id)) - - (run! (partial write-file! cfg) - (bfc/get-team-files-ids cfg team-id)) - - (run! (partial write-storage-object! cfg) - (-> bfc/*state* deref :storage-objects)) - - (write! cfg :manifest "team-id" team-id) - (write! cfg :manifest "objects" (deref bfc/*state*)) - - (::path cfg)))) - (finally - (pu/close! (::db cfg)) - - (let [elapsed (tp)] - (l/inf :hint "end" - :operation "export" - :id (str id) - :elapsed (ct/format-duration elapsed))))))) - -(defn import-team! - [cfg path] - (let [id (uuid/next) - tp (ct/tpoint) - - cfg (-> (create-database cfg path) - (assoc ::bfc/timestamp (ct/now)))] - - (l/inf :hint "start" - :operation "import" - :id (str id) - :path (str (::path cfg))) - - (try - (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}] - (db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"]) - (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]) - - (binding [bfc/*state* (volatile! {:index {}})] - (let [objects (read-obj cfg :manifest "objects")] - - ;; We first process all storage objects, they have - ;; deduplication so we can't rely on simple reindex. This - ;; operation populates the index for all storage objects. - (run! (partial read-storage-object! cfg) (:storage-objects objects)) - - ;; Populate index with all the incoming objects - (vswap! bfc/*state* update :index - (fn [index] - (-> index - (bfc/update-index (:teams objects)) - (bfc/update-index (:projects objects)) - (bfc/update-index (:files objects)) - (bfc/update-index (:file-media-objects objects)) - (bfc/update-index (:team-font-variants objects))))) - - (let [team-id (read-obj cfg :manifest "team-id") - team (read-team! cfg team-id) - features (cfeat/get-team-enabled-features cf/flags team) - cfg (assoc cfg ::bfc/features features)] - - (run! (partial read-project! cfg) (:projects objects)) - (run! (partial read-file! cfg) (:files objects)) - - ;; (run-pending-migrations! cfg) - - team))))) - (finally - (pu/close! (::db cfg)) - - (let [elapsed (tp)] - (l/inf :hint "end" - :operation "import" - :id (str id) - :elapsed (ct/format-duration elapsed))))))) diff --git a/backend/src/app/srepl/binfile.clj b/backend/src/app/srepl/binfile.clj deleted file mode 100644 index 43cb69f8f3..0000000000 --- a/backend/src/app/srepl/binfile.clj +++ /dev/null @@ -1,41 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS SUBSIDIARY SL - -(ns app.srepl.binfile - (:require - [app.binfile.v2 :as binfile.v2] - [app.common.uuid :as uuid] - [app.db :as db] - [app.srepl.helpers :as h] - [app.system :as sys] - [cuerdas.core :as str])) - -(defn export-team! - [team-id] - (let [team-id (h/parse-uuid team-id)] - (binfile.v2/export-team! sys/system team-id))) - -(defn import-team! - [path & {:keys [owner rollback?] :or {rollback? true}}] - (db/tx-run! (assoc sys/system ::db/rollback rollback?) - (fn [cfg] - (let [team (binfile.v2/import-team! cfg path) - owner (cond - (string? owner) - (db/get* cfg :profile {:email (str/lower owner)}) - (uuid? owner) - (db/get* cfg :profile {:id owner}))] - - (when owner - (db/insert! cfg :team-profile-rel - {:id (uuid/next) - :team-id (:id team) - :profile-id (:id owner) - :is-admin true - :is-owner true - :can-edit true})) - - team)))) diff --git a/docs/technical-guide/developer/data-model/penpot-file-format.md b/docs/technical-guide/developer/data-model/penpot-file-format.md index 431dbffaa2..970e9276c8 100644 --- a/docs/technical-guide/developer/data-model/penpot-file-format.md +++ b/docs/technical-guide/developer/data-model/penpot-file-format.md @@ -21,7 +21,7 @@ The `.penpot` format (version 3) uses a ZIP container with JSON files for metada | Version | Description | Status | |---------|-------------|--------| | v1 | Custom binary format | Deprecated | -| v2 | SQLite-based format | Never released (internal only) | +| v2 | SQLite-based format | Removed (never released) | | v3 | ZIP + JSON format | **Current** | ## File Structure