From 510d3cfa33c9c8745273e1e0427e4c1a7a3c5017 Mon Sep 17 00:00:00 2001 From: "alonso.torres" Date: Fri, 29 Jan 2021 15:54:20 +0100 Subject: [PATCH] :sparkles: Allows initial data to be extracted/loaded to file --- backend/src/app/config.clj | 4 +- backend/src/app/db/profile_initial_data.clj | 114 +++++++++----------- backend/src/app/srepl/main.clj | 10 ++ backend/src/app/util/transit.clj | 20 +++- 4 files changed, 81 insertions(+), 67 deletions(-) diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index 5fdf166f16..87357a8bc1 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -70,8 +70,8 @@ :ldap-auth-fullname-attribute "displayName" :ldap-auth-avatar-attribute "jpegPhoto" - ;;:initial-data-project-id "5761a890-3b81-11eb-9e7d-556a2f641513" - ;;:initial-data-project-name "Penpot Onboarding" + ;; :initial-data-project-name "Penpot Onboarding" + ;; :initial-data-file "/internal/initial-data.json" }) (s/def ::http-server-port ::us/integer) diff --git a/backend/src/app/db/profile_initial_data.clj b/backend/src/app/db/profile_initial_data.clj index 9982c6eeb8..9366b9dd5d 100644 --- a/backend/src/app/db/profile_initial_data.clj +++ b/backend/src/app/db/profile_initial_data.clj @@ -14,8 +14,11 @@ [app.db :as db] [app.rpc.mutations.projects :as projects] [app.storage :as storage] + [app.util.transit :as tr] [clojure.tools.logging :as log] - [cuerdas.core :as str])) + [cuerdas.core :as str]) + (:import java.io.FileOutputStream + java.io.FileInputStream)) (def sql:file "select * from file where project_id = ?") @@ -39,15 +42,6 @@ from file_media_thumbnail where media_object_id in (select id from media_ids)") -(def sql:storage-object - "with file_ids as (select id from file where project_id = ?), - media_ids as (select media_id as id from file_media_object where file_id in (select id from file_ids)), - thumbs_ids as (select thumbnail_id as id from file_media_object where file_id in (select id from file_ids)), - storage_ids as (select id from media_ids union select id from thumbs_ids) - select * - from storage_object - where id in (select id from storage_ids)") - (defn change-ids "Given a collection and a map from ID to ID. Changes all the `keys` properties so they point to the new ID existing in `map-ids`" @@ -71,68 +65,66 @@ [new-map-ids (map (partial change-id new-map-ids) coll)])) -(defn allocate-storage-objects - "Copies the storage data to a new object and stores the new id into `map-ids`" - [storage map-ids objects] - (let [clone-object - (fn [map-ids object] - (try - (let [object (storage/row->storage-object object) - new-obj (storage/clone-object storage object)] - (assoc map-ids (:id object) (:id new-obj))) - (catch Exception err - (log/errorf "Error cloning store object %s" (:id object)) - map-ids)))] - (->> objects - (reduce clone-object map-ids)))) +(defn create-initial-data-dump + [conn project-id output-file] + (let [ ;; Retrieve data from templates + file (db/exec! conn [sql:file, project-id]) + file-library-rel (db/exec! conn [sql:file-library-rel, project-id]) + file-media-object (db/exec! conn [sql:file-media-object, project-id]) + file-media-thumbnail (db/exec! conn [sql:file-media-thumbnail, project-id]) + + data {:file file + :file-library-rel file-library-rel + :file-media-object file-media-object + :file-media-thumbnail file-media-thumbnail}] + + (with-open [output (FileOutputStream. output-file)] + (tr/encode-stream data output)))) (defn create-profile-initial-data [conn storage profile] - (when-let [sample-project-id (get cfg/config :initial-data-project-id)] - (let [sample-project-name (get cfg/config :initial-data-project-name "Penpot Onboarding") + (let [initial-data-file (get cfg/config :initial-data-file) + initial-data (when initial-data-file + (with-open [input (FileInputStream. initial-data-file)] + (tr/decode-stream input)))] + (when initial-data + (let [{:keys [file file-library-rel file-media-object file-media-thumbnail]} initial-data - proj (projects/create-project conn {:profile-id (:id profile) - :team-id (:default-team-id profile) - :name sample-project-name}) + sample-project-name (get cfg/config :initial-data-project-name "Penpot Onboarding") - _ (projects/create-project-profile conn {:project-id (:id proj) - :profile-id (:id profile)}) - _ (projects/create-team-project-profile conn {:team-id (:default-team-id profile) - :project-id (:id proj) - :profile-id (:id profile)}) + proj (projects/create-project conn {:profile-id (:id profile) + :team-id (:default-team-id profile) + :name sample-project-name}) - ;; Retrieve data from templates - file (db/exec! conn [sql:file, sample-project-id]) - file-library-rel (db/exec! conn [sql:file-library-rel, sample-project-id]) - file-media-object (db/exec! conn [sql:file-media-object, sample-project-id]) - file-media-thumbnail (db/exec! conn [sql:file-media-thumbnail, sample-project-id]) - storage-object (db/exec! conn [sql:storage-object, sample-project-id]) + _ (projects/create-project-profile conn {:project-id (:id proj) + :profile-id (:id profile)}) - map-ids {} + _ (projects/create-team-project-profile conn {:team-id (:default-team-id profile) + :project-id (:id proj) + :profile-id (:id profile)}) - ;; Create new ID's and change the references - [map-ids file] (change-ids map-ids file #{:id}) + map-ids {} - map-ids (allocate-storage-objects storage map-ids storage-object) + ;; Create new ID's and change the references + [map-ids file] (change-ids map-ids file #{:id}) - [map-ids file-library-rel] (change-ids map-ids file-library-rel #{:file-id :library-file-id}) - [map-ids file-media-object] (change-ids map-ids file-media-object #{:id :file-id :media-id :thumbnail-id}) - [map-ids file-media-thumbnail] (change-ids map-ids file-media-thumbnail #{:id :media-object-id}) + [map-ids file-library-rel] (change-ids map-ids file-library-rel #{:file-id :library-file-id}) + [map-ids file-media-object] (change-ids map-ids file-media-object #{:id :file-id :media-id :thumbnail-id}) + [map-ids file-media-thumbnail] (change-ids map-ids file-media-thumbnail #{:id :media-object-id}) - file (->> file (map (fn [data] (assoc data :project-id (:id proj))))) - file-profile-rel (->> file (map (fn [data] - (hash-map :file-id (:id data) - :profile-id (:id profile) - :is-owner true - :is-admin true - :can-edit true))))] + file (->> file (map (fn [data] (assoc data :project-id (:id proj))))) + file-profile-rel (->> file (map (fn [data] + (hash-map :file-id (:id data) + :profile-id (:id profile) + :is-owner true + :is-admin true + :can-edit true))))] - ;; Re-insert into the database - (db/insert-multi! conn :file file) - (db/insert-multi! conn :file-profile-rel file-profile-rel) - (db/insert-multi! conn :file-library-rel file-library-rel) - (db/insert-multi! conn :file-media-object file-media-object) - (db/insert-multi! conn :file-media-thumbnail file-media-thumbnail)) - {:result "OK"})) + ;; Re-insert into the database + (db/insert-multi! conn :file file) + (db/insert-multi! conn :file-profile-rel file-profile-rel) + (db/insert-multi! conn :file-library-rel file-library-rel) + (db/insert-multi! conn :file-media-object file-media-object) + (db/insert-multi! conn :file-media-thumbnail file-media-thumbnail))))) diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index a923449f81..f62dfa91f1 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -5,6 +5,7 @@ [app.common.pages :as cp] [app.common.pages.migrations :as pmg] [app.db :as db] + [app.db.profile-initial-data :as pid] [app.main :refer [system]] [app.srepl.dev :as dev] [app.util.blob :as blob] @@ -46,3 +47,12 @@ ;; Examples ;; (def backup (update-file #uuid "1586e1f0-3e02-11eb-b1d2-556a2f641513" identity)) ;; (def x (update-file #uuid "1586e1f0-3e02-11eb-b1d2-556a2f641513" (fn [{:keys [data] :as file}] (update-in data [:pages-index #uuid "878278c0-3ef0-11eb-9d67-8551e7624f43" :objects] dissoc nil)))) + +(defn initial-data-dump + ([system file] + (let [default-project-id #uuid "5761a890-3b81-11eb-9e7d-556a2f641513"] + (initial-data-dump system default-project-id file))) + + ([system project-id file] + (db/with-atomic [conn (:app.db/pool system)] + (pid/create-initial-data-dump conn file)))) diff --git a/backend/src/app/util/transit.clj b/backend/src/app/util/transit.clj index 59630313df..4a0b535d53 100644 --- a/backend/src/app/util/transit.clj +++ b/backend/src/app/util/transit.clj @@ -94,21 +94,33 @@ (declare str->bytes) (declare bytes->str) +(defn decode-stream + ([input] + (decode-stream input nil)) + ([input opts] + (read! (reader input opts)))) + (defn decode ([data] (decode data nil)) ([data opts] (with-open [input (ByteArrayInputStream. ^bytes data)] - (read! (reader input opts))))) + (decode-stream input opts)))) + +(defn encode-stream + ([data out] + (encode-stream data out nil)) + ([data out opts] + (let [w (writer out opts)] + (write! w data)))) (defn encode ([data] (encode data nil)) ([data opts] (with-open [out (ByteArrayOutputStream.)] - (let [w (writer out opts)] - (write! w data) - (.toByteArray out))))) + (encode-stream data out opts) + (.toByteArray out)))) (defn decode-str [message]