diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index 170a514704..7fb400e154 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -1079,12 +1079,13 @@ :else :detach-libraries)) (defn export-cfg - "The domain cfg of an export: the frozen set of files and how their - libraries are handled." - [cfg {:keys [ids export-type]}] + "The domain cfg of an export: the frozen set of files, how their + libraries are handled, and where the package is written." + [cfg {:keys [file-ids export-type output]}] (-> cfg - (assoc ::ids ids) - (assoc ::export-type export-type))) + (assoc ::file-ids file-ids) + (assoc ::export-type export-type) + (assoc ::output output))) (defn import-cfg "The domain cfg of an import: the destination team/profile/project, the diff --git a/backend/src/app/binfile/jobs.clj b/backend/src/app/binfile/jobs.clj index 950d157220..3bdfe668db 100644 --- a/backend/src/app/binfile/jobs.clj +++ b/backend/src/app/binfile/jobs.clj @@ -5,7 +5,7 @@ ;; Copyright (c) KALEIDOS SUBSIDIARY SL (ns app.binfile.jobs - "The binfile work a job runs. + "The progress bridge of the binfile jobs. A job has no SSE channel, so the `:progress` points that `bf.v1` and `bf.v3` emit become `progress` events of the job through @@ -17,20 +17,15 @@ so the adapter holds no state between taps. The ids the taps carry are not part of the job event contract. - The adapter checks that the job is still active before it starts, and - the beat checks it on every write: a job that was cancelled, or that + The handlers in `app.tasks.export-binfile` and + `app.tasks.import-binfile` run the core through `run` and + guard it with `check-active`: a job that was cancelled, or that already reached a terminal state, raises an `:interrupt` right there on the run thread, so the work stops at its next event point and the transaction of its handler rolls back with it. A run whose last tap happened before the interruption is discarded once it returns, which is what makes that transaction roll back too." (:require - [app.binfile.common :as bfc] - [app.binfile.v1 :as bf.v1] - [app.binfile.v3 :as bf.v3] - [app.common.exceptions :as ex] - [app.common.time :as ct] - [app.db :as db] [app.jobs :as jobs] [app.util.events :as events])) @@ -39,7 +34,7 @@ The substrate owns both the check and the error; this is the adapter's entry point to it, used by the two handlers before their run and by the - tests." + tests. Answers with the cfg, so a handler can thread it." [cfg context] (jobs/check-active cfg (:id context))) @@ -98,10 +93,13 @@ (assoc :counters (cond-> (or outer-counters {}) (some? own) (assoc stage own)))))) -(defn- with-progress +(defn run "Run `f` with the progress taps of the core turned into milestones of the job. + `f` is called with the cfg, so the adapter forwards what it receives + and a caller can pass the core directly. + Every tap is reported: the substrate throttles the durable events, and a milestone that does not fit loses no information because the next one carries the same state. @@ -117,55 +115,6 @@ :job-id job-id :progress (milestone data))))] (binding [events/*sink* sink] - (let [result (f)] + (let [result (f cfg)] (check-active cfg context) result)))) - -(defn export-files - "Export a set of files into `output` as part of a job, reporting - progress as job events." - [cfg context {:keys [ids export-type output]}] - (check-active cfg context) - (with-progress cfg context - #(bf.v3/export-files! (bfc/export-cfg cfg {:ids ids - :export-type export-type}) - output))) - -(defn- touch-project - "Mark the destination project as modified. - - An import adds files, and the triggers of the schema only touch the - project on an update, so a project that just received files keeps its - old date and stays where it was in the list." - [cfg project-id] - (db/update! cfg :project - {:modified-at (ct/now)} - {:id project-id} - {::db/return-keys false})) - -(defn import-files - "Import the package at `input` as part of a job, reporting progress as - job events. The package was read from the resource of the job, so the - caller resolves the destination team and the adapter only fixes what - the legacy RPC and a job must share. - - Both formats answer with the same shape: the version 1 reader returns - the set of files it created, and the version 3 one returns a map that - already carries them together with the resolution of the libraries." - [cfg context {:keys [profile-id project-id team name input version]}] - (check-active cfg context) - (let [cfg (bfc/import-cfg cfg {:profile-id profile-id - :project-id project-id - :team team - :name name - :input input})] - (with-progress cfg context - #(let [result (case (int version) - 1 (bf.v1/import-files! cfg) - 3 (bf.v3/import-files! cfg) - (ex/raise :type :validation - :code :unsupported-version - :hint "unsupported binfile version" - :version version))] - (touch-project cfg project-id) - result)))) diff --git a/backend/src/app/binfile/v1.clj b/backend/src/app/binfile/v1.clj index 649494a2c6..76fc870f5d 100644 --- a/backend/src/app/binfile/v1.clj +++ b/backend/src/app/binfile/v1.clj @@ -329,13 +329,13 @@ [:v1/metadata :v1/files :v1/rels :v1/sobjects])))) (defmethod write-section :v1/metadata - [{:keys [::output ::bfc/ids ::bfc/include-libraries] :as cfg}] - (if-let [fids (get-files cfg ids)] + [{:keys [::output ::bfc/file-ids ::bfc/include-libraries] :as cfg}] + (if-let [fids (get-files cfg file-ids)] (let [lids (when include-libraries - (bfc/get-libraries cfg ids)) - ids (into fids lids)] - (write-obj! output {:version cf/version :files ids}) - (vswap! bfc/*state* assoc :files ids)) + (bfc/get-libraries cfg file-ids)) + file-ids (into fids lids)] + (write-obj! output {:version cf/version :files file-ids}) + (vswap! bfc/*state* assoc :files file-ids)) (ex/raise :type :not-found :code :files-not-found :hint "unable to retrieve files for export"))) @@ -689,12 +689,12 @@ `::bfc/embed-assets`: instead of including the libraries, embed in the same file library all assets used from external libraries." - [{:keys [::bfc/ids] :as cfg} output] + [{:keys [::bfc/file-ids] :as cfg} output] (dm/assert! - "expected a set of uuid's for `::bfc/ids` parameter" - (and (set? ids) - (every? uuid? ids))) + "expected a set of uuid's for `::bfc/file-ids` parameter" + (and (set? file-ids) + (every? uuid? file-ids))) (dm/assert! "expected instance of jio/IOFactory for `input`" diff --git a/backend/src/app/binfile/v3.clj b/backend/src/app/binfile/v3.clj index d599f766ed..59c344cf02 100644 --- a/backend/src/app/binfile/v3.clj +++ b/backend/src/app/binfile/v3.clj @@ -454,14 +454,15 @@ (write-entry! output path encoded-status))))) (defn- export-files - [{:keys [::bfc/ids ::bfc/export-type ::output] :as cfg}] + [{:keys [::bfc/file-ids ::bfc/export-type ::output] :as cfg}] - (let [original-ids ids - ids (into ids (when (= export-type :include-libraries) (bfc/get-libraries cfg ids))) - rels (if (= export-type :include-libraries) - (->> (bfc/get-files-rels cfg ids) - (mapv (juxt :file-id :library-file-id))) - []) + (let [original-ids file-ids + file-ids (into file-ids (when (= export-type :include-libraries) + (bfc/get-libraries cfg file-ids))) + rels (if (= export-type :include-libraries) + (->> (bfc/get-files-rels cfg file-ids) + (mapv (juxt :file-id :library-file-id))) + []) ;; Compute external libraries: referenced by original files but ;; not included in the export set. Only relevant for :link-later. @@ -486,11 +487,11 @@ (vswap! bfc/*state* assoc :files (d/ordered-map)) ;; Write all the exporting files - (doseq [[index file-id] (d/enumerate ids)] + (doseq [[index file-id] (d/enumerate file-ids)] (-> cfg (assoc ::file-id file-id) (with-outer-counter :files {:current (inc index) - :total (count ids)}) + :total (count file-ids)}) (export-file))) ;; Write manifest file @@ -1384,24 +1385,25 @@ (defn export-files! "Do the exportation of a specified file in custom penpot binary - format. There are some options available for customize the output: + format. The cfg carries what `bfc/export-cfg` froze: `::bfc/file-ids` + (the files to export), `::bfc/export-type` (how linked libraries are + handled) and `::bfc/output` (where the package is written). - `::bfc/export-type`: determines how linked libraries are handled. - Valid values: - `:include-libraries` (include linked libraries), + `::bfc/export-type` determines how linked libraries are handled. + Valid values: `:include-libraries` (include linked libraries), `:merge-libraries` (embed library assets in the file), `:detach-libraries` (treat assets as basic objects), `:link-later` (preserve component metadata for relinking on import)." - [{:keys [::bfc/ids] :as cfg} output] + [{:keys [::bfc/file-ids ::bfc/output] :as cfg}] (assert - (and (set? ids) (every? uuid? ids)) - "expected a set of uuid's for `::bfc/ids` parameter") + (and (set? file-ids) (every? uuid? file-ids)) + "expected a set of uuid's for `::bfc/file-ids` parameter") (assert (satisfies? jio/IOFactory output) - "expected instance of jio/IOFactory for `input`") + "expected instance of jio/IOFactory for `::bfc/output`") (let [id (uuid/next) tp (ct/tpoint) diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index 6995464290..01982605b4 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -359,10 +359,11 @@ (let [path (tmp/tempfile :prefix "penpot.export." :min-age "30m")] (with-open [output (io/output-stream path)] (-> cfg - (assoc ::bfc/ids file-ids) + (assoc ::bfc/file-ids file-ids) (assoc ::bfc/embed-assets embed?) (assoc ::bfc/include-libraries libs?) - (bf.v3/export-files! output))) + (assoc ::bfc/output output) + (bf.v3/export-files!))) (if clone? (let [profile (profile/get-profile pool profile-id) diff --git a/backend/src/app/jobs.clj b/backend/src/app/jobs.clj index caf19b6f4e..271414ebcc 100644 --- a/backend/src/app/jobs.clj +++ b/backend/src/app/jobs.clj @@ -847,7 +847,8 @@ The interrupt is an internal signal, not an error: the caller is a task that must stop the work in course. `heartbeat` calls this when one of its writes finds the job gone, and a caller can also ask explicitly, - which is what the binfile adapter does before and after its run." + which is what the binfile adapter does before and after its run. + Answers with the cfg, so a caller can thread the check into its run." [cfg job-id] (let [job (get-job cfg job-id)] (when-not (and (some? job) (contains? active-statuses (:status job))) @@ -855,7 +856,8 @@ :code :job-interrupted :hint "the job is no longer active" :job-id job-id - :status (or (:status job) "gone"))))) + :status (or (:status job) "gone"))) + cfg)) (defn heartbeat "Keep a running job alive and, optionally, report its progress. diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj index 29d3fcdd52..0dc6189639 100644 --- a/backend/src/app/rpc/commands/binfile.clj +++ b/backend/src/app/rpc/commands/binfile.clj @@ -50,9 +50,10 @@ (let [output (tmp/tempfile*)] (try - (-> (bfc/export-cfg cfg {:ids #{file-id} - :export-type (bfc/resolve-export-type params)}) - (bf.v3/export-files! output)) + (-> (bfc/export-cfg cfg {:file-ids #{file-id} + :export-type (bfc/resolve-export-type params) + :output output}) + (bf.v3/export-files!)) (let [data (sto/content output) object (sto/put-object! storage diff --git a/backend/src/app/tasks/export_binfile.clj b/backend/src/app/tasks/export_binfile.clj index eaa1ba0f55..5ac105a087 100644 --- a/backend/src/app/tasks/export_binfile.clj +++ b/backend/src/app/tasks/export_binfile.clj @@ -8,7 +8,9 @@ "The `:export-binfile` job: it produces the `.penpot` artifact of the set of files frozen in its params." (:require + [app.binfile.common :as bfc] [app.binfile.jobs :as bfj] + [app.binfile.v3 :as bf.v3] [app.common.schema :as sm] [app.db :as db] [app.jobs :as jobs] @@ -64,14 +66,17 @@ (doseq [file-id file-ids] (files/check-read-permissions! cfg profile-id file-id)) - (let [output (tmp/tempfile* :suffix ".penpot")] + (let [output (tmp/tempfile* :suffix ".penpot") + cfg (bfc/export-cfg cfg {:file-ids file-ids + :export-type (:export-type params) + :output output})] (try - (bfj/export-files cfg context {:ids file-ids - :export-type (:export-type params) - :output output}) - (let [resource (js/put-resource cfg profile-id - (assoc artifact :content (sto/content output)))] - {:result (check-result (dissoc resource :resource-id)) + (bfj/check-active cfg context) + (bfj/run cfg context bf.v3/export-files!) + (let [resource (->> (assoc artifact :content (sto/content output)) + (js/put-resource cfg profile-id)) + result (check-result (dissoc resource :resource-id))] + {:result result :resource-id (:resource-id resource)}) (finally (fs/delete output)))))) diff --git a/backend/src/app/tasks/import_binfile.clj b/backend/src/app/tasks/import_binfile.clj index 754bb4bf25..27949b0074 100644 --- a/backend/src/app/tasks/import_binfile.clj +++ b/backend/src/app/tasks/import_binfile.clj @@ -8,9 +8,13 @@ "The `:import-binfile` job: it imports the package the job owns as a resource into the project frozen in its params." (:require + [app.binfile.common :as bfc] [app.binfile.jobs :as bfj] + [app.binfile.v1 :as bf.v1] + [app.binfile.v3 :as bf.v3] [app.common.exceptions :as ex] [app.common.schema :as sm] + [app.common.time :as ct] [app.db :as db] [app.jobs :as jobs] [app.jobs.storage :as js] @@ -101,6 +105,18 @@ :type :validation :code :invalid-result)) +(defn- touch-project + "Mark the destination project as modified. + + An import adds files, and the triggers of the schema only touch the + project on an update, so a project that just received files keeps its + old date and stays where it was in the list." + [cfg project-id] + (db/update! cfg :project + {:modified-at (ct/now)} + {:id project-id} + {::db/return-keys false})) + (defn execute-import "Plain handler, importable and testable without integrant. @@ -124,32 +140,38 @@ (try (projects/check-edition-permissions! cfg profile-id project-id) - (let [team (or (teams/get-team cfg - :profile-id profile-id - :project-id project-id) - (ex/raise :type :not-found - :code :project-not-found - :hint "the destination project is not available" - :profile-id profile-id - :project-id project-id)) - result (db/tx-run! cfg - (fn [tx-cfg] - ;; the core joins this transaction: it - ;; is the one that decides whether the - ;; whole import is kept or rolled back - (check-result - (assoc (bfj/import-files tx-cfg context - {:profile-id profile-id - :project-id project-id - :team team - :name (:name params) - :input input - :version (:version params)}) - :name (:name params) - :version (:version params)))))] - - result) - + (let [team (or (teams/get-team cfg + :profile-id profile-id + :project-id project-id) + (ex/raise :type :not-found + :code :project-not-found + :hint "the destination project is not available" + :profile-id profile-id + :project-id project-id))] + (db/tx-run! cfg (fn [tx-cfg] + ;; the core joins this transaction: it + ;; is the one that decides whether the + ;; whole import is kept or rolled back + (-> (bfc/import-cfg tx-cfg {:profile-id profile-id + :project-id project-id + :team team + :name (:name params) + :input input}) + (bfj/check-active context) + (bfj/run context + (fn [cfg] + (let [result (case (int (:version params)) + 1 (bf.v1/import-files! cfg) + 3 (bf.v3/import-files! cfg) + (ex/raise :type :validation + :code :unsupported-version + :hint "unsupported binfile version" + :version (:version params)))] + (touch-project cfg project-id) + result))) + (assoc :name (:name params)) + (assoc :version (:version params)) + (check-result))))) (finally (fs/delete input) (js/release-input cfg context))))) diff --git a/backend/test/backend_tests/binfile_jobs_test.clj b/backend/test/backend_tests/binfile_jobs_test.clj index e03cb79421..d5f33dbdd9 100644 --- a/backend/test/backend_tests/binfile_jobs_test.clj +++ b/backend/test/backend_tests/binfile_jobs_test.clj @@ -10,6 +10,7 @@ (:require [app.binfile.common :as bfc] [app.binfile.jobs :as bfj] + [app.binfile.v1 :as v1] [app.binfile.v3 :as v3] [app.common.time :as ct] [app.common.uuid :as uuid] @@ -59,6 +60,37 @@ [job-id] (jobs/make-context (jobs/get-job th/*system* job-id))) +(defn- run-export! + "Run an export through the adapter, as the job handler does: the + frozen cfg in, the progress out as job events." + [cfg context file-ids export-type output] + (let [cfg (bfc/export-cfg cfg {:file-ids file-ids + :export-type export-type + :output output})] + (bfj/check-active cfg context) + (bfj/run cfg context v3/export-files!))) + +(defn- run-import! + "Run an import through the adapter, as the job handler does: the + frozen cfg in, the progress out as job events. The project touch + stays in the handler, covered by its own tests." + [cfg context {:keys [profile-id project-id team name input version]}] + (bfj/check-active cfg context) + (let [cfg (bfc/import-cfg cfg {:profile-id profile-id + :project-id project-id + :team team + :name name + :input input})] + (bfj/run cfg context + (fn [cfg] + (case (int version) + 1 (v1/import-files! cfg) + 3 (v3/import-files! cfg) + (throw (ex-info "unsupported binfile version" + {:type :validation + :code :unsupported-version + :version version}))))))) + (defn- progress-events "The progress events of a job, oldest first, with decoded payloads." [job-id] @@ -88,9 +120,9 @@ :library-file-id (:id library)}) (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) (db/update! th/*system* :file {:deleted-at (ct/now)} @@ -118,10 +150,8 @@ job-id (th/create-job* :export-binfile {:profile-id (:id profile)}) output (tmp/tempfile* :suffix ".penpot")] - (bfj/export-files (job-cfg) (job-context job-id) - {:ids #{file-id} - :export-type :detach-libraries - :output output}) + (run-export! (job-cfg) (job-context job-id) + #{file-id} :detach-libraries output) (t/testing "the artifact is a penpot file with its manifest" (t/is (fs/exists? output)) @@ -150,13 +180,13 @@ job-id (th/create-job* :import-binfile {:profile-id (:id profile) :resource-id (:resource-id staged)}) context (job-context job-id) - result (bfj/import-files (job-cfg) context - {:profile-id (:id profile) - :project-id (:default-project-id profile) - :team team - :name "imported" - :input (js/load-input th/*system* context) - :version 3})] + result (run-import! (job-cfg) context + {:profile-id (:id profile) + :project-id (:default-project-id profile) + :team team + :name "imported" + :input (js/load-input th/*system* context) + :version 3})] (t/testing "the package is imported in the destination project" (t/is (= 1 (count (:file-ids result))))) @@ -175,10 +205,9 @@ (with-redefs [jobs/heartbeat (fn [_cfg & {:as options}] (swap! reports conj (:progress options)) 1)] - (bfj/export-files (job-cfg) (job-context job-id) - {:ids #{file-id} - :export-type :detach-libraries - :output (tmp/tempfile* :suffix ".penpot")})) + (run-export! (job-cfg) (job-context job-id) + #{file-id} :detach-libraries + (tmp/tempfile* :suffix ".penpot"))) (t/testing "the file opens the run with its position" (let [event (first @reports)] @@ -207,10 +236,9 @@ ;; the taps themselves, before the adapter turns them into events (with-redefs [events/tap (fn [_type data] (swap! taps conj data))] - (bfj/export-files (job-cfg) (job-context job-id) - {:ids #{file-id} - :export-type :detach-libraries - :output (tmp/tempfile* :suffix ".penpot")})) + (run-export! (job-cfg) (job-context job-id) + #{file-id} :detach-libraries + (tmp/tempfile* :suffix ".penpot"))) (t/testing "the tap that opens the file carries the outer counters" (let [tap (first (filter #(= :file (:section %)) @taps))] @@ -269,13 +297,13 @@ (with-redefs [jobs/heartbeat (fn [_cfg & {:as options}] (swap! reports conj (:progress options)) 1)] - (bfj/import-files (job-cfg) context - {:profile-id (:id profile) - :project-id (:default-project-id profile) - :team team - :name "imported" - :input (js/load-input th/*system* context) - :version 3})) + (run-import! (job-cfg) context + {:profile-id (:id profile) + :project-id (:default-project-id profile) + :team team + :name "imported" + :input (js/load-input th/*system* context) + :version 3})) (t/testing "the run opens with the manifest, which has no units to count" (let [event (first @reports)] @@ -301,8 +329,8 @@ job-id (th/create-job* :export-binfile {:profile-id (:id profile)}) context (job-context job-id)] - (t/testing "a running job is active" - (t/is (nil? (bfj/check-active th/*system* context)))) + (t/testing "a running job is active, and answers with the cfg to thread it" + (t/is (identical? th/*system* (bfj/check-active th/*system* context)))) (t/testing "a cancelled job is not" (th/db-update! :job {:status "cancelled"} {:id job-id}) @@ -314,10 +342,8 @@ (t/testing "so the export stops at its next progress point" (let [output (tmp/tempfile* :suffix ".penpot") cause (try - (bfj/export-files (job-cfg) context - {:ids #{file-id} - :export-type :detach-libraries - :output output}) + (run-export! (job-cfg) context + #{file-id} :detach-libraries output) nil (catch Throwable raised raised))] (t/is (= :interrupt (th/ex-type cause))) @@ -342,14 +368,14 @@ cause (ex-info "the job is no longer active" {:type :interrupt :code :job-interrupted})] (with-redefs [jobs/heartbeat (fn [& _] (throw cause))] - (let [raised (try (@#'bfj/with-progress th/*system* context - (fn [] - (swap! steps conj :before) - ;; the sink calls the beat - ;; inline, so its interrupt - ;; aborts the run here - (events/tap :progress {:section :file}) - (swap! steps conj :after))) + (let [raised (try (bfj/run th/*system* context + (fn [_cfg] + (swap! steps conj :before) + ;; the sink calls the beat + ;; inline, so its interrupt + ;; aborts the run here + (events/tap :progress {:section :file}) + (swap! steps conj :after))) nil (catch Throwable stopped stopped))] (t/is (= :interrupt (:type (ex-data raised)))) diff --git a/backend/test/backend_tests/binfile_svg_test.clj b/backend/test/backend_tests/binfile_svg_test.clj index 12cd353a8d..b954226726 100644 --- a/backend/test/backend_tests/binfile_svg_test.clj +++ b/backend/test/backend_tests/binfile_svg_test.clj @@ -246,10 +246,10 @@ (let [bundle (tmp/tempfile :prefix "penpot-export-" :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) + (assoc ::bfc/file-ids #{(:id file)}) (assoc ::bfc/embed-assets false) - (assoc ::bfc/include-libraries false)) - (io/output-stream bundle)) + (assoc ::bfc/include-libraries false) + (assoc ::bfc/output (io/output-stream bundle)))) (let [evil (tmp/tempfile :prefix "penpot-evil-" :suffix ".zip") _ (tamper-svg-entry! bundle evil media-id (utf8bytes evil-text) :content-type content-type) @@ -364,7 +364,7 @@ (let [bundle (tmp/tempfile :prefix "penpot-export-v1-" :suffix ".bin")] (v1/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) + (assoc ::bfc/file-ids #{(:id file)}) (assoc ::bfc/embed-assets false) (assoc ::bfc/include-libraries false)) (io/output-stream bundle)) diff --git a/backend/test/backend_tests/binfile_test.clj b/backend/test/backend_tests/binfile_test.clj index 2043ad6a19..960bcd483a 100644 --- a/backend/test/backend_tests/binfile_test.clj +++ b/backend/test/backend_tests/binfile_test.clj @@ -284,9 +284,9 @@ (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :detach-libraries)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :detach-libraries) + (assoc ::bfc/output (io/output-stream output)))) (let [result (-> th/*system* (assoc ::bfc/project-id (:default-project-id profile)) @@ -373,9 +373,9 @@ (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :detach-libraries)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :detach-libraries) + (assoc ::bfc/output (io/output-stream output)))) (let [result (-> th/*system* (assoc ::bfc/project-id (:default-project-id profile)) @@ -407,9 +407,9 @@ (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :detach-libraries)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :detach-libraries) + (assoc ::bfc/output (io/output-stream output)))) (let [result (-> th/*system* (assoc ::bfc/project-id (:default-project-id profile)) @@ -473,9 +473,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Read the manifest and check external-libraries (let [manifest (v3/get-manifest th/*system* output)] @@ -801,9 +801,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Remove the source library to simulate a cross-environment import ;; where the original library does not exist in the target team. @@ -865,9 +865,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :detach-libraries)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :detach-libraries) + (assoc ::bfc/output (io/output-stream output)))) ;; Remove the source library to simulate a cross-environment import ;; where no matching library exists in the target team. @@ -907,9 +907,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Remove the source library to simulate a cross-environment import ;; where the original library does not exist in the target team. @@ -985,9 +985,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Remove the source library and recreate a matching one owned by owner (db/update! th/*system* :file @@ -1067,9 +1067,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file1) (:id file2)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file1) (:id file2)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Remove the source library and recreate a matching one (db/update! th/*system* :file @@ -1168,9 +1168,9 @@ ;; Export file with link-later (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{file-id}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{file-id}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Verify manifest has external-libraries (let [manifest (v3/get-manifest th/*system* output) @@ -1205,9 +1205,9 @@ ;; Export file with link-later (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{file-id}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{file-id}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; First import (let [result1 (-> th/*system* @@ -1238,9 +1238,9 @@ ;; Export file with link-later (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{file-id}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{file-id}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Import with overwrite (file-id set) (let [result (-> th/*system* @@ -1267,9 +1267,9 @@ ;; Export file with link-later from team 1 (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{file-id}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{file-id}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Import in team 2 (where library with same name exists) (let [result (-> th/*system* @@ -1300,9 +1300,9 @@ ;; Export file with link-later (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{file-id}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{file-id}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Import in team 2 (no library exists) (let [result (-> th/*system* @@ -1325,9 +1325,9 @@ ;; Export file with link-later (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{file-id}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{file-id}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Import in team 2 (library with different name) (let [result (-> th/*system* @@ -1356,9 +1356,9 @@ ;; Export file with link-later (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{file-id}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{file-id}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Import in team 2 (library exists but not shared) (let [result (-> th/*system* @@ -1381,9 +1381,9 @@ ;; Export file with link-later (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{file-id}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{file-id}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Delete the library in team 2 (db/update! th/*system* :file @@ -1433,9 +1433,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Delete original libraries (db/update! th/*system* :file {:deleted-at (ct/now)} {:id (:id lib1)}) @@ -1494,9 +1494,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Delete both libraries (db/update! th/*system* :file {:deleted-at (ct/now)} {:id (:id lib1)}) @@ -1546,9 +1546,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Delete original libraries (db/update! th/*system* :file {:deleted-at (ct/now)} {:id (:id lib1)}) @@ -1614,9 +1614,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Delete original libraries (db/update! th/*system* :file {:deleted-at (ct/now)} {:id (:id lib1)}) @@ -1680,9 +1680,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file1) (:id file2)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file1) (:id file2)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Delete original library and recreate (db/update! th/*system* :file {:deleted-at (ct/now)} {:id (:id lib)}) @@ -1726,9 +1726,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file1) (:id file2)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file1) (:id file2)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Delete original library and recreate (db/update! th/*system* :file {:deleted-at (ct/now)} {:id (:id lib)}) @@ -1778,9 +1778,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file1) (:id file2)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file1) (:id file2)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Delete original libraries and recreate (db/update! th/*system* :file {:deleted-at (ct/now)} {:id (:id lib-icons)}) @@ -1837,9 +1837,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Import as viewer (no edit permission on library) (let [result (-> th/*system* @@ -1879,9 +1879,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Import as editor (has edit permission) (let [result (-> th/*system* @@ -1919,9 +1919,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Verify slug in manifest (let [manifest (v3/get-manifest th/*system* output) @@ -1946,9 +1946,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Library with empty slug should be dropped from external-libraries (let [manifest (v3/get-manifest th/*system* output) @@ -1966,9 +1966,9 @@ ;; Export file with no libraries (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Manifest should have no external-libraries (let [manifest (v3/get-manifest th/*system* output)] @@ -2000,9 +2000,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Delete original and create with different case (db/update! th/*system* :file {:deleted-at (ct/now)} {:id (:id lib)}) @@ -2037,9 +2037,9 @@ ;; Export file with link-later (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{file-id}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{file-id}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Re-import in same team (let [result (-> th/*system* @@ -2075,9 +2075,9 @@ ;; Export file with link-later (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{file-id}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{file-id}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Import in team 2 (no library exists) (let [result (-> th/*system* @@ -2127,9 +2127,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Delete and recreate library (db/update! th/*system* :file {:deleted-at (ct/now)} {:id (:id lib)}) @@ -2182,9 +2182,9 @@ (let [output (tmp/tempfile :suffix ".zip")] (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Delete and create two libraries with same name (db/update! th/*system* :file {:deleted-at (ct/now)} {:id (:id lib)}) @@ -2240,9 +2240,9 @@ ;; Export file with link-later (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{file-id}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{file-id}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Re-import in same team (let [result (-> th/*system* @@ -2292,9 +2292,9 @@ ;; Export file with link-later (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{file-id}) - (assoc ::bfc/export-type :link-later)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{file-id}) + (assoc ::bfc/export-type :link-later) + (assoc ::bfc/output (io/output-stream output)))) ;; Create a viewer profile (no edit permission on the library) (let [viewer (th/create-profile* th/*system* 2 {}) @@ -2342,10 +2342,10 @@ ;; type should win (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{file-id}) + (assoc ::bfc/file-ids #{file-id}) (assoc ::bfc/export-type :link-later) - (assoc ::bfc/include-libraries true)) - (io/output-stream output)) + (assoc ::bfc/include-libraries true) + (assoc ::bfc/output (io/output-stream output)))) ;; Verify manifest has external-libraries (only produced by link-later) (let [manifest (v3/get-manifest th/*system* output) @@ -2362,9 +2362,9 @@ (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :detach-libraries)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :detach-libraries) + (assoc ::bfc/output (io/output-stream output)))) ;; Import with max-zip-entries=1 — the exported ZIP has more entries (let [cfg (-> th/*system* @@ -2419,9 +2419,9 @@ (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) - (assoc ::bfc/export-type :detach-libraries)) - (io/output-stream output)) + (assoc ::bfc/file-ids #{(:id file)}) + (assoc ::bfc/export-type :detach-libraries) + (assoc ::bfc/output (io/output-stream output)))) ;; Import with max-binary-entry-size=1 — the media object will exceed this (let [cfg (-> th/*system* @@ -2504,10 +2504,10 @@ (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) + (assoc ::bfc/file-ids #{(:id file)}) (assoc ::bfc/embed-assets false) - (assoc ::bfc/include-libraries false)) - (io/output-stream exported)) + (assoc ::bfc/include-libraries false) + (assoc ::bfc/output (io/output-stream exported)))) (let [bombed (tmp/tempfile :suffix ".zip")] (replace-zip-entry! exported bombed @@ -2550,10 +2550,10 @@ (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) + (assoc ::bfc/file-ids #{(:id file)}) (assoc ::bfc/embed-assets false) - (assoc ::bfc/include-libraries false)) - (io/output-stream output)) + (assoc ::bfc/include-libraries false) + (assoc ::bfc/output (io/output-stream output)))) (let [cfg (-> th/*system* (assoc ::bfc/project-id (:default-project-id profile)) @@ -2597,10 +2597,10 @@ (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) + (assoc ::bfc/file-ids #{(:id file)}) (assoc ::bfc/embed-assets false) - (assoc ::bfc/include-libraries false)) - (io/output-stream exported)) + (assoc ::bfc/include-libraries false) + (assoc ::bfc/output (io/output-stream exported)))) (let [sizes (text-entries-sizes exported) max-single (apply max 0 sizes) @@ -2795,10 +2795,10 @@ (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id simple) (:id svg)}) + (assoc ::bfc/file-ids #{(:id simple) (:id svg)}) (assoc ::bfc/embed-assets false) - (assoc ::bfc/include-libraries false)) - (io/output-stream output)) + (assoc ::bfc/include-libraries false) + (assoc ::bfc/output (io/output-stream output)))) ;; import returns the imported file ids plus the library link ;; resolution; the test only needs the ids @@ -2874,10 +2874,10 @@ (v3/export-files! (-> th/*system* - (assoc ::bfc/ids #{(:id file)}) + (assoc ::bfc/file-ids #{(:id file)}) (assoc ::bfc/embed-assets false) - (assoc ::bfc/include-libraries false)) - (io/output-stream output)) + (assoc ::bfc/include-libraries false) + (assoc ::bfc/output (io/output-stream output)))) ;; import returns the imported file ids plus the library link ;; resolution; the test only needs the ids diff --git a/backend/test/backend_tests/tasks/binfile_jobs_integration_test.clj b/backend/test/backend_tests/tasks/binfile_jobs_integration_test.clj index 4d6277a0ff..07a959aa1a 100644 --- a/backend/test/backend_tests/tasks/binfile_jobs_integration_test.clj +++ b/backend/test/backend_tests/tasks/binfile_jobs_integration_test.clj @@ -249,11 +249,13 @@ (with-redefs [jobs/heartbeat (fn [_cfg & {:as options}] (swap! calls conj options) 1)] - (bfj/export-files th/*system* - (jobs/make-context (jobs/get-job th/*system* job-id)) - {:ids #{file-id} - :export-type :detach-libraries - :output (tmp/tempfile* :suffix ".penpot")})) + (let [cfg (bfc/export-cfg th/*system* + {:file-ids #{file-id} + :export-type :detach-libraries + :output (tmp/tempfile* :suffix ".penpot")}) + context (jobs/make-context (jobs/get-job th/*system* job-id))] + (bfj/check-active cfg context) + (bfj/run cfg context v3/export-files!))) (let [reports (map :progress @calls)] diff --git a/backend/test/backend_tests/tasks/import_binfile_test.clj b/backend/test/backend_tests/tasks/import_binfile_test.clj index eadc44104f..e82d462df3 100644 --- a/backend/test/backend_tests/tasks/import_binfile_test.clj +++ b/backend/test/backend_tests/tasks/import_binfile_test.clj @@ -9,7 +9,6 @@ it leaves behind when it fails." (:require [app.binfile.common :as bfc] - [app.binfile.jobs :as bfj] [app.binfile.v1 :as v1] [app.binfile.v3 :as v3] [app.common.uuid :as uuid] @@ -50,7 +49,7 @@ [file-id] (let [output (tmp/tempfile* :suffix ".penpot")] (v1/export-files! (-> th/*system* - (assoc ::bfc/ids #{file-id}) + (assoc ::bfc/file-ids #{file-id}) (assoc ::bfc/include-libraries false) (assoc ::bfc/embed-assets false)) output) @@ -249,12 +248,12 @@ ;; the core does its work inside the transaction and answers with a result ;; the schema of the task does not accept: the check runs before the ;; transaction commits, so the work is rolled back with it - (with-redefs [bfj/import-files - (fn [tx-cfg _context _params] - (th/create-file* tx-cfg 7 {:profile-id (:id profile) - :project-id project-id - :is-shared false - :name "half imported"}) + (with-redefs [v3/import-files! + (fn [cfg] + (th/create-file* cfg 7 {:profile-id (:id profile) + :project-id project-id + :is-shared false + :name "half imported"}) {:file-ids [(uuid/next)] :resolution "not-a-map"})]