diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 87d0b31f29..95f2818c25 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -591,6 +591,7 @@ :offload-file-data (ig/ref :app.tasks.offload-file-data/job-def) :objects-gc (ig/ref :app.tasks.objects-gc/job-def) :export-binfile (ig/ref :app.tasks.export-binfile/job-def) + :export-assets (ig/ref :app.tasks.export-assets/job-def) :import-binfile (ig/ref :app.tasks.import-binfile/job-def) :storage-gc-deleted (ig/ref ::sto.gc-deleted/job-def) :storage-gc-touched (ig/ref ::sto.gc-touched/job-def) @@ -657,6 +658,11 @@ ::mtx/metrics (ig/ref ::mtx/metrics) ::mbus/msgbus (ig/ref ::mbus/msgbus)} + ;; the export of shapes/frames: the work runs on the external exporter + ;; worker, so this def carries no handler work and no components + :app.tasks.export-assets/job-def + {} + :app.storage.gc-deleted/job-def {::db/pool (ig/ref ::db/pool) ::sto/storage (ig/ref ::sto/storage) diff --git a/backend/src/app/rpc/commands/jobs.clj b/backend/src/app/rpc/commands/jobs.clj index d1aa5b85cb..b2eb2e653c 100644 --- a/backend/src/app/rpc/commands/jobs.clj +++ b/backend/src/app/rpc/commands/jobs.clj @@ -32,6 +32,7 @@ [app.rpc.doc :as-alias doc] [app.rpc.quotes :as quotes] [app.storage :as sto] + [app.tasks.export-assets :as export-assets] [app.tasks.export-binfile :as export-binfile] [app.tasks.import-binfile :as import-binfile] [app.util.services :as sv] @@ -44,6 +45,13 @@ [:map {:title "create-export-binfile-job" :closed true} [:params export-binfile/schema:params]]) +(def ^:private schema:create-export-assets-job + "The shapes and frames frozen in the params, rendered as image files + by the external exporter worker; the business params are what the + `:export-assets` job-def declares." + [:map {:title "create-export-assets-job" :closed true} + [:params export-assets/schema:params]]) + (def ^:private schema:create-import-binfile-job "The `.penpot` package the caller uploaded, imported into a project by the backend runner; the business params are what the `:import-binfile` @@ -110,6 +118,18 @@ ;; EXPORT ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(defn- check-file-permissions + "The files the job will read, checked before anything is stored: a + refusal arrives synchronously. The handler checks it again before + reading, because a permission can be revoked while the job waits." + [cfg profile-id file-ids] + (when (empty? file-ids) + (ex/raise :type :validation + :code :no-files-to-export + :hint "expected at least one file to export")) + (doseq [file-id file-ids] + (files/check-read-permissions! cfg profile-id file-id))) + (sv/defmethod ::create-export-binfile-job "Create a durable job that exports a set of files as a `.penpot` package. @@ -134,13 +154,7 @@ params (jobs/validate-params job-def (:params envelope)) file-ids (:file-ids params)] - (when (empty? file-ids) - (ex/raise :type :validation - :code :no-files-to-export - :hint "expected at least one file to export")) - - (doseq [file-id file-ids] - (files/check-read-permissions! cfg profile-id file-id)) + (check-file-permissions cfg profile-id file-ids) (quotes/check! cfg {::quotes/id ::quotes/export-jobs-per-profile ::quotes/profile-id profile-id}) @@ -152,6 +166,40 @@ :files (count file-ids) :export-type (d/name (:export-type params))}})))) +(sv/defmethod ::create-export-assets-job + "Create a durable job that renders shapes and frames as image files. + + Its params freeze the items to render (file, page, object, type, + scale, name and the optional share id) the way the exporter receives + them. The work runs on the external exporter worker, which consumes + the queue the job-def names; the backend ships no runner for it. + + The job is created, not run: the caller follows it by its id. The + read permission of every file named in the items is checked here so a + refusal arrives synchronously, and the handler checks it again before + reading." + {::doc/added "2.20" + ::webhooks/event? true + ::sm/params schema:create-export-assets-job + ::sm/result schema:job-summary} + [cfg {:keys [::rpc/profile-id] :as envelope}] + (let [job-def (resolve-job-def cfg :export :export-assets) + ;; the RPC layer decoded the params against the job-def schema + ;; before the command ran + params (jobs/validate-params job-def (:params envelope))] + + (check-file-permissions cfg profile-id + (into #{} (map :file-id) (:exports params))) + + (quotes/check! cfg {::quotes/id ::quotes/export-jobs-per-profile + ::quotes/profile-id profile-id}) + + (let [summary (get-job-summary cfg (submit-job cfg job-def :export-assets + params profile-id))] + (with-meta summary + {::audit/props {:job-id (:id summary) + :files (count (:exports params))}})))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; IMPORT ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/backend/src/app/tasks/export_assets.clj b/backend/src/app/tasks/export_assets.clj new file mode 100644 index 0000000000..b6538b5289 --- /dev/null +++ b/backend/src/app/tasks/export_assets.clj @@ -0,0 +1,79 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.tasks.export-assets + "The `:export-assets` job: the shapes and frames a caller froze at + creation time, rendered as image files. + + This is an EXTERNAL job: the work runs on the exporter worker, a + process outside the JVM, so the job is created with `::jobs/queue-name + :exporter` (read by the creation command; the substrate never looks at + it) and the backend ships no runner for that queue. The handler here + is a guard, not a worker: it is never executed under a correct + configuration, because the only runner that could claim the job is + external, and `claim-job` does not look at the queue. Should a wrong + configuration ever route the job to a JVM runner, it fails visibly + here instead of \"completing\" empty. + + Progress vocabulary of the external worker (the substrate validates + the shape, the worker owns the words): stages `:preparing`, + `:rendering` and `:packaging`, with counters `objects` (one per + exported shape) and `pages` (one per exported frame)." + (:require + [app.common.exceptions :as ex] + [app.common.schema :as sm] + [app.jobs :as jobs] + [integrant.core :as ig])) + +(def schema:export-item + "One thing to render: where it lives (file, page, object), what to + produce of it (type, scale) and what to call it. The same item shape + the legacy `/api/export` surface received, frozen in the params of the + job; the optional `share-id` marks the item as coming from a public + share." + [:map {:title "export-item" :closed true} + [:file-id ::sm/uuid] + [:page-id ::sm/uuid] + [:object-id ::sm/uuid] + [:type [::sm/one-of #{:png :jpeg :webp :svg :pdf}]] + [:name ::sm/text] + [:suffix {:optional true} ::sm/text] + [:scale ::sm/number] + [:share-id {:optional true} ::sm/uuid]]) + +(def schema:params + "Business params of an asset export: the items to render, frozen when + the job was created, plus the options that shape the output. There is + no `wait` here: a job is followed by its id, not waited on." + [:map {:title "export-assets-params" :closed true} + [:exports [:vector {:min 1} schema:export-item]] + [:name {:optional true} ::sm/text] + [:skip-children {:optional true} ::sm/boolean] + [:force-multiple {:optional true} ::sm/boolean] + [:is-wasm {:optional true} ::sm/boolean]]) + +(defn- execute-export-assets + "Plain handler, importable and testable without integrant. + + It only raises: the work of this job belongs to the external worker. + See the namespace docstring." + [_context _params] + (ex/raise :type :internal + :code :no-handler-for-external + :hint "this job runs on the external exporter worker")) + +(defmethod ig/init-key ::job-def + [_ _cfg] + {::jobs/name :export-assets + ::jobs/family :export + ::jobs/resource-role :output + ;; the queue the creation command routes the job to: an external + ;; worker consumes it, the backend ships no runner for it + ::jobs/queue-name :exporter + ::jobs/schema schema:params + ::jobs/handler execute-export-assets + ::jobs/decoder (sm/decoder schema:params sm/json-transformer) + ::jobs/validator (sm/validator schema:params)}) diff --git a/backend/test/backend_tests/rpc/commands/jobs_test.clj b/backend/test/backend_tests/rpc/commands/jobs_test.clj index a03b5a9ca4..242dfd8ace 100644 --- a/backend/test/backend_tests/rpc/commands/jobs_test.clj +++ b/backend/test/backend_tests/rpc/commands/jobs_test.clj @@ -179,6 +179,113 @@ (t/is (= "detach-libraries" (:export-type props))) (t/is (= #{:job-id :files :export-type} (set (keys props))))))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; ASSETS EXPORT +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- export-item + [file-id & {:keys [page-id object-id type name scale suffix share-id]}] + (cond-> {:file-id file-id + :page-id (or page-id (uuid/next)) + :object-id (or object-id (uuid/next)) + :type (or type :png) + :name (or name "shape") + :scale (or scale 1)} + suffix (assoc :suffix suffix) + share-id (assoc :share-id share-id))) + +(defn- create-export-assets-job + [profile-id exports & {:as extra}] + (th/command! (merge {::th/type :create-export-assets-job + ::rpc/profile-id profile-id + :params {:exports exports}} + extra))) + +(t/deftest create-export-assets-job-answers-with-a-pending-job + (let [profile (th/create-profile* 1) + file-id (first (:file-ids (import-fixture! profile))) + item (export-item file-id) + before (count-storage-objects) + out (create-export-assets-job (:id profile) [item])] + + (t/is (th/success? out)) + + (let [result (:result out) + job (jobs/get-job th/*system* (:id result))] + + (t/testing "the answer is enough to follow the job" + (t/is (uuid? (:id result))) + (t/is (= "pending" (:status result))) + (t/is (= "export-assets" (:name result))) + (t/is (ct/inst? (:created-at result)))) + + (t/testing "and the row routes to the external worker queue" + (let [job-def (jobs/get-job-def (::jobs/defs th/*system*) :export-assets) + params (jobs/decode-params job-def (:params job))] + (t/is (= "exporter" (:queue job))) + (t/is (= 0 (:max-retries job))) + (t/is (= (:id profile) (:profile-id job))) + (t/is (= 1 (count (:exports params)))) + (t/is (= :png (:type (first (:exports params))))))) + + (t/testing "creating the job stores no artifact" + (t/is (= before (count-storage-objects))))))) + +(t/deftest create-export-assets-job-checks-the-read-permission + (let [owner (th/create-profile* 1) + other (th/create-profile* 2) + file-id (first (:file-ids (import-fixture! owner))) + out (create-export-assets-job (:id other) [(export-item file-id)])] + + (t/testing "a file the caller cannot read is not exported" + (t/is (not (th/success? out))) + (t/is (= :not-found (th/ex-type (:error out))))) + + (t/testing "and no job was created for it" + (t/is (zero? (count-jobs)))))) + +(t/deftest create-export-assets-job-rejects-an-unknown-type + (let [profile (th/create-profile* 1) + file-id (first (:file-ids (import-fixture! profile))) + out (create-export-assets-job (:id profile) + [(export-item file-id :type :psd)])] + + (t/is (not (th/success? out))) + (t/is (= :data-validation (th/ex-code (:error out)))) + (t/testing "and nothing was created" + (t/is (zero? (count-jobs)))))) + +(t/deftest create-export-assets-job-adds-the-job-id-to-the-audit-props + (let [profile (th/create-profile* 1) + file-id (first (:file-ids (import-fixture! profile))) + out (create-export-assets-job (:id profile) + [(export-item file-id)]) + props (::audit/props (meta (:result out)))] + (t/is (th/success? out)) + (t/testing "the audit of the call points at the job and its items" + (t/is (= (:id (:result out)) (:job-id props))) + (t/is (= 1 (:files props))) + (t/is (= #{:job-id :files} (set (keys props))))))) + +(t/deftest create-export-assets-job-counts-in-the-export-quote + ;; the quote of the family counts every job of it, so a mixed backlog + ;; of binfile and assets jobs fills the same quote + (with-mocks [mock {:target 'app.config/get + :return (th/config-get-mock + {:quotes-export-jobs-per-profile 1})}] + (let [profile (th/create-profile* 1) + file-id (first (:file-ids (import-fixture! profile)))] + + (t/is (th/success? (create-export-assets-job (:id profile) + [(export-item file-id)]))) + + (t/testing "the next export job of the family is refused, whatever its type" + (let [out (create-export-binfile-job (:id profile) #{file-id})] + (t/is (not (th/success? out))) + (t/is (= :max-quote-reached (th/ex-code (:error out)))))) + + (t/is (= 1 (count-jobs)))))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; IMPORT ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;