diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index b9aef15d04..1fde3aa13c 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -72,6 +72,7 @@ :telemetry-uri "https://telemetry.penpot.app/" :media-max-file-size (* 1024 1024 30) ; 30MiB + :font-max-file-size (* 1024 1024 30) ; 30MiB :ldap-user-query "(|(uid=:username)(mail=:username))" :ldap-attrs-username "uid" @@ -120,6 +121,7 @@ [:auto-file-snapshot-timeout {:optional true} ::ct/duration] [:media-max-file-size {:optional true} ::sm/int] + [:font-max-file-size {:optional true} ::sm/int] [:deletion-delay {:optional true} ::ct/duration] [:file-clean-delay {:optional true} ::ct/duration] [:telemetry-enabled {:optional true} ::sm/boolean] diff --git a/backend/src/app/media.clj b/backend/src/app/media.clj index 58151a800c..ff13c3572c 100644 --- a/backend/src/app/media.clj +++ b/backend/src/app/media.clj @@ -38,9 +38,6 @@ org.im4java.core.ConvertCmd org.im4java.core.IMOperation)) -(def default-max-file-size - (* 1024 1024 10)) ; 10 MiB - (def schema:upload [:map {:title "Upload"} [:filename :string] @@ -79,6 +76,20 @@ max-size))) upload)) +(defn validate-font-size! + "Validates that the font file `upload` does not exceed the configured + `:font-max-file-size` limit. Accepts the same map shape as + `validate-media-size!` — requires a `:size` key in bytes." + [upload] + (let [max-size (cf/get :font-max-file-size)] + (when (> (:size upload) max-size) + (ex/raise :type :restriction + :code :font-max-file-size-reached + :hint (str/ffmt "the uploaded font size % is greater than the maximum %" + (:size upload) + max-size))) + upload)) + (defmulti process :cmd) (defmulti process-error class) @@ -296,9 +307,7 @@ [{:keys [::http/client]} uri] (letfn [(parse-and-validate [{:keys [status headers] :as response}] (let [size (some-> (get headers "content-length") d/parse-integer) - mtype (get headers "content-type") - format (cm/mtype->format mtype) - max-size (cf/get :media-max-file-size default-max-file-size)] + mtype (get headers "content-type")] (when-not (<= 200 status 299) (ex/raise :type :validation @@ -310,19 +319,9 @@ :code :unknown-size :hint "seems like the url points to resource with unknown size")) - (when (> size max-size) - (ex/raise :type :validation - :code :file-too-large - :hint (str/ffmt "the file size % is greater than the maximum %" - size - default-max-file-size))) - - (when (nil? format) - (ex/raise :type :validation - :code :media-type-not-allowed - :hint "seems like the url points to an invalid media object")) - - {:size size :mtype mtype :format format}))] + (-> {:size size :mtype mtype} + (validate-media-type!) + (validate-media-size!))))] (let [{:keys [body] :as response} (try diff --git a/backend/src/app/rpc/commands/fonts.clj b/backend/src/app/rpc/commands/fonts.clj index 03c66a968f..6d86efd798 100644 --- a/backend/src/app/rpc/commands/fonts.clj +++ b/backend/src/app/rpc/commands/fonts.clj @@ -9,6 +9,8 @@ [app.binfile.common :as bfc] [app.common.data.macros :as dm] [app.common.exceptions :as ex] + [app.common.logging :as l] + [app.common.media :as cm] [app.common.schema :as sm] [app.common.time :as ct] [app.common.uuid :as uuid] @@ -21,6 +23,7 @@ [app.rpc :as-alias rpc] [app.rpc.climit :as-alias climit] [app.rpc.commands.files :as files] + [app.rpc.commands.media :as cmedia] [app.rpc.commands.projects :as projects] [app.rpc.commands.teams :as teams] [app.rpc.doc :as-alias doc] @@ -29,6 +32,8 @@ [app.storage :as sto] [app.storage.tmp :as tmp] [app.util.services :as sv] + [cuerdas.core :as str] + [datoteka.fs :as fs] [datoteka.io :as io]) (:import java.io.InputStream @@ -87,32 +92,92 @@ (declare create-font-variant) (def ^:private schema:create-font-variant - [:map {:title "create-font-variant"} - [:team-id ::sm/uuid] - [:data [:map-of ::sm/text [:or ::sm/bytes - [::sm/vec ::sm/bytes]]]] - [:font-id ::sm/uuid] - [:font-family ::sm/text] - [:font-weight [::sm/one-of {:format "number"} valid-weight]] - [:font-style [::sm/one-of {:format "string"} valid-style]]]) + [:and + [:map {:title "create-font-variant"} + [:team-id ::sm/uuid] + [:font-id ::sm/uuid] + [:font-family ::sm/text] + [:font-weight [::sm/one-of {:format "number"} valid-weight]] + [:font-style [::sm/one-of {:format "string"} valid-style]] + [:data {:optional true} [:map-of ::sm/text [:or ::sm/bytes [::sm/vec ::sm/bytes]]]] + [:uploads {:optional true} [:map-of ::sm/text ::sm/uuid]]] + [:fn {:error/message "one of :data or :uploads is required"} + (fn [{:keys [data uploads]}] + (or (seq data) (seq uploads)))]]) ;; FIXME: IMPORTANT: refactor this, we should not hold a whole db ;; connection around the font creation +(defn- prepare-font-data-from-uploads + "Assembles each chunked-upload session in `uploads` (a `{mtype → + session-id}` map) into a temp file, validates the media type and + size of every entry, and returns a `{mtype → path}` data map." + [cfg {:keys [uploads] :as params}] + (let [data (reduce-kv + (fn [acc mtype session-id] + (let [assembled (cmedia/assemble-chunks cfg session-id)] + (-> {:mtype mtype :size (:size assembled)} + (media/validate-media-type! cm/font-types) + (media/validate-font-size!)) + (assoc acc mtype (:path assembled)))) + {} + uploads)] + + (-> params + (assoc :data data) + (dissoc :uploads)))) + +(defn- prepare-font-data-from-legacy + "Validates the media type and size of every entry in the legacy + `:data` map (a `{mtype → bytes | [bytes]}` map). Normalises every + entry to a tempfile. Returns params with a normalised + `{mtype → path}` data map." + [{:keys [data] :as params}] + (let [data (reduce-kv + (fn [acc mtype content] + (let [tmp (tmp/tempfile :prefix "penpot.tempfont." :suffix "") + chunks (if (vector? content) content [content]) + streams (map io/input-stream chunks) + streams (Collections/enumeration streams)] + + ;; Generate the tempfile from all chunks + (with-open [^OutputStream output (io/output-stream tmp) + ^InputStream input (SequenceInputStream. streams)] + (io/copy input output)) + + ;; Validate + (-> {:mtype mtype :size (fs/size tmp)} + (media/validate-media-type! cm/font-types) + (media/validate-font-size!)) + + (assoc acc mtype tmp))) + {} + data)] + (assoc params :data data))) + (sv/defmethod ::create-font-variant + "Upload a font variant. Font data may be provided either as a + Transit-encoded `:data` map (keyed by mime-type) for small fonts, or + as an `:uploads` map (keyed by mime-type, values are upload-session + UUIDs from the chunked-upload API) for large fonts. Exactly one of + the two must be present." {::doc/added "1.18" + ::doc/changes ["2.16" "Add :uploads param for chunked upload support"] ::climit/id [[:process-font/by-profile ::rpc/profile-id] [:process-font/global]] ::webhooks/event? true ::sm/params schema:create-font-variant} - [cfg {:keys [::rpc/profile-id team-id] :as params}] + [cfg {:keys [::rpc/profile-id team-id uploads] :as params}] (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}] (teams/check-edition-permissions! conn profile-id team-id) (quotes/check! cfg {::quotes/id ::quotes/font-variants-per-team ::quotes/profile-id profile-id ::quotes/team-id team-id}) - (create-font-variant cfg (assoc params :profile-id profile-id))))) + (let [params (if (some? uploads) + (prepare-font-data-from-uploads cfg params) + (prepare-font-data-from-legacy params))] + (create-font-variant cfg (assoc params :profile-id profile-id)))))) (defn create-font-variant [{:keys [::sto/storage ::db/conn]} {:keys [data] :as params}] @@ -127,23 +192,6 @@ :hint "invalid font upload, unable to generate missing font assets")) data)) - (process-chunks [chunks] - (let [tmp (tmp/tempfile :prefix "penpot.tempfont." :suffix "") - streams (map io/input-stream chunks) - streams (Collections/enumeration streams)] - (with-open [^OutputStream output (io/output-stream tmp) - ^InputStream input (SequenceInputStream. streams)] - (io/copy input output)) - tmp)) - - (join-chunks [data] - (reduce-kv (fn [data mtype content] - (if (vector? content) - (assoc data mtype (process-chunks content)) - data)) - data - data)) - (prepare-font [data mtype] (when-let [resource (get data mtype)] @@ -185,11 +233,38 @@ :otf-file-id (:id otf) :ttf-file-id (:id ttf)}))] - (let [data (join-chunks data) - data (generate-missing data) - assets (persist-fonts-files! data) - result (insert-font-variant! assets)] - (vary-meta result assoc ::audit/replace-props (update params :data (comp vec keys)))))) + (let [tpoint (ct/tpoint) + mtypes (vec (keys data)) + total-size (reduce-kv (fn [acc _ content] + (+ acc (if (bytes? content) + (alength ^bytes content) + (fs/size content)))) + 0 + data)] + + (l/dbg :hint "create-font-variant" + :step "init" + :font-family (:font-family params) + :font-weight (:font-weight params) + :font-style (:font-style params) + :mtypes (str/join mtypes ",") + :size total-size) + + (let [data (generate-missing data) + assets (persist-fonts-files! data) + result (insert-font-variant! assets) + elapsed (tpoint)] + + (l/dbg :hint "create-font-variant" + :step "end" + :font-family (:font-family params) + :font-weight (:font-weight params) + :font-style (:font-style params) + :mtypes (str/join mtypes ",") + :size total-size + :elapsed (ct/format-duration elapsed)) + + (vary-meta result assoc ::audit/replace-props (update params :data (comp vec keys))))))) ;; --- UPDATE FONT FAMILY diff --git a/backend/src/app/rpc/commands/media.clj b/backend/src/app/rpc/commands/media.clj index 5bea17d379..405bdb8117 100644 --- a/backend/src/app/rpc/commands/media.clj +++ b/backend/src/app/rpc/commands/media.clj @@ -8,6 +8,7 @@ (:require [app.common.data :as d] [app.common.exceptions :as ex] + [app.common.logging :as l] [app.common.schema :as sm] [app.common.time :as ct] [app.common.uuid :as uuid] @@ -58,8 +59,8 @@ (db/run! cfg (fn [{:keys [::db/conn] :as cfg}] ;; We get the minimal file for proper checking if ;; file is not already deleted - (let [_ (files/get-minimal-file conn file-id) - mobj (create-file-media-object cfg params)] + (let [_ (files/get-minimal-file conn file-id) + mobj (create-file-media-object cfg params)] (db/update! conn :file {:modified-at (ct/now) @@ -149,20 +150,49 @@ (defn- create-file-media-object [{:keys [::sto/storage ::db/conn] :as cfg} - {:keys [id file-id is-local name content]}] - (let [result (process-image content) - image (sto/put-object! storage (::image result)) - thumb (when-let [params (::thumb result)] - (sto/put-object! storage params))] + {:keys [id file-id is-local name content from-url? from-chunks?]}] - (db/exec-one! conn [sql:create-file-media-object - (or id (uuid/next)) - file-id is-local name - (:id image) - (:id thumb) - (:width result) - (:height result) - (:mtype result)]))) + (let [tpoint (ct/tpoint) + id (or id (uuid/next)) + origin (cond + from-url? + "url" + from-chunks? + "chunks" + :else + "direct")] + + (l/dbg :hint "create file-media-object" + :step "init" + :id (str id) + :mtype (:mtype content) + :size (:size content) + :path (str (:path content)) + :origin origin) + + (let [result (process-image content) + image (sto/put-object! storage (::image result)) + thumb (when-let [params (::thumb result)] + (sto/put-object! storage params)) + elapsed (tpoint)] + + (l/dbg :hint "create file-media-object" + :step "end" + :id (str id) + :mtype (:mtype content) + :size (:size content) + :path (str (:path content)) + :origin origin + :elapsed (ct/format-duration elapsed)) + + (db/exec-one! conn [sql:create-file-media-object + id + file-id is-local name + (:id image) + (:id thumb) + (:width result) + (:height result) + (:mtype result)])))) ;; --- Create File Media Object (from URL) @@ -198,6 +228,7 @@ [cfg {:keys [url name] :as params}] (let [content (media/download-image cfg url) params (-> params + (assoc :from-url? true) (assoc :content content) (assoc :name (d/nilv name "unknown")))] @@ -305,7 +336,14 @@ :hint "chunk index is out of range for this session" :session-id session-id :total-chunks (:total-chunks session) - :index index))) + :index index)) + + + (l/trc :hint "upload-chunk" + :session-id session-id + :chunk (str index "/" (:total-chunks session)) + :size (:size content) + :path (:path content))) (let [storage (sto/resolve cfg) data (sto/content (:path content))] @@ -399,14 +437,15 @@ (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}] - (let [{:keys [path size]} (assemble-chunks cfg session-id) - content {:filename "upload" - :size size - :path path - :mtype mtype} - _ (media/validate-media-type! content) + (let [content (assemble-chunks cfg session-id) + content (-> content + (assoc :filename (str "upload:" name)) + (assoc :mtype mtype) + (media/validate-media-type!) + (media/validate-media-size!)) mobj (create-file-media-object cfg (assoc params - :id (or id (uuid/next)) + :id id + :from-chunks? true :content content))] (db/update! conn :file diff --git a/backend/src/app/rpc/commands/profile.clj b/backend/src/app/rpc/commands/profile.clj index efe99c4a70..a9e766bd9b 100644 --- a/backend/src/app/rpc/commands/profile.clj +++ b/backend/src/app/rpc/commands/profile.clj @@ -264,6 +264,7 @@ [cfg {:keys [::rpc/profile-id file] :as params}] ;; Validate incoming mime type (media/validate-media-type! file #{"image/jpeg" "image/png" "image/webp"}) + (media/validate-media-size! file) (update-profile-photo cfg (assoc params :profile-id profile-id))) (defn update-profile-photo diff --git a/backend/src/app/rpc/commands/teams.clj b/backend/src/app/rpc/commands/teams.clj index 79d2008be9..3ebd34fe00 100644 --- a/backend/src/app/rpc/commands/teams.clj +++ b/backend/src/app/rpc/commands/teams.clj @@ -827,6 +827,7 @@ ;; Validate incoming mime type (media/validate-media-type! file #{"image/jpeg" "image/png" "image/webp"}) + (media/validate-media-size! file) (update-team-photo cfg (assoc params :profile-id profile-id))) (defn update-team-photo diff --git a/backend/test/backend_tests/rpc_font_test.clj b/backend/test/backend_tests/rpc_font_test.clj index d68f657c59..dce0348e63 100644 --- a/backend/test/backend_tests/rpc_font_test.clj +++ b/backend/test/backend_tests/rpc_font_test.clj @@ -17,7 +17,9 @@ [clojure.test :as t] [datoteka.fs :as fs] [datoteka.io :as io] - [mockery.core :refer [with-mocks]])) + [mockery.core :refer [with-mocks]]) + (:import + java.io.RandomAccessFile)) (t/use-fixtures :once th/state-init) (t/use-fixtures :each th/database-reset) @@ -292,3 +294,499 @@ (let [error (:error out) error-data (ex-data error)] (t/is (th/ex-info? error)))))) + +;; ----------------------------------------------------------------------- +;; Helpers for chunked-upload font tests +;; ----------------------------------------------------------------------- + +(defn- split-bytes-into-chunks + "Splits `data` (byte array) into chunks of at most `chunk-size` bytes. + Returns a vector of byte arrays." + [^bytes data chunk-size] + (let [length (alength data)] + (loop [offset 0 chunks []] + (if (>= offset length) + chunks + (let [remaining (- length offset) + size (min chunk-size remaining) + buf (byte-array size)] + (System/arraycopy data offset buf 0 size) + (recur (+ offset size) (conj chunks buf))))))) + +(defn- make-chunk-mfile + "Writes `data` (byte array) to a tempfile and returns a map + compatible with the upload-chunk :content parameter." + [^bytes data mtype] + (let [tmp (fs/create-tempfile :dir "/tmp/penpot" :prefix "test-font-chunk-")] + (io/write* tmp data) + {:filename "chunk" + :path tmp + :mtype mtype + :size (alength data)})) + +(defn- create-upload-session! + "Creates an upload session for `prof` with `total-chunks`. Returns the session-id UUID." + [prof total-chunks] + (let [out (th/command! {::th/type :create-upload-session + ::rpc/profile-id (:id prof) + :total-chunks total-chunks})] + (t/is (nil? (:error out))) + (:session-id (:result out)))) + +(defn- upload-font-chunked! + "Splits `font-bytes` into chunks of `chunk-size` bytes, creates an upload + session, uploads all chunks, and returns the session-id UUID." + [prof ^bytes font-bytes mtype chunk-size] + (let [chunks (split-bytes-into-chunks font-bytes chunk-size) + session-id (create-upload-session! prof (count chunks))] + (doseq [[idx chunk-data] (map-indexed vector chunks)] + (let [mfile (make-chunk-mfile chunk-data mtype) + out (th/command! {::th/type :upload-chunk + ::rpc/profile-id (:id prof) + :session-id session-id + :index idx + :content mfile})] + (t/is (nil? (:error out))))) + session-id)) + +(defn- assert-font-variant-result + "Checks that a successful create-font-variant result has valid UUIDs and + the expected scalar fields matching `params`." + [params result] + (t/is (uuid? (:id result))) + (t/is (uuid? (:ttf-file-id result))) + (t/is (uuid? (:otf-file-id result))) + (t/is (uuid? (:woff1-file-id result))) + (t/are [k] (= (get params k) (get result k)) + :team-id + :font-id + :font-family + :font-weight + :font-style)) + +;; ----------------------------------------------------------------------- +;; Path 1 – Normal (direct :data bytes) +;; ----------------------------------------------------------------------- + +(t/deftest create-font-variant-normal-ttf + (with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 10) + data (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*)) + params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "chunked-test" + :font-weight 400 + :font-style "normal" + :data {"font/ttf" data}} + out (th/command! params)] + (t/is (= 1 (:call-count @mock))) + (t/is (nil? (:error out))) + (assert-font-variant-result params (:result out))))) + +(t/deftest create-font-variant-normal-otf + (with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 11) + data (-> (io/resource "backend_tests/test_files/font-1.otf") (io/read*)) + params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "chunked-test" + :font-weight 400 + :font-style "normal" + :data {"font/otf" data}} + out (th/command! params)] + (t/is (= 1 (:call-count @mock))) + (t/is (nil? (:error out))) + (assert-font-variant-result params (:result out))))) + +(t/deftest create-font-variant-normal-woff + (with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 12) + data (-> (io/resource "backend_tests/test_files/font-1.woff") (io/read*)) + params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "chunked-test" + :font-weight 400 + :font-style "normal" + :data {"font/woff" data}} + out (th/command! params)] + (t/is (= 1 (:call-count @mock))) + (t/is (nil? (:error out))) + (assert-font-variant-result params (:result out))))) + +;; ----------------------------------------------------------------------- +;; Path 2 – Legacy chunking (:data with vector of byte-arrays per mtype) +;; ----------------------------------------------------------------------- + +(t/deftest create-font-variant-legacy-chunked-ttf + "Upload a TTF via the legacy :data path where each mtype value is a + vector of byte-array chunks (4 MiB each) instead of a single byte-array." + (with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 20) + full-bytes (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*)) + ;; Simulate 4 MiB legacy chunks – font is small so a single chunk suffices + chunks (split-bytes-into-chunks full-bytes (* 4 1024 1024)) + params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "legacy-chunked" + :font-weight 700 + :font-style "italic" + :data {"font/ttf" (vec chunks)}} + out (th/command! params)] + (t/is (= 1 (:call-count @mock))) + (t/is (nil? (:error out))) + (assert-font-variant-result params (:result out))))) + +(t/deftest create-font-variant-legacy-chunked-woff + "Upload a WOFF via the legacy :data path with multiple sub-4 KiB chunks + to exercise the SequenceInputStream concatenation path." + (with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 21) + full-bytes (-> (io/resource "backend_tests/test_files/font-1.woff") (io/read*)) + ;; Split into small chunks to exercise the SequenceInputStream path + chunks (split-bytes-into-chunks full-bytes 512) + params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "legacy-chunked-woff" + :font-weight 400 + :font-style "normal" + :data {"font/woff" (vec chunks)}} + out (th/command! params)] + (t/is (= 1 (:call-count @mock))) + (t/is (nil? (:error out))) + (assert-font-variant-result params (:result out))))) + +;; ----------------------------------------------------------------------- +;; Path 3 – New standardized chunked upload (:uploads map) +;; ----------------------------------------------------------------------- + +(t/deftest create-font-variant-chunked-upload-ttf + "Upload a TTF via the new :uploads path (chunked-upload API)." + (with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 30) + font-bytes (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*)) + session-id (upload-font-chunked! prof font-bytes "font/ttf" (* 4 1024 1024)) + params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "new-chunked" + :font-weight 400 + :font-style "normal" + :uploads {"font/ttf" session-id}} + out (th/command! params)] + ;; quotes/check! is called at least once (for the font-variant quota) plus + ;; once during session creation — assert it fired at least once. + (t/is (>= (:call-count @mock) 1)) + (t/is (nil? (:error out))) + (assert-font-variant-result params (:result out))))) + +(t/deftest create-font-variant-chunked-upload-otf + "Upload an OTF via the new :uploads path." + (with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 31) + font-bytes (-> (io/resource "backend_tests/test_files/font-1.otf") (io/read*)) + session-id (upload-font-chunked! prof font-bytes "font/otf" (* 4 1024 1024)) + params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "new-chunked-otf" + :font-weight 400 + :font-style "normal" + :uploads {"font/otf" session-id}} + out (th/command! params)] + (t/is (>= (:call-count @mock) 1)) + (t/is (nil? (:error out))) + (assert-font-variant-result params (:result out))))) + +(t/deftest create-font-variant-chunked-upload-woff + "Upload a WOFF via the new :uploads path." + (with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 32) + font-bytes (-> (io/resource "backend_tests/test_files/font-1.woff") (io/read*)) + session-id (upload-font-chunked! prof font-bytes "font/woff" (* 4 1024 1024)) + params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "new-chunked-woff" + :font-weight 400 + :font-style "normal" + :uploads {"font/woff" session-id}} + out (th/command! params)] + (t/is (>= (:call-count @mock) 1)) + (t/is (nil? (:error out))) + (assert-font-variant-result params (:result out))))) + +(t/deftest create-font-variant-chunked-upload-multi-chunk + "Upload a WOFF split into many small chunks to exercise multi-chunk assembly." + (with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 33) + font-bytes (-> (io/resource "backend_tests/test_files/font-1.woff") (io/read*)) + ;; Use a chunk-size smaller than 4 MiB to force multiple chunks while + ;; staying within the 20-chunk-per-session quota limit (29836 / 2000 = ~15 chunks). + session-id (upload-font-chunked! prof font-bytes "font/woff" 2000) + params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "multi-chunk-woff" + :font-weight 400 + :font-style "normal" + :uploads {"font/woff" session-id}} + out (th/command! params)] + (t/is (>= (:call-count @mock) 1)) + (t/is (nil? (:error out))) + (assert-font-variant-result params (:result out))))) + +;; ----------------------------------------------------------------------- +;; Error cases +;; ----------------------------------------------------------------------- + +(t/deftest create-font-variant-missing-data-and-uploads + "Neither :data nor :uploads is present — schema validation must reject it." + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 40) + params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "bad" + :font-weight 400 + :font-style "normal"} + out (th/command! params)] + (t/is (some? (:error out))) + (t/is (= :validation (-> out :error ex-data :type))))) + +(t/deftest create-font-variant-chunked-upload-missing-chunks + "When only some chunks are uploaded the assembly step must fail." + (with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 41) + font-bytes (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*)) + ;; 5000-byte chunks → 68640/5000 = 14 chunks; declare 15 but only upload 13 + chunks (split-bytes-into-chunks font-bytes 5000) + ;; Declare one extra chunk so assembly will fail (not all chunks present) + session-id (create-upload-session! prof (inc (count chunks)))] + + ;; Upload all real chunks except the last one (omit it so the session is incomplete) + (doseq [[idx chunk-data] (map-indexed vector (butlast chunks))] + (let [mfile (make-chunk-mfile chunk-data "font/ttf") + out (th/command! {::th/type :upload-chunk + ::rpc/profile-id (:id prof) + :session-id session-id + :index idx + :content mfile})] + (t/is (nil? (:error out))))) + + (let [out (th/command! {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "missing-chunks" + :font-weight 400 + :font-style "normal" + :uploads {"font/ttf" session-id}})] + (t/is (some? (:error out))))))) + +(t/deftest create-font-variant-chunked-upload-invalid-session + "Passing a non-existent session-id must fail at assembly time." + (with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 42) + out (th/command! {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "bad-session" + :font-weight 400 + :font-style "normal" + :uploads {"font/ttf" (uuid/next)}})] + (t/is (some? (:error out)))))) + +;; ----------------------------------------------------------------------- +;; Font size validation tests +;; ----------------------------------------------------------------------- + +(t/deftest create-font-variant-size-exceeded-normal + "Direct :data upload exceeding font-max-file-size must be rejected." + (with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}] + (with-redefs [app.config/config (assoc app.config/config :font-max-file-size 1)] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 50) + data (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*)) + params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "size-exceeded" + :font-weight 400 + :font-style "normal" + :data {"font/ttf" data}} + out (th/command! params)] + (t/is (some? (:error out))) + (t/is (= :restriction (-> out :error ex-data :type))) + (t/is (= :font-max-file-size-reached (-> out :error ex-data :code))))))) + +(t/deftest create-font-variant-size-exceeded-legacy-chunked + "Legacy :data chunk-vector upload exceeding font-max-file-size must be rejected." + (with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}] + (with-redefs [app.config/config (assoc app.config/config :font-max-file-size 1)] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 51) + full-bytes (-> (io/resource "backend_tests/test_files/font-1.woff") (io/read*)) + chunks (split-bytes-into-chunks full-bytes (* 4 1024 1024)) + params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "size-exceeded-legacy" + :font-weight 400 + :font-style "normal" + :data {"font/woff" (vec chunks)}} + out (th/command! params)] + (t/is (some? (:error out))) + (t/is (= :restriction (-> out :error ex-data :type))) + (t/is (= :font-max-file-size-reached (-> out :error ex-data :code))))))) + +(t/deftest create-font-variant-size-exceeded-chunked-upload + "New :uploads path exceeding font-max-file-size must be rejected after assembly." + (with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 52) + font-bytes (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*)) + session-id (upload-font-chunked! prof font-bytes "font/ttf" (* 4 1024 1024))] + (with-redefs [app.config/config (assoc app.config/config :font-max-file-size 1)] + (let [out (th/command! {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "size-exceeded-chunked" + :font-weight 400 + :font-style "normal" + :uploads {"font/ttf" session-id}})] + (t/is (some? (:error out))) + (t/is (= :restriction (-> out :error ex-data :type))) + (t/is (= :font-max-file-size-reached (-> out :error ex-data :code)))))))) + +(t/deftest create-font-variant-size-within-limit + "Upload exactly at the limit must succeed." + (with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 53) + font-bytes (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*)) + font-size (alength ^bytes font-bytes)] + (with-redefs [app.config/config (assoc app.config/config :font-max-file-size font-size)] + (let [params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "size-at-limit" + :font-weight 400 + :font-style "normal" + :data {"font/ttf" font-bytes}} + out (th/command! params)] + (t/is (nil? (:error out))) + (assert-font-variant-result params (:result out))))))) + +;; ----------------------------------------------------------------------- +;; Font media-type validation tests +;; ----------------------------------------------------------------------- + +(t/deftest create-font-variant-invalid-type-normal + "Direct :data upload with a disallowed mtype must be rejected." + (with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 60) + data (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*)) + params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "invalid-type" + :font-weight 400 + :font-style "normal" + :data {"application/octet-stream" data}} + out (th/command! params)] + (t/is (some? (:error out))) + (t/is (= :validation (-> out :error ex-data :type))) + (t/is (= :media-type-not-allowed (-> out :error ex-data :code)))))) + +(t/deftest create-font-variant-invalid-type-legacy-chunked + "Legacy :data chunk-vector upload with a disallowed mtype must be rejected." + (with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 61) + full-bytes (-> (io/resource "backend_tests/test_files/font-1.woff") (io/read*)) + chunks (split-bytes-into-chunks full-bytes (* 4 1024 1024)) + params {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "invalid-type-legacy" + :font-weight 400 + :font-style "normal" + :data {"image/png" (vec chunks)}} + out (th/command! params)] + (t/is (some? (:error out))) + (t/is (= :validation (-> out :error ex-data :type))) + (t/is (= :media-type-not-allowed (-> out :error ex-data :code)))))) + +(t/deftest create-font-variant-invalid-type-chunked-upload + "New :uploads path with a disallowed mtype must be rejected after assembly." + (with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}] + (let [prof (th/create-profile* 1 {:is-active true}) + team-id (:default-team-id prof) + font-id (uuid/custom 10 62) + font-bytes (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*)) + ;; Upload the bytes under a valid session but lie about the mtype + ;; when calling create-font-variant. + session-id (upload-font-chunked! prof font-bytes "font/ttf" (* 4 1024 1024)) + out (th/command! {::th/type :create-font-variant + ::rpc/profile-id (:id prof) + :team-id team-id + :font-id font-id + :font-family "invalid-type-chunked" + :font-weight 400 + :font-style "normal" + :uploads {"image/jpeg" session-id}})] + (t/is (some? (:error out))) + (t/is (= :validation (-> out :error ex-data :type))) + (t/is (= :media-type-not-allowed (-> out :error ex-data :code)))))) diff --git a/common/src/app/common/media.cljc b/common/src/app/common/media.cljc index 4cdf8488ce..721bea2dc6 100644 --- a/common/src/app/common/media.cljc +++ b/common/src/app/common/media.cljc @@ -12,8 +12,8 @@ (def font-types #{"font/ttf" "font/woff" - "font/otf" - "font/opentype"}) + "font/woff2" + "font/otf"}) (def image-types #{"image/jpeg" diff --git a/frontend/src/app/main/data/fonts.cljs b/frontend/src/app/main/data/fonts.cljs index 63c7e61bf5..0864247f5c 100644 --- a/frontend/src/app/main/data/fonts.cljs +++ b/frontend/src/app/main/data/fonts.cljs @@ -14,6 +14,7 @@ [app.common.uuid :as uuid] [app.main.data.event :as ev] [app.main.data.notifications :as ntf] + [app.main.data.uploads :as uploads] [app.main.fonts :as fonts] [app.main.repo :as rp] [app.main.store :as st] @@ -24,24 +25,14 @@ [cuerdas.core :as str] [potok.v2.core :as ptk])) -(def ^:const default-chunk-size - (* 1024 1024 4)) ;; 4MiB - -(defn- chunk-array - [data chunk-size] - (let [total-size (alength data)] - (loop [offset 0 - chunks []] - (if (< offset total-size) - (let [end (min (+ offset chunk-size) total-size) - chunk (.subarray ^js data offset end)] - (recur end (conj chunks chunk))) - chunks)))) - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; General purpose events & IMPL ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(def ^:private font-upload-chunk-size + "Size in bytes of each chunk when uploading font files (10 MiB)." + (* 1024 1024 10)) + (defn fonts-fetched [fonts] (letfn [;; Prepare font to the internal font database format. @@ -94,9 +85,44 @@ (->> (rp/cmd! :get-font-variants {:team-id team-id}) (rx/map fonts-fetched))))) +(defn upload-font-variant + "Uploads a single font variant item using the chunked upload API. + + For each mime-type in `data`, creates a Blob and uploads it via the + session-based chunked upload. Once all sessions are created, calls + `create-font-variant` with the resulting `:uploads` map so the server + can assemble the chunks and materialise the final font-variant record. + + Returns an observable that emits the created font-variant." + [{:keys [data team-id font-id font-family font-weight font-style] :as _item}] + ;; Upload each mtype as a separate chunked session in parallel, collect + ;; all [mtype session-id] pairs, then call create-font-variant with :uploads. + (->> (rx/from (seq data)) + (rx/mapcat (fn [[mtype buffer]] + (let [blob (js/Blob. #js [buffer] #js {:type mtype})] + (->> (uploads/upload-blob-chunked blob :chunk-size font-upload-chunk-size) + (rx/map (fn [{:keys [session-id]}] + [mtype session-id])))))) + (rx/reduce (fn [acc [mtype session-id]] + (assoc acc mtype session-id)) + {}) + (rx/mapcat (fn [uploads] + (rp/cmd! :create-font-variant + {:team-id team-id + :font-id font-id + :font-family font-family + :font-weight font-weight + :font-style font-style + :uploads uploads}))))) + (defn process-upload "Given a seq of blobs and the team id, creates a ready-to-use fonts - map with temporal ID's associated to each font entry." + map with temporal ID's associated to each font entry. + + Each font entry's `:data` is a map of `{mtype -> ArrayBuffer}`. The + raw `ArrayBuffer` is kept as-is so that `upload-font-variant` can + wrap it in a `Blob` and hand it directly to `upload-blob-chunked` + without any intermediate client-side chunking." [blobs team-id] (letfn [(prepare [{:keys [font type name data] :as params}] (let [family (or (.getEnglishName ^js font "preferredFamily") @@ -130,9 +156,8 @@ (not= hhea-descender win-descent) (and f-selection (or (not= hhea-ascender os2-ascent) - (not= hhea-descender os2-descent)))) - data (js/Uint8Array. data)] - {:content {:data (chunk-array data default-chunk-size) + (not= hhea-descender os2-descent))))] + {:content {:data data :name name :type type} :font-family (or family "") diff --git a/frontend/src/app/main/data/uploads.cljs b/frontend/src/app/main/data/uploads.cljs index 06e87f02d9..2721cd99cb 100644 --- a/frontend/src/app/main/data/uploads.cljs +++ b/frontend/src/app/main/data/uploads.cljs @@ -24,10 +24,6 @@ [app.main.repo :as rp] [beicon.v2.core :as rx])) -;; Size of each upload chunk in bytes. Reads the penpotUploadChunkSize global -;; variable at startup; defaults to 25 MiB (overridden in production). -(def ^:private chunk-size cf/upload-chunk-size) - (def ^:private max-parallel-chunk-uploads "Maximum number of chunk upload requests that may be in-flight at the same time within a single chunked upload session." @@ -44,8 +40,11 @@ Returns an observable that emits exactly one map: `{:session-id }` - The caller is responsible for the final step (assemble / import)." - [blob] + The caller is responsible for the final step (assemble / import). + + The optional `opts` map accepts: + `:chunk-size` – size in bytes of each chunk (default: `cf/upload-chunk-size`, 25 MiB)." + [blob & {:keys [chunk-size] :or {chunk-size cf/upload-chunk-size}}] (let [total-size (.-size blob) total-chunks (js/Math.ceil (/ total-size chunk-size))] (->> (rp/cmd! :create-upload-session diff --git a/frontend/src/app/main/ui/dashboard/fonts.cljs b/frontend/src/app/main/ui/dashboard/fonts.cljs index f84d694893..92942b9c52 100644 --- a/frontend/src/app/main/ui/dashboard/fonts.cljs +++ b/frontend/src/app/main/ui/dashboard/fonts.cljs @@ -14,7 +14,6 @@ [app.main.data.fonts :as df] [app.main.data.modal :as modal] [app.main.data.notifications :as ntf] - [app.main.repo :as rp] [app.main.store :as st] [app.main.ui.components.context-menu-a11y :refer [context-menu*]] [app.main.ui.components.file-uploader :refer [file-uploader]] @@ -110,7 +109,7 @@ (mf/use-fn (fn [{:keys [id] :as item}] (swap! uploading* conj id) - (->> (rp/cmd! :create-font-variant item) + (->> (df/upload-font-variant item) (rx/delay-at-least 2000) (rx/subs! (fn [font] (swap! fonts* dissoc id) diff --git a/scripts/check-commit b/scripts/check-commit new file mode 100755 index 0000000000..478aee5156 --- /dev/null +++ b/scripts/check-commit @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +""" +Check commit messages against Penpot's commit guidelines. + +Validates commit messages using the rules defined in: + - .github/workflows/commit-checker.yml (regex pattern) + - CONTRIBUTING.md (formatting rules, subject length, DCO) + +By default, checks HEAD. Use --commit to specify a different commit. + +Usage: + ./scripts/check-commit + ./scripts/check-commit --commit HEAD~1 + ./scripts/check-commit -c abc1234 +""" + +import argparse +import re +import subprocess +import sys + +# ── Emoji list ─────────────────────────────────────────────────────────────── +# Combined from commit-checker.yml AND CONTRIBUTING.md +VALID_EMOJIS = ( + "lipstick|globe_with_meridians|wrench|books|" + "arrow_up|arrow_down|zap|ambulance|construction|" + "boom|fire|whale|bug|sparkles|paperclip|tada|" + "recycle|rewind|construction_worker|rocket" +) + +# ── Regex from .github/workflows/commit-checker.yml ────────────────────────── +# Matches: +# 1) ":emoji: " +# 2) "Merge|Revert|Reapply ... without trailing dot" +COMMIT_PATTERN = re.compile( + r"^((:(" + VALID_EMOJIS + r"):\s[A-Z].*[^.]))$" +) + +MERGE_PATTERN = re.compile(r"^(Merge|Revert|Reapply).+[^.]$") + +# ═══════════════════════════════════════════════════════════════════════════════ +# Helpers +# ═══════════════════════════════════════════════════════════════════════════════ + +def run_git(args): + """Run a git command and return (returncode, stdout, stderr).""" + try: + result = subprocess.run( + ["git"] + args, + capture_output=True, + text=True, + check=False, + ) + return result.returncode, result.stdout, result.stderr + except FileNotFoundError: + print("ERROR: git not found. Is it installed?", file=sys.stderr) + sys.exit(1) + + +def get_commit_message(commit_ref): + """Return the full commit message for *commit_ref*.""" + rc, out, err = run_git(["log", "--format=%B", "-n", "1", commit_ref]) + if rc != 0: + print(f"ERROR: could not read commit {commit_ref}: {err.strip()}", file=sys.stderr) + sys.exit(1) + if not out.strip(): + print(f"ERROR: commit {commit_ref} has no message", file=sys.stderr) + sys.exit(1) + return out.rstrip("\n") + + +# ═══════════════════════════════════════════════════════════════════════════════ +# Validators +# ═══════════════════════════════════════════════════════════════════════════════ + +def check_regex(message): + """Check the commit message against the CI regex pattern.""" + # Normalise: strip trailing newlines for single-line matching + first_line = message.split("\n")[0] + + if MERGE_PATTERN.match(first_line): + return True, None + + if COMMIT_PATTERN.match(first_line): + return True, None + + return False, ( + "Commit subject must match one of:\n" + " :emoji: \n" + " Merge|Revert|Reapply \n" + f"Got: {first_line!r}" + ) + + +def check_subject_length(message): + """Subject line must be ≤ 90 characters.""" + first_line = message.split("\n")[0] + if len(first_line) > 90: + return False, ( + f"Subject line exceeds 90 characters ({len(first_line)} chars):\n" + f" {first_line}" + ) + return True, None + + +def check_subject_no_trailing_dot(message): + """Subject line must not end with a period ('.').""" + first_line = message.split("\n")[0] + if first_line.endswith("."): + return False, ( + "Subject line must not end with a period:\n" + f" {first_line}" + ) + return True, None + + +def check_subject_capitalized(message): + """Subject must be capitalized, but only if it's a regular commit (not Merge/Revert/Reapply).""" + first_line = message.split("\n")[0] + + # Skip check for Merge/Revert/Reapply commits + if MERGE_PATTERN.match(first_line): + return True, None + + # Strip emoji prefix before checking capitalization + emoji_match = re.match(r"^:([a-z_]+):\s+(.*)", first_line) + if emoji_match: + rest = emoji_match.group(2) + else: + rest = first_line + + if rest and not rest[0].isupper(): + return False, ( + "Subject line must start with a capital letter " + "(after the emoji prefix):\n" + f" {first_line}" + ) + return True, None + + +def check_body_blank_line(message): + """If a body exists, there must be a blank line between subject and body.""" + lines = message.split("\n") + if len(lines) >= 3 and lines[1] != "": + return False, ( + "A blank line must separate the subject from the body." + ) + return True, None + + +def check_signed_off_by(message): + """Check for the DCO Signed-off-by line (required for code changes).""" + if "Signed-off-by:" not in message: + return False, ( + "Missing 'Signed-off-by:' line in the commit footer.\n" + " Add it with 'git commit -s' or append it manually:\n" + " Signed-off-by: Your Real Name " + ) + return True, None + + +# ═══════════════════════════════════════════════════════════════════════════════ + +def main(): + parser = argparse.ArgumentParser( + description="Check a commit message against Penpot commit guidelines." + ) + parser.add_argument( + "-c", "--commit", + default="HEAD", + help="Commit to check (default: HEAD)", + ) + args = parser.parse_args() + + commit_ref = args.commit + message = get_commit_message(commit_ref) + + print(f"Checking commit {commit_ref} ...\n") + + validators = [ + ("Regex pattern", check_regex), + ("Subject ≤ 90 chars", check_subject_length), + ("No trailing period in subject", check_subject_no_trailing_dot), + ("Subject capitalized", check_subject_capitalized), + ("Blank line after subject", check_body_blank_line), + ] + + all_ok = True + + for name, validator in validators: + ok, error_msg = validator(message) + status = "✓" if ok else "✗" + print(f" [{status}] {name}") + if not ok: + all_ok = False + print(f" {error_msg}", file=sys.stderr) + + print() + if all_ok: + print("All checks passed.") + sys.exit(0) + else: + print("Some checks FAILED. See messages above.", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main()