mirror of
https://github.com/penpot/penpot.git
synced 2026-05-13 12:04:06 +00:00
🎉 Add chunked upload support for font variants (#9551)
* ✨ Add additional logging and validation for image upload * 🎉 Add chunked upload support for font variants Extend the font variant upload flow across frontend, backend, and common to support the standardized chunked upload protocol. **Backend:** - Add \`:font-max-file-size\` config default (30 MiB) and schema entry - Add \`validate-font-size!\` in \`media.clj\` (mirrors \`validate-media-size!\`, raises \`:font-max-file-size-reached\`) - Extend \`schema:create-font-variant\` to accept either \`:data\` (legacy bytes or chunk-vector) or \`:uploads\` (new chunked session map), with a validator requiring exactly one - Add \`prepare-font-data-from-uploads\`: assembles each chunked session via \`cmedia/assemble-chunks\`, validates type+size - Add \`prepare-font-data-from-legacy\`: normalises legacy byte/chunk entries, writing to a tempfile (joining via SequenceInputStream), validates type+size - Add structured logging ("init"/"end") with \`:size\`, \`:mtypes\`, and \`:elapsed\` in \`create-font-variant\` **Frontend:** - \`upload-blob-chunked\` accepts a per-caller \`:chunk-size\` option - Add \`font-upload-chunk-size\` (10 MiB) and \`upload-font-variant\` fn that uploads each mtype as a separate chunked session - \`on-upload*\` in dashboard fonts now calls \`upload-font-variant\` instead of issuing \`create-font-variant\` RPC directly - \`process-upload\` stores raw ArrayBuffer instead of chunking client-side **Common:** - Replace \`"font/opentype"\` with \`"font/woff2"\` in \`font-types\` **Tests:** - 25 tests / 224 assertions covering all three upload paths (direct bytes, legacy chunk-vector, new chunked sessions), size validation, and media type validation Signed-off-by: Andrey Antukh <niwi@niwi.nz> * 📎 Add a script for check the commit format locally --------- Signed-off-by: Andrey Antukh <niwi@niwi.nz>
This commit is contained in:
Andrey Antukh
GitHub
parent
ade0d2d0a8
commit
947f6d392d
@ -72,6 +72,7 @@
|
||||
:telemetry-uri "https://telemetry.penpot.app/"
|
||||
|
||||
:media-max-file-size (* 1024 1024 30) ; 30MiB
|
||||
:font-max-file-size (* 1024 1024 30) ; 30MiB
|
||||
|
||||
:ldap-user-query "(|(uid=:username)(mail=:username))"
|
||||
:ldap-attrs-username "uid"
|
||||
@ -120,6 +121,7 @@
|
||||
[:auto-file-snapshot-timeout {:optional true} ::ct/duration]
|
||||
|
||||
[:media-max-file-size {:optional true} ::sm/int]
|
||||
[:font-max-file-size {:optional true} ::sm/int]
|
||||
[:deletion-delay {:optional true} ::ct/duration]
|
||||
[:file-clean-delay {:optional true} ::ct/duration]
|
||||
[:telemetry-enabled {:optional true} ::sm/boolean]
|
||||
|
||||
@ -38,9 +38,6 @@
|
||||
org.im4java.core.ConvertCmd
|
||||
org.im4java.core.IMOperation))
|
||||
|
||||
(def default-max-file-size
|
||||
(* 1024 1024 10)) ; 10 MiB
|
||||
|
||||
(def schema:upload
|
||||
[:map {:title "Upload"}
|
||||
[:filename :string]
|
||||
@ -79,6 +76,20 @@
|
||||
max-size)))
|
||||
upload))
|
||||
|
||||
(defn validate-font-size!
|
||||
"Validates that the font file `upload` does not exceed the configured
|
||||
`:font-max-file-size` limit. Accepts the same map shape as
|
||||
`validate-media-size!` — requires a `:size` key in bytes."
|
||||
[upload]
|
||||
(let [max-size (cf/get :font-max-file-size)]
|
||||
(when (> (:size upload) max-size)
|
||||
(ex/raise :type :restriction
|
||||
:code :font-max-file-size-reached
|
||||
:hint (str/ffmt "the uploaded font size % is greater than the maximum %"
|
||||
(:size upload)
|
||||
max-size)))
|
||||
upload))
|
||||
|
||||
(defmulti process :cmd)
|
||||
(defmulti process-error class)
|
||||
|
||||
@ -296,9 +307,7 @@
|
||||
[{:keys [::http/client]} uri]
|
||||
(letfn [(parse-and-validate [{:keys [status headers] :as response}]
|
||||
(let [size (some-> (get headers "content-length") d/parse-integer)
|
||||
mtype (get headers "content-type")
|
||||
format (cm/mtype->format mtype)
|
||||
max-size (cf/get :media-max-file-size default-max-file-size)]
|
||||
mtype (get headers "content-type")]
|
||||
|
||||
(when-not (<= 200 status 299)
|
||||
(ex/raise :type :validation
|
||||
@ -310,19 +319,9 @@
|
||||
:code :unknown-size
|
||||
:hint "seems like the url points to resource with unknown size"))
|
||||
|
||||
(when (> size max-size)
|
||||
(ex/raise :type :validation
|
||||
:code :file-too-large
|
||||
:hint (str/ffmt "the file size % is greater than the maximum %"
|
||||
size
|
||||
default-max-file-size)))
|
||||
|
||||
(when (nil? format)
|
||||
(ex/raise :type :validation
|
||||
:code :media-type-not-allowed
|
||||
:hint "seems like the url points to an invalid media object"))
|
||||
|
||||
{:size size :mtype mtype :format format}))]
|
||||
(-> {:size size :mtype mtype}
|
||||
(validate-media-type!)
|
||||
(validate-media-size!))))]
|
||||
|
||||
(let [{:keys [body] :as response}
|
||||
(try
|
||||
|
||||
@ -9,6 +9,8 @@
|
||||
[app.binfile.common :as bfc]
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.media :as cm]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.time :as ct]
|
||||
[app.common.uuid :as uuid]
|
||||
@ -21,6 +23,7 @@
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.rpc.climit :as-alias climit]
|
||||
[app.rpc.commands.files :as files]
|
||||
[app.rpc.commands.media :as cmedia]
|
||||
[app.rpc.commands.projects :as projects]
|
||||
[app.rpc.commands.teams :as teams]
|
||||
[app.rpc.doc :as-alias doc]
|
||||
@ -29,6 +32,8 @@
|
||||
[app.storage :as sto]
|
||||
[app.storage.tmp :as tmp]
|
||||
[app.util.services :as sv]
|
||||
[cuerdas.core :as str]
|
||||
[datoteka.fs :as fs]
|
||||
[datoteka.io :as io])
|
||||
(:import
|
||||
java.io.InputStream
|
||||
@ -87,32 +92,92 @@
|
||||
(declare create-font-variant)
|
||||
|
||||
(def ^:private schema:create-font-variant
|
||||
[:map {:title "create-font-variant"}
|
||||
[:team-id ::sm/uuid]
|
||||
[:data [:map-of ::sm/text [:or ::sm/bytes
|
||||
[::sm/vec ::sm/bytes]]]]
|
||||
[:font-id ::sm/uuid]
|
||||
[:font-family ::sm/text]
|
||||
[:font-weight [::sm/one-of {:format "number"} valid-weight]]
|
||||
[:font-style [::sm/one-of {:format "string"} valid-style]]])
|
||||
[:and
|
||||
[:map {:title "create-font-variant"}
|
||||
[:team-id ::sm/uuid]
|
||||
[:font-id ::sm/uuid]
|
||||
[:font-family ::sm/text]
|
||||
[:font-weight [::sm/one-of {:format "number"} valid-weight]]
|
||||
[:font-style [::sm/one-of {:format "string"} valid-style]]
|
||||
[:data {:optional true} [:map-of ::sm/text [:or ::sm/bytes [::sm/vec ::sm/bytes]]]]
|
||||
[:uploads {:optional true} [:map-of ::sm/text ::sm/uuid]]]
|
||||
[:fn {:error/message "one of :data or :uploads is required"}
|
||||
(fn [{:keys [data uploads]}]
|
||||
(or (seq data) (seq uploads)))]])
|
||||
|
||||
;; FIXME: IMPORTANT: refactor this, we should not hold a whole db
|
||||
;; connection around the font creation
|
||||
|
||||
(defn- prepare-font-data-from-uploads
|
||||
"Assembles each chunked-upload session in `uploads` (a `{mtype →
|
||||
session-id}` map) into a temp file, validates the media type and
|
||||
size of every entry, and returns a `{mtype → path}` data map."
|
||||
[cfg {:keys [uploads] :as params}]
|
||||
(let [data (reduce-kv
|
||||
(fn [acc mtype session-id]
|
||||
(let [assembled (cmedia/assemble-chunks cfg session-id)]
|
||||
(-> {:mtype mtype :size (:size assembled)}
|
||||
(media/validate-media-type! cm/font-types)
|
||||
(media/validate-font-size!))
|
||||
(assoc acc mtype (:path assembled))))
|
||||
{}
|
||||
uploads)]
|
||||
|
||||
(-> params
|
||||
(assoc :data data)
|
||||
(dissoc :uploads))))
|
||||
|
||||
(defn- prepare-font-data-from-legacy
|
||||
"Validates the media type and size of every entry in the legacy
|
||||
`:data` map (a `{mtype → bytes | [bytes]}` map). Normalises every
|
||||
entry to a tempfile. Returns params with a normalised
|
||||
`{mtype → path}` data map."
|
||||
[{:keys [data] :as params}]
|
||||
(let [data (reduce-kv
|
||||
(fn [acc mtype content]
|
||||
(let [tmp (tmp/tempfile :prefix "penpot.tempfont." :suffix "")
|
||||
chunks (if (vector? content) content [content])
|
||||
streams (map io/input-stream chunks)
|
||||
streams (Collections/enumeration streams)]
|
||||
|
||||
;; Generate the tempfile from all chunks
|
||||
(with-open [^OutputStream output (io/output-stream tmp)
|
||||
^InputStream input (SequenceInputStream. streams)]
|
||||
(io/copy input output))
|
||||
|
||||
;; Validate
|
||||
(-> {:mtype mtype :size (fs/size tmp)}
|
||||
(media/validate-media-type! cm/font-types)
|
||||
(media/validate-font-size!))
|
||||
|
||||
(assoc acc mtype tmp)))
|
||||
{}
|
||||
data)]
|
||||
(assoc params :data data)))
|
||||
|
||||
(sv/defmethod ::create-font-variant
|
||||
"Upload a font variant. Font data may be provided either as a
|
||||
Transit-encoded `:data` map (keyed by mime-type) for small fonts, or
|
||||
as an `:uploads` map (keyed by mime-type, values are upload-session
|
||||
UUIDs from the chunked-upload API) for large fonts. Exactly one of
|
||||
the two must be present."
|
||||
{::doc/added "1.18"
|
||||
::doc/changes ["2.16" "Add :uploads param for chunked upload support"]
|
||||
::climit/id [[:process-font/by-profile ::rpc/profile-id]
|
||||
[:process-font/global]]
|
||||
::webhooks/event? true
|
||||
::sm/params schema:create-font-variant}
|
||||
[cfg {:keys [::rpc/profile-id team-id] :as params}]
|
||||
[cfg {:keys [::rpc/profile-id team-id uploads] :as params}]
|
||||
(db/tx-run! cfg
|
||||
(fn [{:keys [::db/conn] :as cfg}]
|
||||
(teams/check-edition-permissions! conn profile-id team-id)
|
||||
(quotes/check! cfg {::quotes/id ::quotes/font-variants-per-team
|
||||
::quotes/profile-id profile-id
|
||||
::quotes/team-id team-id})
|
||||
(create-font-variant cfg (assoc params :profile-id profile-id)))))
|
||||
(let [params (if (some? uploads)
|
||||
(prepare-font-data-from-uploads cfg params)
|
||||
(prepare-font-data-from-legacy params))]
|
||||
(create-font-variant cfg (assoc params :profile-id profile-id))))))
|
||||
|
||||
(defn create-font-variant
|
||||
[{:keys [::sto/storage ::db/conn]} {:keys [data] :as params}]
|
||||
@ -127,23 +192,6 @@
|
||||
:hint "invalid font upload, unable to generate missing font assets"))
|
||||
data))
|
||||
|
||||
(process-chunks [chunks]
|
||||
(let [tmp (tmp/tempfile :prefix "penpot.tempfont." :suffix "")
|
||||
streams (map io/input-stream chunks)
|
||||
streams (Collections/enumeration streams)]
|
||||
(with-open [^OutputStream output (io/output-stream tmp)
|
||||
^InputStream input (SequenceInputStream. streams)]
|
||||
(io/copy input output))
|
||||
tmp))
|
||||
|
||||
(join-chunks [data]
|
||||
(reduce-kv (fn [data mtype content]
|
||||
(if (vector? content)
|
||||
(assoc data mtype (process-chunks content))
|
||||
data))
|
||||
data
|
||||
data))
|
||||
|
||||
(prepare-font [data mtype]
|
||||
(when-let [resource (get data mtype)]
|
||||
|
||||
@ -185,11 +233,38 @@
|
||||
:otf-file-id (:id otf)
|
||||
:ttf-file-id (:id ttf)}))]
|
||||
|
||||
(let [data (join-chunks data)
|
||||
data (generate-missing data)
|
||||
assets (persist-fonts-files! data)
|
||||
result (insert-font-variant! assets)]
|
||||
(vary-meta result assoc ::audit/replace-props (update params :data (comp vec keys))))))
|
||||
(let [tpoint (ct/tpoint)
|
||||
mtypes (vec (keys data))
|
||||
total-size (reduce-kv (fn [acc _ content]
|
||||
(+ acc (if (bytes? content)
|
||||
(alength ^bytes content)
|
||||
(fs/size content))))
|
||||
0
|
||||
data)]
|
||||
|
||||
(l/dbg :hint "create-font-variant"
|
||||
:step "init"
|
||||
:font-family (:font-family params)
|
||||
:font-weight (:font-weight params)
|
||||
:font-style (:font-style params)
|
||||
:mtypes (str/join mtypes ",")
|
||||
:size total-size)
|
||||
|
||||
(let [data (generate-missing data)
|
||||
assets (persist-fonts-files! data)
|
||||
result (insert-font-variant! assets)
|
||||
elapsed (tpoint)]
|
||||
|
||||
(l/dbg :hint "create-font-variant"
|
||||
:step "end"
|
||||
:font-family (:font-family params)
|
||||
:font-weight (:font-weight params)
|
||||
:font-style (:font-style params)
|
||||
:mtypes (str/join mtypes ",")
|
||||
:size total-size
|
||||
:elapsed (ct/format-duration elapsed))
|
||||
|
||||
(vary-meta result assoc ::audit/replace-props (update params :data (comp vec keys)))))))
|
||||
|
||||
;; --- UPDATE FONT FAMILY
|
||||
|
||||
|
||||
@ -8,6 +8,7 @@
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.common.schema :as sm]
|
||||
[app.common.time :as ct]
|
||||
[app.common.uuid :as uuid]
|
||||
@ -58,8 +59,8 @@
|
||||
(db/run! cfg (fn [{:keys [::db/conn] :as cfg}]
|
||||
;; We get the minimal file for proper checking if
|
||||
;; file is not already deleted
|
||||
(let [_ (files/get-minimal-file conn file-id)
|
||||
mobj (create-file-media-object cfg params)]
|
||||
(let [_ (files/get-minimal-file conn file-id)
|
||||
mobj (create-file-media-object cfg params)]
|
||||
|
||||
(db/update! conn :file
|
||||
{:modified-at (ct/now)
|
||||
@ -149,20 +150,49 @@
|
||||
|
||||
(defn- create-file-media-object
|
||||
[{:keys [::sto/storage ::db/conn] :as cfg}
|
||||
{:keys [id file-id is-local name content]}]
|
||||
(let [result (process-image content)
|
||||
image (sto/put-object! storage (::image result))
|
||||
thumb (when-let [params (::thumb result)]
|
||||
(sto/put-object! storage params))]
|
||||
{:keys [id file-id is-local name content from-url? from-chunks?]}]
|
||||
|
||||
(db/exec-one! conn [sql:create-file-media-object
|
||||
(or id (uuid/next))
|
||||
file-id is-local name
|
||||
(:id image)
|
||||
(:id thumb)
|
||||
(:width result)
|
||||
(:height result)
|
||||
(:mtype result)])))
|
||||
(let [tpoint (ct/tpoint)
|
||||
id (or id (uuid/next))
|
||||
origin (cond
|
||||
from-url?
|
||||
"url"
|
||||
from-chunks?
|
||||
"chunks"
|
||||
:else
|
||||
"direct")]
|
||||
|
||||
(l/dbg :hint "create file-media-object"
|
||||
:step "init"
|
||||
:id (str id)
|
||||
:mtype (:mtype content)
|
||||
:size (:size content)
|
||||
:path (str (:path content))
|
||||
:origin origin)
|
||||
|
||||
(let [result (process-image content)
|
||||
image (sto/put-object! storage (::image result))
|
||||
thumb (when-let [params (::thumb result)]
|
||||
(sto/put-object! storage params))
|
||||
elapsed (tpoint)]
|
||||
|
||||
(l/dbg :hint "create file-media-object"
|
||||
:step "end"
|
||||
:id (str id)
|
||||
:mtype (:mtype content)
|
||||
:size (:size content)
|
||||
:path (str (:path content))
|
||||
:origin origin
|
||||
:elapsed (ct/format-duration elapsed))
|
||||
|
||||
(db/exec-one! conn [sql:create-file-media-object
|
||||
id
|
||||
file-id is-local name
|
||||
(:id image)
|
||||
(:id thumb)
|
||||
(:width result)
|
||||
(:height result)
|
||||
(:mtype result)]))))
|
||||
|
||||
;; --- Create File Media Object (from URL)
|
||||
|
||||
@ -198,6 +228,7 @@
|
||||
[cfg {:keys [url name] :as params}]
|
||||
(let [content (media/download-image cfg url)
|
||||
params (-> params
|
||||
(assoc :from-url? true)
|
||||
(assoc :content content)
|
||||
(assoc :name (d/nilv name "unknown")))]
|
||||
|
||||
@ -305,7 +336,14 @@
|
||||
:hint "chunk index is out of range for this session"
|
||||
:session-id session-id
|
||||
:total-chunks (:total-chunks session)
|
||||
:index index)))
|
||||
:index index))
|
||||
|
||||
|
||||
(l/trc :hint "upload-chunk"
|
||||
:session-id session-id
|
||||
:chunk (str index "/" (:total-chunks session))
|
||||
:size (:size content)
|
||||
:path (:path content)))
|
||||
|
||||
(let [storage (sto/resolve cfg)
|
||||
data (sto/content (:path content))]
|
||||
@ -399,14 +437,15 @@
|
||||
|
||||
(db/tx-run! cfg
|
||||
(fn [{:keys [::db/conn] :as cfg}]
|
||||
(let [{:keys [path size]} (assemble-chunks cfg session-id)
|
||||
content {:filename "upload"
|
||||
:size size
|
||||
:path path
|
||||
:mtype mtype}
|
||||
_ (media/validate-media-type! content)
|
||||
(let [content (assemble-chunks cfg session-id)
|
||||
content (-> content
|
||||
(assoc :filename (str "upload:" name))
|
||||
(assoc :mtype mtype)
|
||||
(media/validate-media-type!)
|
||||
(media/validate-media-size!))
|
||||
mobj (create-file-media-object cfg (assoc params
|
||||
:id (or id (uuid/next))
|
||||
:id id
|
||||
:from-chunks? true
|
||||
:content content))]
|
||||
|
||||
(db/update! conn :file
|
||||
|
||||
@ -264,6 +264,7 @@
|
||||
[cfg {:keys [::rpc/profile-id file] :as params}]
|
||||
;; Validate incoming mime type
|
||||
(media/validate-media-type! file #{"image/jpeg" "image/png" "image/webp"})
|
||||
(media/validate-media-size! file)
|
||||
(update-profile-photo cfg (assoc params :profile-id profile-id)))
|
||||
|
||||
(defn update-profile-photo
|
||||
|
||||
@ -827,6 +827,7 @@
|
||||
;; Validate incoming mime type
|
||||
|
||||
(media/validate-media-type! file #{"image/jpeg" "image/png" "image/webp"})
|
||||
(media/validate-media-size! file)
|
||||
(update-team-photo cfg (assoc params :profile-id profile-id)))
|
||||
|
||||
(defn update-team-photo
|
||||
|
||||
@ -17,7 +17,9 @@
|
||||
[clojure.test :as t]
|
||||
[datoteka.fs :as fs]
|
||||
[datoteka.io :as io]
|
||||
[mockery.core :refer [with-mocks]]))
|
||||
[mockery.core :refer [with-mocks]])
|
||||
(:import
|
||||
java.io.RandomAccessFile))
|
||||
|
||||
(t/use-fixtures :once th/state-init)
|
||||
(t/use-fixtures :each th/database-reset)
|
||||
@ -292,3 +294,499 @@
|
||||
(let [error (:error out)
|
||||
error-data (ex-data error)]
|
||||
(t/is (th/ex-info? error))))))
|
||||
|
||||
;; -----------------------------------------------------------------------
|
||||
;; Helpers for chunked-upload font tests
|
||||
;; -----------------------------------------------------------------------
|
||||
|
||||
(defn- split-bytes-into-chunks
|
||||
"Splits `data` (byte array) into chunks of at most `chunk-size` bytes.
|
||||
Returns a vector of byte arrays."
|
||||
[^bytes data chunk-size]
|
||||
(let [length (alength data)]
|
||||
(loop [offset 0 chunks []]
|
||||
(if (>= offset length)
|
||||
chunks
|
||||
(let [remaining (- length offset)
|
||||
size (min chunk-size remaining)
|
||||
buf (byte-array size)]
|
||||
(System/arraycopy data offset buf 0 size)
|
||||
(recur (+ offset size) (conj chunks buf)))))))
|
||||
|
||||
(defn- make-chunk-mfile
|
||||
"Writes `data` (byte array) to a tempfile and returns a map
|
||||
compatible with the upload-chunk :content parameter."
|
||||
[^bytes data mtype]
|
||||
(let [tmp (fs/create-tempfile :dir "/tmp/penpot" :prefix "test-font-chunk-")]
|
||||
(io/write* tmp data)
|
||||
{:filename "chunk"
|
||||
:path tmp
|
||||
:mtype mtype
|
||||
:size (alength data)}))
|
||||
|
||||
(defn- create-upload-session!
|
||||
"Creates an upload session for `prof` with `total-chunks`. Returns the session-id UUID."
|
||||
[prof total-chunks]
|
||||
(let [out (th/command! {::th/type :create-upload-session
|
||||
::rpc/profile-id (:id prof)
|
||||
:total-chunks total-chunks})]
|
||||
(t/is (nil? (:error out)))
|
||||
(:session-id (:result out))))
|
||||
|
||||
(defn- upload-font-chunked!
|
||||
"Splits `font-bytes` into chunks of `chunk-size` bytes, creates an upload
|
||||
session, uploads all chunks, and returns the session-id UUID."
|
||||
[prof ^bytes font-bytes mtype chunk-size]
|
||||
(let [chunks (split-bytes-into-chunks font-bytes chunk-size)
|
||||
session-id (create-upload-session! prof (count chunks))]
|
||||
(doseq [[idx chunk-data] (map-indexed vector chunks)]
|
||||
(let [mfile (make-chunk-mfile chunk-data mtype)
|
||||
out (th/command! {::th/type :upload-chunk
|
||||
::rpc/profile-id (:id prof)
|
||||
:session-id session-id
|
||||
:index idx
|
||||
:content mfile})]
|
||||
(t/is (nil? (:error out)))))
|
||||
session-id))
|
||||
|
||||
(defn- assert-font-variant-result
|
||||
"Checks that a successful create-font-variant result has valid UUIDs and
|
||||
the expected scalar fields matching `params`."
|
||||
[params result]
|
||||
(t/is (uuid? (:id result)))
|
||||
(t/is (uuid? (:ttf-file-id result)))
|
||||
(t/is (uuid? (:otf-file-id result)))
|
||||
(t/is (uuid? (:woff1-file-id result)))
|
||||
(t/are [k] (= (get params k) (get result k))
|
||||
:team-id
|
||||
:font-id
|
||||
:font-family
|
||||
:font-weight
|
||||
:font-style))
|
||||
|
||||
;; -----------------------------------------------------------------------
|
||||
;; Path 1 – Normal (direct :data bytes)
|
||||
;; -----------------------------------------------------------------------
|
||||
|
||||
(t/deftest create-font-variant-normal-ttf
|
||||
(with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 10)
|
||||
data (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "chunked-test"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:data {"font/ttf" data}}
|
||||
out (th/command! params)]
|
||||
(t/is (= 1 (:call-count @mock)))
|
||||
(t/is (nil? (:error out)))
|
||||
(assert-font-variant-result params (:result out)))))
|
||||
|
||||
(t/deftest create-font-variant-normal-otf
|
||||
(with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 11)
|
||||
data (-> (io/resource "backend_tests/test_files/font-1.otf") (io/read*))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "chunked-test"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:data {"font/otf" data}}
|
||||
out (th/command! params)]
|
||||
(t/is (= 1 (:call-count @mock)))
|
||||
(t/is (nil? (:error out)))
|
||||
(assert-font-variant-result params (:result out)))))
|
||||
|
||||
(t/deftest create-font-variant-normal-woff
|
||||
(with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 12)
|
||||
data (-> (io/resource "backend_tests/test_files/font-1.woff") (io/read*))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "chunked-test"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:data {"font/woff" data}}
|
||||
out (th/command! params)]
|
||||
(t/is (= 1 (:call-count @mock)))
|
||||
(t/is (nil? (:error out)))
|
||||
(assert-font-variant-result params (:result out)))))
|
||||
|
||||
;; -----------------------------------------------------------------------
|
||||
;; Path 2 – Legacy chunking (:data with vector of byte-arrays per mtype)
|
||||
;; -----------------------------------------------------------------------
|
||||
|
||||
(t/deftest create-font-variant-legacy-chunked-ttf
|
||||
"Upload a TTF via the legacy :data path where each mtype value is a
|
||||
vector of byte-array chunks (4 MiB each) instead of a single byte-array."
|
||||
(with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 20)
|
||||
full-bytes (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*))
|
||||
;; Simulate 4 MiB legacy chunks – font is small so a single chunk suffices
|
||||
chunks (split-bytes-into-chunks full-bytes (* 4 1024 1024))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "legacy-chunked"
|
||||
:font-weight 700
|
||||
:font-style "italic"
|
||||
:data {"font/ttf" (vec chunks)}}
|
||||
out (th/command! params)]
|
||||
(t/is (= 1 (:call-count @mock)))
|
||||
(t/is (nil? (:error out)))
|
||||
(assert-font-variant-result params (:result out)))))
|
||||
|
||||
(t/deftest create-font-variant-legacy-chunked-woff
|
||||
"Upload a WOFF via the legacy :data path with multiple sub-4 KiB chunks
|
||||
to exercise the SequenceInputStream concatenation path."
|
||||
(with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 21)
|
||||
full-bytes (-> (io/resource "backend_tests/test_files/font-1.woff") (io/read*))
|
||||
;; Split into small chunks to exercise the SequenceInputStream path
|
||||
chunks (split-bytes-into-chunks full-bytes 512)
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "legacy-chunked-woff"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:data {"font/woff" (vec chunks)}}
|
||||
out (th/command! params)]
|
||||
(t/is (= 1 (:call-count @mock)))
|
||||
(t/is (nil? (:error out)))
|
||||
(assert-font-variant-result params (:result out)))))
|
||||
|
||||
;; -----------------------------------------------------------------------
|
||||
;; Path 3 – New standardized chunked upload (:uploads map)
|
||||
;; -----------------------------------------------------------------------
|
||||
|
||||
(t/deftest create-font-variant-chunked-upload-ttf
|
||||
"Upload a TTF via the new :uploads path (chunked-upload API)."
|
||||
(with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 30)
|
||||
font-bytes (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*))
|
||||
session-id (upload-font-chunked! prof font-bytes "font/ttf" (* 4 1024 1024))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "new-chunked"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:uploads {"font/ttf" session-id}}
|
||||
out (th/command! params)]
|
||||
;; quotes/check! is called at least once (for the font-variant quota) plus
|
||||
;; once during session creation — assert it fired at least once.
|
||||
(t/is (>= (:call-count @mock) 1))
|
||||
(t/is (nil? (:error out)))
|
||||
(assert-font-variant-result params (:result out)))))
|
||||
|
||||
(t/deftest create-font-variant-chunked-upload-otf
|
||||
"Upload an OTF via the new :uploads path."
|
||||
(with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 31)
|
||||
font-bytes (-> (io/resource "backend_tests/test_files/font-1.otf") (io/read*))
|
||||
session-id (upload-font-chunked! prof font-bytes "font/otf" (* 4 1024 1024))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "new-chunked-otf"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:uploads {"font/otf" session-id}}
|
||||
out (th/command! params)]
|
||||
(t/is (>= (:call-count @mock) 1))
|
||||
(t/is (nil? (:error out)))
|
||||
(assert-font-variant-result params (:result out)))))
|
||||
|
||||
(t/deftest create-font-variant-chunked-upload-woff
|
||||
"Upload a WOFF via the new :uploads path."
|
||||
(with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 32)
|
||||
font-bytes (-> (io/resource "backend_tests/test_files/font-1.woff") (io/read*))
|
||||
session-id (upload-font-chunked! prof font-bytes "font/woff" (* 4 1024 1024))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "new-chunked-woff"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:uploads {"font/woff" session-id}}
|
||||
out (th/command! params)]
|
||||
(t/is (>= (:call-count @mock) 1))
|
||||
(t/is (nil? (:error out)))
|
||||
(assert-font-variant-result params (:result out)))))
|
||||
|
||||
(t/deftest create-font-variant-chunked-upload-multi-chunk
|
||||
"Upload a WOFF split into many small chunks to exercise multi-chunk assembly."
|
||||
(with-mocks [mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 33)
|
||||
font-bytes (-> (io/resource "backend_tests/test_files/font-1.woff") (io/read*))
|
||||
;; Use a chunk-size smaller than 4 MiB to force multiple chunks while
|
||||
;; staying within the 20-chunk-per-session quota limit (29836 / 2000 = ~15 chunks).
|
||||
session-id (upload-font-chunked! prof font-bytes "font/woff" 2000)
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "multi-chunk-woff"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:uploads {"font/woff" session-id}}
|
||||
out (th/command! params)]
|
||||
(t/is (>= (:call-count @mock) 1))
|
||||
(t/is (nil? (:error out)))
|
||||
(assert-font-variant-result params (:result out)))))
|
||||
|
||||
;; -----------------------------------------------------------------------
|
||||
;; Error cases
|
||||
;; -----------------------------------------------------------------------
|
||||
|
||||
(t/deftest create-font-variant-missing-data-and-uploads
|
||||
"Neither :data nor :uploads is present — schema validation must reject it."
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 40)
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "bad"
|
||||
:font-weight 400
|
||||
:font-style "normal"}
|
||||
out (th/command! params)]
|
||||
(t/is (some? (:error out)))
|
||||
(t/is (= :validation (-> out :error ex-data :type)))))
|
||||
|
||||
(t/deftest create-font-variant-chunked-upload-missing-chunks
|
||||
"When only some chunks are uploaded the assembly step must fail."
|
||||
(with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 41)
|
||||
font-bytes (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*))
|
||||
;; 5000-byte chunks → 68640/5000 = 14 chunks; declare 15 but only upload 13
|
||||
chunks (split-bytes-into-chunks font-bytes 5000)
|
||||
;; Declare one extra chunk so assembly will fail (not all chunks present)
|
||||
session-id (create-upload-session! prof (inc (count chunks)))]
|
||||
|
||||
;; Upload all real chunks except the last one (omit it so the session is incomplete)
|
||||
(doseq [[idx chunk-data] (map-indexed vector (butlast chunks))]
|
||||
(let [mfile (make-chunk-mfile chunk-data "font/ttf")
|
||||
out (th/command! {::th/type :upload-chunk
|
||||
::rpc/profile-id (:id prof)
|
||||
:session-id session-id
|
||||
:index idx
|
||||
:content mfile})]
|
||||
(t/is (nil? (:error out)))))
|
||||
|
||||
(let [out (th/command! {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "missing-chunks"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:uploads {"font/ttf" session-id}})]
|
||||
(t/is (some? (:error out)))))))
|
||||
|
||||
(t/deftest create-font-variant-chunked-upload-invalid-session
|
||||
"Passing a non-existent session-id must fail at assembly time."
|
||||
(with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 42)
|
||||
out (th/command! {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "bad-session"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:uploads {"font/ttf" (uuid/next)}})]
|
||||
(t/is (some? (:error out))))))
|
||||
|
||||
;; -----------------------------------------------------------------------
|
||||
;; Font size validation tests
|
||||
;; -----------------------------------------------------------------------
|
||||
|
||||
(t/deftest create-font-variant-size-exceeded-normal
|
||||
"Direct :data upload exceeding font-max-file-size must be rejected."
|
||||
(with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(with-redefs [app.config/config (assoc app.config/config :font-max-file-size 1)]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 50)
|
||||
data (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "size-exceeded"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:data {"font/ttf" data}}
|
||||
out (th/command! params)]
|
||||
(t/is (some? (:error out)))
|
||||
(t/is (= :restriction (-> out :error ex-data :type)))
|
||||
(t/is (= :font-max-file-size-reached (-> out :error ex-data :code)))))))
|
||||
|
||||
(t/deftest create-font-variant-size-exceeded-legacy-chunked
|
||||
"Legacy :data chunk-vector upload exceeding font-max-file-size must be rejected."
|
||||
(with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(with-redefs [app.config/config (assoc app.config/config :font-max-file-size 1)]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 51)
|
||||
full-bytes (-> (io/resource "backend_tests/test_files/font-1.woff") (io/read*))
|
||||
chunks (split-bytes-into-chunks full-bytes (* 4 1024 1024))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "size-exceeded-legacy"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:data {"font/woff" (vec chunks)}}
|
||||
out (th/command! params)]
|
||||
(t/is (some? (:error out)))
|
||||
(t/is (= :restriction (-> out :error ex-data :type)))
|
||||
(t/is (= :font-max-file-size-reached (-> out :error ex-data :code)))))))
|
||||
|
||||
(t/deftest create-font-variant-size-exceeded-chunked-upload
|
||||
"New :uploads path exceeding font-max-file-size must be rejected after assembly."
|
||||
(with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 52)
|
||||
font-bytes (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*))
|
||||
session-id (upload-font-chunked! prof font-bytes "font/ttf" (* 4 1024 1024))]
|
||||
(with-redefs [app.config/config (assoc app.config/config :font-max-file-size 1)]
|
||||
(let [out (th/command! {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "size-exceeded-chunked"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:uploads {"font/ttf" session-id}})]
|
||||
(t/is (some? (:error out)))
|
||||
(t/is (= :restriction (-> out :error ex-data :type)))
|
||||
(t/is (= :font-max-file-size-reached (-> out :error ex-data :code))))))))
|
||||
|
||||
(t/deftest create-font-variant-size-within-limit
|
||||
"Upload exactly at the limit must succeed."
|
||||
(with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 53)
|
||||
font-bytes (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*))
|
||||
font-size (alength ^bytes font-bytes)]
|
||||
(with-redefs [app.config/config (assoc app.config/config :font-max-file-size font-size)]
|
||||
(let [params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "size-at-limit"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:data {"font/ttf" font-bytes}}
|
||||
out (th/command! params)]
|
||||
(t/is (nil? (:error out)))
|
||||
(assert-font-variant-result params (:result out)))))))
|
||||
|
||||
;; -----------------------------------------------------------------------
|
||||
;; Font media-type validation tests
|
||||
;; -----------------------------------------------------------------------
|
||||
|
||||
(t/deftest create-font-variant-invalid-type-normal
|
||||
"Direct :data upload with a disallowed mtype must be rejected."
|
||||
(with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 60)
|
||||
data (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "invalid-type"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:data {"application/octet-stream" data}}
|
||||
out (th/command! params)]
|
||||
(t/is (some? (:error out)))
|
||||
(t/is (= :validation (-> out :error ex-data :type)))
|
||||
(t/is (= :media-type-not-allowed (-> out :error ex-data :code))))))
|
||||
|
||||
(t/deftest create-font-variant-invalid-type-legacy-chunked
|
||||
"Legacy :data chunk-vector upload with a disallowed mtype must be rejected."
|
||||
(with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 61)
|
||||
full-bytes (-> (io/resource "backend_tests/test_files/font-1.woff") (io/read*))
|
||||
chunks (split-bytes-into-chunks full-bytes (* 4 1024 1024))
|
||||
params {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "invalid-type-legacy"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:data {"image/png" (vec chunks)}}
|
||||
out (th/command! params)]
|
||||
(t/is (some? (:error out)))
|
||||
(t/is (= :validation (-> out :error ex-data :type)))
|
||||
(t/is (= :media-type-not-allowed (-> out :error ex-data :code))))))
|
||||
|
||||
(t/deftest create-font-variant-invalid-type-chunked-upload
|
||||
"New :uploads path with a disallowed mtype must be rejected after assembly."
|
||||
(with-mocks [_mock {:target 'app.rpc.quotes/check! :return nil}]
|
||||
(let [prof (th/create-profile* 1 {:is-active true})
|
||||
team-id (:default-team-id prof)
|
||||
font-id (uuid/custom 10 62)
|
||||
font-bytes (-> (io/resource "backend_tests/test_files/font-1.ttf") (io/read*))
|
||||
;; Upload the bytes under a valid session but lie about the mtype
|
||||
;; when calling create-font-variant.
|
||||
session-id (upload-font-chunked! prof font-bytes "font/ttf" (* 4 1024 1024))
|
||||
out (th/command! {::th/type :create-font-variant
|
||||
::rpc/profile-id (:id prof)
|
||||
:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family "invalid-type-chunked"
|
||||
:font-weight 400
|
||||
:font-style "normal"
|
||||
:uploads {"image/jpeg" session-id}})]
|
||||
(t/is (some? (:error out)))
|
||||
(t/is (= :validation (-> out :error ex-data :type)))
|
||||
(t/is (= :media-type-not-allowed (-> out :error ex-data :code))))))
|
||||
|
||||
@ -12,8 +12,8 @@
|
||||
(def font-types
|
||||
#{"font/ttf"
|
||||
"font/woff"
|
||||
"font/otf"
|
||||
"font/opentype"})
|
||||
"font/woff2"
|
||||
"font/otf"})
|
||||
|
||||
(def image-types
|
||||
#{"image/jpeg"
|
||||
|
||||
@ -14,6 +14,7 @@
|
||||
[app.common.uuid :as uuid]
|
||||
[app.main.data.event :as ev]
|
||||
[app.main.data.notifications :as ntf]
|
||||
[app.main.data.uploads :as uploads]
|
||||
[app.main.fonts :as fonts]
|
||||
[app.main.repo :as rp]
|
||||
[app.main.store :as st]
|
||||
@ -24,24 +25,14 @@
|
||||
[cuerdas.core :as str]
|
||||
[potok.v2.core :as ptk]))
|
||||
|
||||
(def ^:const default-chunk-size
|
||||
(* 1024 1024 4)) ;; 4MiB
|
||||
|
||||
(defn- chunk-array
|
||||
[data chunk-size]
|
||||
(let [total-size (alength data)]
|
||||
(loop [offset 0
|
||||
chunks []]
|
||||
(if (< offset total-size)
|
||||
(let [end (min (+ offset chunk-size) total-size)
|
||||
chunk (.subarray ^js data offset end)]
|
||||
(recur end (conj chunks chunk)))
|
||||
chunks))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; General purpose events & IMPL
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
||||
(def ^:private font-upload-chunk-size
|
||||
"Size in bytes of each chunk when uploading font files (10 MiB)."
|
||||
(* 1024 1024 10))
|
||||
|
||||
(defn fonts-fetched
|
||||
[fonts]
|
||||
(letfn [;; Prepare font to the internal font database format.
|
||||
@ -94,9 +85,44 @@
|
||||
(->> (rp/cmd! :get-font-variants {:team-id team-id})
|
||||
(rx/map fonts-fetched)))))
|
||||
|
||||
(defn upload-font-variant
|
||||
"Uploads a single font variant item using the chunked upload API.
|
||||
|
||||
For each mime-type in `data`, creates a Blob and uploads it via the
|
||||
session-based chunked upload. Once all sessions are created, calls
|
||||
`create-font-variant` with the resulting `:uploads` map so the server
|
||||
can assemble the chunks and materialise the final font-variant record.
|
||||
|
||||
Returns an observable that emits the created font-variant."
|
||||
[{:keys [data team-id font-id font-family font-weight font-style] :as _item}]
|
||||
;; Upload each mtype as a separate chunked session in parallel, collect
|
||||
;; all [mtype session-id] pairs, then call create-font-variant with :uploads.
|
||||
(->> (rx/from (seq data))
|
||||
(rx/mapcat (fn [[mtype buffer]]
|
||||
(let [blob (js/Blob. #js [buffer] #js {:type mtype})]
|
||||
(->> (uploads/upload-blob-chunked blob :chunk-size font-upload-chunk-size)
|
||||
(rx/map (fn [{:keys [session-id]}]
|
||||
[mtype session-id]))))))
|
||||
(rx/reduce (fn [acc [mtype session-id]]
|
||||
(assoc acc mtype session-id))
|
||||
{})
|
||||
(rx/mapcat (fn [uploads]
|
||||
(rp/cmd! :create-font-variant
|
||||
{:team-id team-id
|
||||
:font-id font-id
|
||||
:font-family font-family
|
||||
:font-weight font-weight
|
||||
:font-style font-style
|
||||
:uploads uploads})))))
|
||||
|
||||
(defn process-upload
|
||||
"Given a seq of blobs and the team id, creates a ready-to-use fonts
|
||||
map with temporal ID's associated to each font entry."
|
||||
map with temporal ID's associated to each font entry.
|
||||
|
||||
Each font entry's `:data` is a map of `{mtype -> ArrayBuffer}`. The
|
||||
raw `ArrayBuffer` is kept as-is so that `upload-font-variant` can
|
||||
wrap it in a `Blob` and hand it directly to `upload-blob-chunked`
|
||||
without any intermediate client-side chunking."
|
||||
[blobs team-id]
|
||||
(letfn [(prepare [{:keys [font type name data] :as params}]
|
||||
(let [family (or (.getEnglishName ^js font "preferredFamily")
|
||||
@ -130,9 +156,8 @@
|
||||
(not= hhea-descender win-descent)
|
||||
(and f-selection (or
|
||||
(not= hhea-ascender os2-ascent)
|
||||
(not= hhea-descender os2-descent))))
|
||||
data (js/Uint8Array. data)]
|
||||
{:content {:data (chunk-array data default-chunk-size)
|
||||
(not= hhea-descender os2-descent))))]
|
||||
{:content {:data data
|
||||
:name name
|
||||
:type type}
|
||||
:font-family (or family "")
|
||||
|
||||
@ -24,10 +24,6 @@
|
||||
[app.main.repo :as rp]
|
||||
[beicon.v2.core :as rx]))
|
||||
|
||||
;; Size of each upload chunk in bytes. Reads the penpotUploadChunkSize global
|
||||
;; variable at startup; defaults to 25 MiB (overridden in production).
|
||||
(def ^:private chunk-size cf/upload-chunk-size)
|
||||
|
||||
(def ^:private max-parallel-chunk-uploads
|
||||
"Maximum number of chunk upload requests that may be in-flight at the
|
||||
same time within a single chunked upload session."
|
||||
@ -44,8 +40,11 @@
|
||||
Returns an observable that emits exactly one map:
|
||||
`{:session-id <uuid>}`
|
||||
|
||||
The caller is responsible for the final step (assemble / import)."
|
||||
[blob]
|
||||
The caller is responsible for the final step (assemble / import).
|
||||
|
||||
The optional `opts` map accepts:
|
||||
`:chunk-size` – size in bytes of each chunk (default: `cf/upload-chunk-size`, 25 MiB)."
|
||||
[blob & {:keys [chunk-size] :or {chunk-size cf/upload-chunk-size}}]
|
||||
(let [total-size (.-size blob)
|
||||
total-chunks (js/Math.ceil (/ total-size chunk-size))]
|
||||
(->> (rp/cmd! :create-upload-session
|
||||
|
||||
@ -14,7 +14,6 @@
|
||||
[app.main.data.fonts :as df]
|
||||
[app.main.data.modal :as modal]
|
||||
[app.main.data.notifications :as ntf]
|
||||
[app.main.repo :as rp]
|
||||
[app.main.store :as st]
|
||||
[app.main.ui.components.context-menu-a11y :refer [context-menu*]]
|
||||
[app.main.ui.components.file-uploader :refer [file-uploader]]
|
||||
@ -110,7 +109,7 @@
|
||||
(mf/use-fn
|
||||
(fn [{:keys [id] :as item}]
|
||||
(swap! uploading* conj id)
|
||||
(->> (rp/cmd! :create-font-variant item)
|
||||
(->> (df/upload-font-variant item)
|
||||
(rx/delay-at-least 2000)
|
||||
(rx/subs! (fn [font]
|
||||
(swap! fonts* dissoc id)
|
||||
|
||||
208
scripts/check-commit
Executable file
208
scripts/check-commit
Executable file
@ -0,0 +1,208 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Check commit messages against Penpot's commit guidelines.
|
||||
|
||||
Validates commit messages using the rules defined in:
|
||||
- .github/workflows/commit-checker.yml (regex pattern)
|
||||
- CONTRIBUTING.md (formatting rules, subject length, DCO)
|
||||
|
||||
By default, checks HEAD. Use --commit to specify a different commit.
|
||||
|
||||
Usage:
|
||||
./scripts/check-commit
|
||||
./scripts/check-commit --commit HEAD~1
|
||||
./scripts/check-commit -c abc1234
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
# ── Emoji list ───────────────────────────────────────────────────────────────
|
||||
# Combined from commit-checker.yml AND CONTRIBUTING.md
|
||||
VALID_EMOJIS = (
|
||||
"lipstick|globe_with_meridians|wrench|books|"
|
||||
"arrow_up|arrow_down|zap|ambulance|construction|"
|
||||
"boom|fire|whale|bug|sparkles|paperclip|tada|"
|
||||
"recycle|rewind|construction_worker|rocket"
|
||||
)
|
||||
|
||||
# ── Regex from .github/workflows/commit-checker.yml ──────────────────────────
|
||||
# Matches:
|
||||
# 1) ":emoji: <Capitalized subject without trailing dot>"
|
||||
# 2) "Merge|Revert|Reapply ... without trailing dot"
|
||||
COMMIT_PATTERN = re.compile(
|
||||
r"^((:(" + VALID_EMOJIS + r"):\s[A-Z].*[^.]))$"
|
||||
)
|
||||
|
||||
MERGE_PATTERN = re.compile(r"^(Merge|Revert|Reapply).+[^.]$")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# Helpers
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def run_git(args):
|
||||
"""Run a git command and return (returncode, stdout, stderr)."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git"] + args,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
return result.returncode, result.stdout, result.stderr
|
||||
except FileNotFoundError:
|
||||
print("ERROR: git not found. Is it installed?", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def get_commit_message(commit_ref):
|
||||
"""Return the full commit message for *commit_ref*."""
|
||||
rc, out, err = run_git(["log", "--format=%B", "-n", "1", commit_ref])
|
||||
if rc != 0:
|
||||
print(f"ERROR: could not read commit {commit_ref}: {err.strip()}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not out.strip():
|
||||
print(f"ERROR: commit {commit_ref} has no message", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return out.rstrip("\n")
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# Validators
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def check_regex(message):
|
||||
"""Check the commit message against the CI regex pattern."""
|
||||
# Normalise: strip trailing newlines for single-line matching
|
||||
first_line = message.split("\n")[0]
|
||||
|
||||
if MERGE_PATTERN.match(first_line):
|
||||
return True, None
|
||||
|
||||
if COMMIT_PATTERN.match(first_line):
|
||||
return True, None
|
||||
|
||||
return False, (
|
||||
"Commit subject must match one of:\n"
|
||||
" :emoji: <Capitalized subject without trailing dot>\n"
|
||||
" Merge|Revert|Reapply <rest without trailing dot>\n"
|
||||
f"Got: {first_line!r}"
|
||||
)
|
||||
|
||||
|
||||
def check_subject_length(message):
|
||||
"""Subject line must be ≤ 90 characters."""
|
||||
first_line = message.split("\n")[0]
|
||||
if len(first_line) > 90:
|
||||
return False, (
|
||||
f"Subject line exceeds 90 characters ({len(first_line)} chars):\n"
|
||||
f" {first_line}"
|
||||
)
|
||||
return True, None
|
||||
|
||||
|
||||
def check_subject_no_trailing_dot(message):
|
||||
"""Subject line must not end with a period ('.')."""
|
||||
first_line = message.split("\n")[0]
|
||||
if first_line.endswith("."):
|
||||
return False, (
|
||||
"Subject line must not end with a period:\n"
|
||||
f" {first_line}"
|
||||
)
|
||||
return True, None
|
||||
|
||||
|
||||
def check_subject_capitalized(message):
|
||||
"""Subject must be capitalized, but only if it's a regular commit (not Merge/Revert/Reapply)."""
|
||||
first_line = message.split("\n")[0]
|
||||
|
||||
# Skip check for Merge/Revert/Reapply commits
|
||||
if MERGE_PATTERN.match(first_line):
|
||||
return True, None
|
||||
|
||||
# Strip emoji prefix before checking capitalization
|
||||
emoji_match = re.match(r"^:([a-z_]+):\s+(.*)", first_line)
|
||||
if emoji_match:
|
||||
rest = emoji_match.group(2)
|
||||
else:
|
||||
rest = first_line
|
||||
|
||||
if rest and not rest[0].isupper():
|
||||
return False, (
|
||||
"Subject line must start with a capital letter "
|
||||
"(after the emoji prefix):\n"
|
||||
f" {first_line}"
|
||||
)
|
||||
return True, None
|
||||
|
||||
|
||||
def check_body_blank_line(message):
|
||||
"""If a body exists, there must be a blank line between subject and body."""
|
||||
lines = message.split("\n")
|
||||
if len(lines) >= 3 and lines[1] != "":
|
||||
return False, (
|
||||
"A blank line must separate the subject from the body."
|
||||
)
|
||||
return True, None
|
||||
|
||||
|
||||
def check_signed_off_by(message):
|
||||
"""Check for the DCO Signed-off-by line (required for code changes)."""
|
||||
if "Signed-off-by:" not in message:
|
||||
return False, (
|
||||
"Missing 'Signed-off-by:' line in the commit footer.\n"
|
||||
" Add it with 'git commit -s' or append it manually:\n"
|
||||
" Signed-off-by: Your Real Name <your.email@example.com>"
|
||||
)
|
||||
return True, None
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Check a commit message against Penpot commit guidelines."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-c", "--commit",
|
||||
default="HEAD",
|
||||
help="Commit to check (default: HEAD)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
commit_ref = args.commit
|
||||
message = get_commit_message(commit_ref)
|
||||
|
||||
print(f"Checking commit {commit_ref} ...\n")
|
||||
|
||||
validators = [
|
||||
("Regex pattern", check_regex),
|
||||
("Subject ≤ 90 chars", check_subject_length),
|
||||
("No trailing period in subject", check_subject_no_trailing_dot),
|
||||
("Subject capitalized", check_subject_capitalized),
|
||||
("Blank line after subject", check_body_blank_line),
|
||||
]
|
||||
|
||||
all_ok = True
|
||||
|
||||
for name, validator in validators:
|
||||
ok, error_msg = validator(message)
|
||||
status = "✓" if ok else "✗"
|
||||
print(f" [{status}] {name}")
|
||||
if not ok:
|
||||
all_ok = False
|
||||
print(f" {error_msg}", file=sys.stderr)
|
||||
|
||||
print()
|
||||
if all_ok:
|
||||
print("All checks passed.")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("Some checks FAILED. See messages above.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user