mirror of
https://github.com/penpot/penpot.git
synced 2026-04-25 11:18:36 +00:00
e5f9c1e863
Introduce a purpose-agnostic three-step session-based upload API that
allows uploading large binary blobs (media files and .penpot imports)
without hitting multipart size limits.
Backend:
- Migration 0147: new `upload_session` table (profile_id, total_chunks,
created_at) with indexes on profile_id and created_at.
- Three new RPC commands in media.clj:
* `create-upload-session` – allocates a session row; enforces
`upload-sessions-per-profile` and `upload-chunks-per-session`
quota limits (configurable in config.clj, defaults 5 / 20).
* `upload-chunk` – stores each slice as a storage object;
validates chunk index bounds and profile ownership.
* `assemble-file-media-object` – reassembles chunks via the shared
`assemble-chunks!` helper and creates the final media object.
- `assemble-chunks!` is a public helper in media.clj shared by both
`assemble-file-media-object` and `import-binfile`.
- `import-binfile` (binfile.clj): accepts an optional `upload-id` param;
when provided, materialises the temp file from chunks instead of
expecting an inline multipart body, removing the 200 MiB body limit
on .penpot imports. Schema updated with an `:and` validator requiring
either `:file` or `:upload-id`.
- quotes.clj: new `upload-sessions-per-profile` quota check.
- Background GC task (`tasks/upload_session_gc.clj`): deletes stalled
(never-completed) sessions older than 1 hour; scheduled daily at
midnight via the cron system in main.clj.
- backend/AGENTS.md: document the background-task wiring pattern.
Frontend:
- New `app.main.data.uploads` namespace: generic `upload-blob-chunked`
helper drives steps 1–2 (create session + upload all chunks with a
concurrency cap of 2) and emits `{:session-id uuid}` for callers.
- `config.cljs`: expose `upload-chunk-size` (default 25 MiB, overridable
via `penpotUploadChunkSize` global).
- `workspace/media.cljs`: blobs ≥ chunk-size go through the chunked path
(`upload-blob-chunked` → `assemble-file-media-object`); smaller blobs
use the existing direct `upload-file-media-object` path.
`handle-media-error` simplified; `on-error` callback removed.
- `worker/import.cljs`: new `import-blob-via-upload` helper replaces the
inline multipart approach for both binfile-v1 and binfile-v3 imports.
- `repo.cljs`: `:upload-chunk` derived as a `::multipart-upload`;
`form-data?` removed from `import-binfile` (JSON params only).
Tests:
- Backend (rpc_media_test.clj): happy path, idempotency, permission
isolation, invalid media type, missing chunks, session-not-found,
chunk-index out-of-range, and quota-limit scenarios.
- Frontend (uploads_test.cljs): session creation and chunk-count
correctness for `upload-blob-chunked`.
- Frontend (workspace_media_test.cljs): direct-upload path for small
blobs, chunked path for large blobs, and chunk-count correctness for
`process-blobs`.
- `helpers/http.cljs`: shared fetch-mock helpers (`install-fetch-mock!`,
`make-json-response`, `make-transit-response`, `url->cmd`).
Signed-off-by: Andrey Antukh <niwi@niwi.nz>
190 lines
6.9 KiB
Clojure
190 lines
6.9 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS INC
|
|
|
|
(ns frontend-tests.data.workspace-media-test
|
|
"Integration tests for the chunked-upload logic in
|
|
app.main.data.workspace.media."
|
|
(:require
|
|
[app.common.uuid :as uuid]
|
|
[app.config :as cf]
|
|
[app.main.data.workspace.media :as media]
|
|
[beicon.v2.core :as rx]
|
|
[cljs.test :as t :include-macros true]
|
|
[frontend-tests.helpers.http :as http]))
|
|
|
|
;; ---------------------------------------------------------------------------
|
|
;; Local helpers
|
|
;; ---------------------------------------------------------------------------
|
|
|
|
(defn- make-blob
|
|
"Creates a JS Blob of exactly `size` bytes with the given `mtype`."
|
|
[size mtype]
|
|
(let [buf (js/Uint8Array. size)]
|
|
(js/Blob. #js [buf] #js {:type mtype})))
|
|
|
|
;; ---------------------------------------------------------------------------
|
|
;; Small-file path: direct upload (no chunking)
|
|
;; ---------------------------------------------------------------------------
|
|
|
|
(t/deftest small-file-uses-direct-upload
|
|
(t/testing "blobs below chunk-size use :upload-file-media-object directly"
|
|
(t/async done
|
|
(let [file-id (uuid/next)
|
|
;; One byte below the threshold so the blob takes the direct path
|
|
blob-size (dec cf/upload-chunk-size)
|
|
blob (make-blob blob-size "image/jpeg")
|
|
calls (atom [])
|
|
|
|
fetch-mock
|
|
(fn [url _opts]
|
|
(let [cmd (http/url->cmd url)]
|
|
(swap! calls conj cmd)
|
|
(js/Promise.resolve
|
|
(http/make-json-response
|
|
{:id (str (uuid/next))
|
|
:name "img"
|
|
:width 100
|
|
:height 100
|
|
:mtype "image/jpeg"
|
|
:file-id (str file-id)}))))
|
|
|
|
orig (http/install-fetch-mock! fetch-mock)]
|
|
|
|
(->> (media/process-blobs
|
|
{:file-id file-id
|
|
:local? true
|
|
:blobs [blob]
|
|
:on-image (fn [_] nil)
|
|
:on-svg (fn [_] nil)})
|
|
(rx/subs!
|
|
(fn [_] nil)
|
|
(fn [err]
|
|
(t/is false (str "unexpected error: " (ex-message err)))
|
|
(done))
|
|
(fn []
|
|
(http/restore-fetch! orig)
|
|
;; Should call :upload-file-media-object, NOT the chunked API
|
|
(t/is (= 1 (count @calls)))
|
|
(t/is (= :upload-file-media-object (first @calls)))
|
|
(done))))))))
|
|
|
|
;; ---------------------------------------------------------------------------
|
|
;; Large-file path: chunked upload via uploads namespace
|
|
;; ---------------------------------------------------------------------------
|
|
|
|
(t/deftest large-file-uses-chunked-upload
|
|
(t/testing "blobs at or above chunk-size use the three-step session API"
|
|
(t/async done
|
|
(let [file-id (uuid/next)
|
|
session-id (uuid/next)
|
|
chunk-size cf/upload-chunk-size
|
|
;; Exactly two full chunks
|
|
blob-size (* 2 chunk-size)
|
|
blob (make-blob blob-size "image/jpeg")
|
|
calls (atom [])
|
|
|
|
fetch-mock
|
|
(fn [url _opts]
|
|
(let [cmd (http/url->cmd url)]
|
|
(swap! calls conj cmd)
|
|
(js/Promise.resolve
|
|
(http/make-json-response
|
|
(case cmd
|
|
:create-upload-session
|
|
{:session-id (str session-id)}
|
|
|
|
:upload-chunk
|
|
{:session-id (str session-id) :index 0}
|
|
|
|
:assemble-file-media-object
|
|
{:id (str (uuid/next))
|
|
:name "img"
|
|
:width 100
|
|
:height 100
|
|
:mtype "image/jpeg"
|
|
:file-id (str file-id)}
|
|
|
|
;; Default: return an error response
|
|
{:error (str "unexpected cmd: " cmd)})))))
|
|
|
|
orig (http/install-fetch-mock! fetch-mock)]
|
|
|
|
(->> (media/process-blobs
|
|
{:file-id file-id
|
|
:local? true
|
|
:blobs [blob]
|
|
:on-image (fn [_] nil)
|
|
:on-svg (fn [_] nil)})
|
|
(rx/subs!
|
|
(fn [_] nil)
|
|
(fn [err]
|
|
(t/is false (str "unexpected error: " (ex-message err)))
|
|
(done))
|
|
(fn []
|
|
(http/restore-fetch! orig)
|
|
(let [cmd-seq @calls]
|
|
;; First call must create the session
|
|
(t/is (= :create-upload-session (first cmd-seq)))
|
|
;; Two chunk uploads
|
|
(t/is (= 2 (count (filter #(= :upload-chunk %) cmd-seq))))
|
|
;; Last call must assemble
|
|
(t/is (= :assemble-file-media-object (last cmd-seq)))
|
|
;; Direct upload must NOT be called
|
|
(t/is (not (some #(= :upload-file-media-object %) cmd-seq))))
|
|
(done))))))))
|
|
|
|
(t/deftest chunked-upload-chunk-count-matches-blob
|
|
(t/testing "number of chunk upload calls equals ceil(blob-size / chunk-size)"
|
|
(t/async done
|
|
(let [file-id (uuid/next)
|
|
session-id (uuid/next)
|
|
chunk-size cf/upload-chunk-size
|
|
;; Three chunks: 2 full + 1 partial
|
|
blob-size (+ (* 2 chunk-size) 1)
|
|
blob (make-blob blob-size "image/jpeg")
|
|
chunk-calls (atom 0)
|
|
|
|
fetch-mock
|
|
(fn [url _opts]
|
|
(let [cmd (http/url->cmd url)]
|
|
(js/Promise.resolve
|
|
(http/make-json-response
|
|
(case cmd
|
|
:create-upload-session
|
|
{:session-id (str session-id)}
|
|
|
|
:upload-chunk
|
|
(do (swap! chunk-calls inc)
|
|
{:session-id (str session-id) :index 0})
|
|
|
|
:assemble-file-media-object
|
|
{:id (str (uuid/next))
|
|
:name "img"
|
|
:width 100
|
|
:height 100
|
|
:mtype "image/jpeg"
|
|
:file-id (str file-id)}
|
|
|
|
{:error (str "unexpected cmd: " cmd)})))))
|
|
|
|
orig (http/install-fetch-mock! fetch-mock)]
|
|
|
|
(->> (media/process-blobs
|
|
{:file-id file-id
|
|
:local? true
|
|
:blobs [blob]
|
|
:on-image (fn [_] nil)
|
|
:on-svg (fn [_] nil)})
|
|
(rx/subs!
|
|
(fn [_] nil)
|
|
(fn [err]
|
|
(t/is false (str "unexpected error: " (ex-message err)))
|
|
(done))
|
|
(fn []
|
|
(http/restore-fetch! orig)
|
|
(t/is (= 3 @chunk-calls))
|
|
(done))))))))
|