mirror of
https://github.com/penpot/penpot.git
synced 2026-04-25 11:18:36 +00:00
6fa440cf92
Introduce a purpose-agnostic three-step session-based upload API that
allows uploading large binary blobs (media files and .penpot imports)
without hitting multipart size limits.
Backend:
- Migration 0147: new `upload_session` table (profile_id, total_chunks,
created_at) with indexes on profile_id and created_at.
- Three new RPC commands in media.clj:
* `create-upload-session` – allocates a session row; enforces
`upload-sessions-per-profile` and `upload-chunks-per-session`
quota limits (configurable in config.clj, defaults 5 / 20).
* `upload-chunk` – stores each slice as a storage object;
validates chunk index bounds and profile ownership.
* `assemble-file-media-object` – reassembles chunks via the shared
`assemble-chunks!` helper and creates the final media object.
- `assemble-chunks!` is a public helper in media.clj shared by both
`assemble-file-media-object` and `import-binfile`.
- `import-binfile` (binfile.clj): accepts an optional `upload-id` param;
when provided, materialises the temp file from chunks instead of
expecting an inline multipart body, removing the 200 MiB body limit
on .penpot imports. Schema updated with an `:and` validator requiring
either `:file` or `:upload-id`.
- quotes.clj: new `upload-sessions-per-profile` quota check.
- Background GC task (`tasks/upload_session_gc.clj`): deletes stalled
(never-completed) sessions older than 1 hour; scheduled daily at
midnight via the cron system in main.clj.
- backend/AGENTS.md: document the background-task wiring pattern.
Frontend:
- New `app.main.data.uploads` namespace: generic `upload-blob-chunked`
helper drives steps 1–2 (create session + upload all chunks with a
concurrency cap of 2) and emits `{:session-id uuid}` for callers.
- `config.cljs`: expose `upload-chunk-size` (default 25 MiB, overridable
via `penpotUploadChunkSize` global).
- `workspace/media.cljs`: blobs ≥ chunk-size go through the chunked path
(`upload-blob-chunked` → `assemble-file-media-object`); smaller blobs
use the existing direct `upload-file-media-object` path.
`handle-media-error` simplified; `on-error` callback removed.
- `worker/import.cljs`: new `import-blob-via-upload` helper replaces the
inline multipart approach for both binfile-v1 and binfile-v3 imports.
- `repo.cljs`: `:upload-chunk` derived as a `::multipart-upload`;
`form-data?` removed from `import-binfile` (JSON params only).
Tests:
- Backend (rpc_media_test.clj): happy path, idempotency, permission
isolation, invalid media type, missing chunks, session-not-found,
chunk-index out-of-range, and quota-limit scenarios.
- Frontend (uploads_test.cljs): session creation and chunk-count
correctness for `upload-blob-chunked`.
- Frontend (workspace_media_test.cljs): direct-upload path for small
blobs, chunked path for large blobs, and chunk-count correctness for
`process-blobs`.
- `helpers/http.cljs`: shared fetch-mock helpers (`install-fetch-mock!`,
`make-json-response`, `make-transit-response`, `url->cmd`).
Signed-off-by: Andrey Antukh <niwi@niwi.nz>
704 lines
30 KiB
Clojure
704 lines
30 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS INC
|
|
|
|
(ns backend-tests.rpc-media-test
|
|
(:require
|
|
[app.common.uuid :as uuid]
|
|
[app.http.client :as http]
|
|
[app.media :as media]
|
|
[app.rpc :as-alias rpc]
|
|
[app.storage :as sto]
|
|
[backend-tests.helpers :as th]
|
|
[clojure.test :as t]
|
|
[datoteka.fs :as fs]
|
|
[datoteka.io :as io]
|
|
[mockery.core :refer [with-mocks]])
|
|
(:import
|
|
java.io.RandomAccessFile))
|
|
|
|
(t/use-fixtures :once th/state-init)
|
|
(t/use-fixtures :each th/database-reset)
|
|
|
|
(t/deftest media-object-from-url
|
|
(let [prof (th/create-profile* 1)
|
|
proj (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
url "https://raw.githubusercontent.com/uxbox/uxbox/develop/sample_media/images/unsplash/anna-pelzer.jpg"
|
|
params {::th/type :create-file-media-object-from-url
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:url url}
|
|
out (th/command! params)]
|
|
|
|
;; (th/print-result! out)
|
|
(t/is (nil? (:error out)))
|
|
(let [{:keys [media-id thumbnail-id] :as result} (:result out)]
|
|
(t/is (= (:id file) (:file-id result)))
|
|
(t/is (= 1024 (:width result)))
|
|
(t/is (= 683 (:height result)))
|
|
(t/is (= "image/jpeg" (:mtype result)))
|
|
(t/is (uuid? media-id))
|
|
(t/is (uuid? thumbnail-id))
|
|
(let [storage (:app.storage/storage th/*system*)
|
|
mobj1 (sto/get-object storage media-id)
|
|
mobj2 (sto/get-object storage thumbnail-id)]
|
|
(t/is (sto/object? mobj1))
|
|
(t/is (sto/object? mobj2))
|
|
(t/is (= 122785 (:size mobj1)))
|
|
(t/is (= 3297 (:size mobj2)))))))
|
|
|
|
(t/deftest media-object-upload
|
|
(let [prof (th/create-profile* 1)
|
|
proj (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
mfile {:filename "sample.jpg"
|
|
:path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
|
|
params {::th/type :upload-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "testfile"
|
|
:content mfile}
|
|
out (th/command! params)]
|
|
|
|
;; (th/print-result! out)
|
|
(t/is (nil? (:error out)))
|
|
(let [{:keys [media-id thumbnail-id] :as result} (:result out)]
|
|
(t/is (= (:id file) (:file-id result)))
|
|
(t/is (= 800 (:width result)))
|
|
(t/is (= 800 (:height result)))
|
|
(t/is (= "image/jpeg" (:mtype result)))
|
|
(t/is (uuid? media-id))
|
|
(t/is (uuid? thumbnail-id))
|
|
(let [storage (:app.storage/storage th/*system*)
|
|
mobj1 (sto/get-object storage media-id)
|
|
mobj2 (sto/get-object storage thumbnail-id)]
|
|
(t/is (sto/object? mobj1))
|
|
(t/is (sto/object? mobj2))
|
|
(t/is (= 312043 (:size mobj1)))
|
|
(t/is (= 3890 (:size mobj2)))))))
|
|
|
|
|
|
(t/deftest media-object-upload-idempotency
|
|
(let [prof (th/create-profile* 1)
|
|
proj (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
mfile {:filename "sample.jpg"
|
|
:path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
|
|
params {::th/type :upload-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "testfile"
|
|
:content mfile
|
|
:id (uuid/next)}]
|
|
|
|
;; First try
|
|
(let [{:keys [result error] :as out} (th/command! params)]
|
|
;; (th/print-result! out)
|
|
(t/is (nil? error))
|
|
(t/is (= (:id params) (:id result)))
|
|
(t/is (= (:file-id params) (:file-id result)))
|
|
(t/is (= 800 (:width result)))
|
|
(t/is (= 800 (:height result)))
|
|
(t/is (= "image/jpeg" (:mtype result)))
|
|
(t/is (uuid? (:media-id result)))
|
|
(t/is (uuid? (:thumbnail-id result))))
|
|
|
|
;; Second try
|
|
(let [{:keys [result error] :as out} (th/command! params)]
|
|
;; (th/print-result! out)
|
|
(t/is (nil? error))
|
|
(t/is (= (:id params) (:id result)))
|
|
(t/is (= (:file-id params) (:file-id result)))
|
|
(t/is (= 800 (:width result)))
|
|
(t/is (= 800 (:height result)))
|
|
(t/is (= "image/jpeg" (:mtype result)))
|
|
(t/is (uuid? (:media-id result)))
|
|
(t/is (uuid? (:thumbnail-id result))))))
|
|
|
|
|
|
(t/deftest media-object-from-url-command
|
|
(let [prof (th/create-profile* 1)
|
|
proj (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
url "https://raw.githubusercontent.com/uxbox/uxbox/develop/sample_media/images/unsplash/anna-pelzer.jpg"
|
|
params {::th/type :create-file-media-object-from-url
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:url url}
|
|
out (th/command! params)]
|
|
|
|
;; (th/print-result! out)
|
|
(t/is (nil? (:error out)))
|
|
(let [{:keys [media-id thumbnail-id] :as result} (:result out)]
|
|
(t/is (= (:id file) (:file-id result)))
|
|
(t/is (= 1024 (:width result)))
|
|
(t/is (= 683 (:height result)))
|
|
(t/is (= "image/jpeg" (:mtype result)))
|
|
(t/is (uuid? media-id))
|
|
(t/is (uuid? thumbnail-id))
|
|
(let [storage (:app.storage/storage th/*system*)
|
|
mobj1 (sto/get-object storage media-id)
|
|
mobj2 (sto/get-object storage thumbnail-id)]
|
|
(t/is (sto/object? mobj1))
|
|
(t/is (sto/object? mobj2))
|
|
(t/is (= 122785 (:size mobj1)))
|
|
(t/is (= 3297 (:size mobj2)))))))
|
|
|
|
(t/deftest media-object-upload-command
|
|
(let [prof (th/create-profile* 1)
|
|
proj (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
mfile {:filename "sample.jpg"
|
|
:path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
|
|
params {::th/type :upload-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "testfile"
|
|
:content mfile}
|
|
out (th/command! params)]
|
|
|
|
;; (th/print-result! out)
|
|
(t/is (nil? (:error out)))
|
|
(let [{:keys [media-id thumbnail-id] :as result} (:result out)]
|
|
(t/is (= (:id file) (:file-id result)))
|
|
(t/is (= 800 (:width result)))
|
|
(t/is (= 800 (:height result)))
|
|
(t/is (= "image/jpeg" (:mtype result)))
|
|
(t/is (uuid? media-id))
|
|
(t/is (uuid? thumbnail-id))
|
|
(let [storage (:app.storage/storage th/*system*)
|
|
mobj1 (sto/get-object storage media-id)
|
|
mobj2 (sto/get-object storage thumbnail-id)]
|
|
(t/is (sto/object? mobj1))
|
|
(t/is (sto/object? mobj2))
|
|
(t/is (= 312043 (:size mobj1)))
|
|
(t/is (= 3890 (:size mobj2)))))))
|
|
|
|
|
|
(t/deftest media-object-upload-idempotency-command
|
|
(let [prof (th/create-profile* 1)
|
|
proj (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
mfile {:filename "sample.jpg"
|
|
:path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
|
|
params {::th/type :upload-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "testfile"
|
|
:content mfile
|
|
:id (uuid/next)}]
|
|
|
|
;; First try
|
|
(let [{:keys [result error] :as out} (th/command! params)]
|
|
;; (th/print-result! out)
|
|
(t/is (nil? error))
|
|
(t/is (= (:id params) (:id result)))
|
|
(t/is (= (:file-id params) (:file-id result)))
|
|
(t/is (= 800 (:width result)))
|
|
(t/is (= 800 (:height result)))
|
|
(t/is (= "image/jpeg" (:mtype result)))
|
|
(t/is (uuid? (:media-id result)))
|
|
(t/is (uuid? (:thumbnail-id result))))
|
|
|
|
;; Second try
|
|
(let [{:keys [result error] :as out} (th/command! params)]
|
|
;; (th/print-result! out)
|
|
(t/is (nil? error))
|
|
(t/is (= (:id params) (:id result)))
|
|
(t/is (= (:file-id params) (:file-id result)))
|
|
(t/is (= 800 (:width result)))
|
|
(t/is (= 800 (:height result)))
|
|
(t/is (= "image/jpeg" (:mtype result)))
|
|
(t/is (uuid? (:media-id result)))
|
|
(t/is (uuid? (:thumbnail-id result))))))
|
|
|
|
|
|
(t/deftest media-object-upload-command-when-file-is-deleted
|
|
(let [prof (th/create-profile* 1)
|
|
proj (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
|
|
_ (th/db-update! :file
|
|
{:deleted-at (app.common.time/now)}
|
|
{:id (:id file)})
|
|
|
|
mfile {:filename "sample.jpg"
|
|
:path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
|
|
params {::th/type :upload-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "testfile"
|
|
:content mfile}
|
|
|
|
out (th/command! params)]
|
|
|
|
(let [error (:error out)
|
|
error-data (ex-data error)]
|
|
(t/is (th/ex-info? error))
|
|
(t/is (= (:type error-data) :not-found)))))
|
|
|
|
|
|
(t/deftest download-image-connection-error
|
|
(t/testing "connection refused raises validation error"
|
|
(with-mocks [http-mock {:target 'app.http.client/req!
|
|
:throw (java.net.ConnectException. "Connection refused")}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "http://unreachable.invalid/image.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :validation (:type (ex-data err))))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err)))))))
|
|
|
|
(t/testing "connection timeout raises validation error"
|
|
(with-mocks [http-mock {:target 'app.http.client/req!
|
|
:throw (java.net.http.HttpConnectTimeoutException. "Connect timed out")}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "http://unreachable.invalid/image.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :validation (:type (ex-data err))))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err)))))))
|
|
|
|
(t/testing "request timeout raises validation error"
|
|
(with-mocks [http-mock {:target 'app.http.client/req!
|
|
:throw (java.net.http.HttpTimeoutException. "Request timed out")}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "http://unreachable.invalid/image.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :validation (:type (ex-data err))))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err)))))))
|
|
|
|
(t/testing "I/O error raises validation error"
|
|
(with-mocks [http-mock {:target 'app.http.client/req!
|
|
:throw (java.io.IOException. "Stream closed")}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "http://unreachable.invalid/image.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :validation (:type (ex-data err))))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err))))))))
|
|
|
|
|
|
(t/deftest download-image-status-code-error
|
|
(t/testing "404 status raises validation error"
|
|
(with-mocks [http-mock {:target 'app.http.client/req!
|
|
:return {:status 404
|
|
:headers {"content-type" "text/html"
|
|
"content-length" "0"}
|
|
:body nil}}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "http://example.com/not-found.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :validation (:type (ex-data err))))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err)))))))
|
|
|
|
(t/testing "500 status raises validation error"
|
|
(with-mocks [http-mock {:target 'app.http.client/req!
|
|
:return {:status 500
|
|
:headers {"content-type" "text/html"
|
|
"content-length" "0"}
|
|
:body nil}}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "http://example.com/server-error.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :validation (:type (ex-data err))))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err)))))))
|
|
|
|
(t/testing "302 status raises validation error"
|
|
(with-mocks [http-mock {:target 'app.http.client/req!
|
|
:return {:status 302
|
|
:headers {"content-type" "text/html"
|
|
"content-length" "0"}
|
|
:body nil}}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "http://example.com/redirect.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :validation (:type (ex-data err))))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err))))))))
|
|
|
|
;; --------------------------------------------------------------------
|
|
;; Helpers for chunked-upload tests
|
|
;; --------------------------------------------------------------------
|
|
|
|
(defn- split-file-into-chunks
|
|
"Splits the file at `path` into byte-array chunks of at most
|
|
`chunk-size` bytes. Returns a vector of byte arrays."
|
|
[path chunk-size]
|
|
(let [file (RandomAccessFile. (str path) "r")
|
|
length (.length file)]
|
|
(try
|
|
(loop [offset 0 chunks []]
|
|
(if (>= offset length)
|
|
chunks
|
|
(let [remaining (- length offset)
|
|
size (min chunk-size remaining)
|
|
buf (byte-array size)]
|
|
(.seek file offset)
|
|
(.readFully file buf)
|
|
(recur (+ offset size) (conj chunks buf)))))
|
|
(finally
|
|
(.close file)))))
|
|
|
|
(defn- make-chunk-mfile
|
|
"Writes `data` (byte array) to a tempfile and returns a map
|
|
compatible with `media/schema:upload`."
|
|
[data mtype]
|
|
(let [tmp (fs/create-tempfile :dir "/tmp/penpot" :prefix "test-chunk-")]
|
|
(io/write* tmp data)
|
|
{:filename "chunk"
|
|
:path tmp
|
|
:mtype mtype
|
|
:size (alength data)}))
|
|
|
|
;; --------------------------------------------------------------------
|
|
;; Chunked-upload tests
|
|
;; --------------------------------------------------------------------
|
|
|
|
(defn- create-session!
|
|
"Creates an upload session for `prof` with `total-chunks`. Returns the session-id UUID."
|
|
[prof total-chunks]
|
|
(let [out (th/command! {::th/type :create-upload-session
|
|
::rpc/profile-id (:id prof)
|
|
:total-chunks total-chunks})]
|
|
(t/is (nil? (:error out)))
|
|
(:session-id (:result out))))
|
|
|
|
(t/deftest chunked-upload-happy-path
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
chunks (split-file-into-chunks source-path 110000) ; ~107 KB each
|
|
mtype "image/jpeg"
|
|
total-size (reduce + (map alength chunks))
|
|
session-id (create-session! prof (count chunks))]
|
|
|
|
(t/is (= 3 (count chunks)))
|
|
|
|
;; --- 1. Upload chunks ---
|
|
(doseq [[idx chunk-data] (map-indexed vector chunks)]
|
|
(let [mfile (make-chunk-mfile chunk-data mtype)
|
|
out (th/command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index idx
|
|
:content mfile})]
|
|
(t/is (nil? (:error out)))
|
|
(t/is (= session-id (:session-id (:result out))))
|
|
(t/is (= idx (:index (:result out))))))
|
|
|
|
;; --- 2. Assemble ---
|
|
(let [assemble-out (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "assembled-image"
|
|
:mtype mtype})]
|
|
|
|
(t/is (nil? (:error assemble-out)))
|
|
(let [{:keys [media-id thumbnail-id] :as result} (:result assemble-out)]
|
|
(t/is (= (:id file) (:file-id result)))
|
|
(t/is (= 800 (:width result)))
|
|
(t/is (= 800 (:height result)))
|
|
(t/is (= mtype (:mtype result)))
|
|
(t/is (uuid? media-id))
|
|
(t/is (uuid? thumbnail-id))
|
|
|
|
(let [storage (:app.storage/storage th/*system*)
|
|
mobj1 (sto/get-object storage media-id)
|
|
mobj2 (sto/get-object storage thumbnail-id)]
|
|
(t/is (sto/object? mobj1))
|
|
(t/is (sto/object? mobj2))
|
|
(t/is (= total-size (:size mobj1))))))))
|
|
|
|
(t/deftest chunked-upload-idempotency
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
media-id (uuid/next)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
chunks (split-file-into-chunks source-path 312043) ; single chunk = whole file
|
|
mtype "image/jpeg"
|
|
mfile (make-chunk-mfile (first chunks) mtype)
|
|
session-id (create-session! prof 1)]
|
|
|
|
(th/command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})
|
|
|
|
;; First assemble succeeds; session row is deleted afterwards
|
|
(let [out1 (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "sample"
|
|
:mtype mtype
|
|
:id media-id})]
|
|
(t/is (nil? (:error out1)))
|
|
(t/is (= media-id (:id (:result out1)))))
|
|
|
|
;; Second assemble with the same session-id must fail because the
|
|
;; session row has been deleted after the first assembly
|
|
(let [out2 (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "sample"
|
|
:mtype mtype
|
|
:id media-id})]
|
|
(t/is (some? (:error out2)))
|
|
(t/is (= :not-found (-> out2 :error ex-data :type)))
|
|
(t/is (= :object-not-found (-> out2 :error ex-data :code))))))
|
|
|
|
(t/deftest chunked-upload-no-permission
|
|
;; A second profile must not be able to upload chunks into a session
|
|
;; that belongs to another profile: the DB lookup includes profile-id,
|
|
;; so the session will not be found.
|
|
(let [prof1 (th/create-profile* 1)
|
|
prof2 (th/create-profile* 2)
|
|
session-id (create-session! prof1 1)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
|
|
;; prof2 tries to upload a chunk into prof1's session
|
|
out (th/command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof2)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})]
|
|
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :not-found (-> out :error ex-data :type)))))
|
|
|
|
(t/deftest chunked-upload-invalid-media-type
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
session-id (create-session! prof 1)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}]
|
|
|
|
(th/command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})
|
|
|
|
;; Assemble with a wrong mtype should fail validation
|
|
(let [out (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "bad-type"
|
|
:mtype "application/octet-stream"})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :validation (-> out :error ex-data :type))))))
|
|
|
|
(t/deftest chunked-upload-missing-chunks
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
;; Session expects 3 chunks
|
|
session-id (create-session! prof 3)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}]
|
|
|
|
;; Upload only 1 chunk
|
|
(th/command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})
|
|
|
|
;; Assemble: session says 3 expected, only 1 stored → :missing-chunks
|
|
(let [out (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "incomplete"
|
|
:mtype "image/jpeg"})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :validation (-> out :error ex-data :type)))
|
|
(t/is (= :missing-chunks (-> out :error ex-data :code))))))
|
|
|
|
(t/deftest chunked-upload-session-not-found
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
bogus-id (uuid/next)]
|
|
|
|
;; Assemble with a session-id that was never created
|
|
(let [out (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id bogus-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "ghost"
|
|
:mtype "image/jpeg"})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :not-found (-> out :error ex-data :type)))
|
|
(t/is (= :object-not-found (-> out :error ex-data :code))))))
|
|
|
|
(t/deftest chunked-upload-over-chunk-limit
|
|
;; Verify that requesting more chunks than the configured maximum
|
|
;; (quotes-upload-chunks-per-session) raises a :restriction error.
|
|
(with-mocks [mock {:target 'app.config/get
|
|
:return (th/config-get-mock
|
|
{:quotes-upload-chunks-per-session 3})}]
|
|
(let [prof (th/create-profile* 1)
|
|
out (th/command! {::th/type :create-upload-session
|
|
::rpc/profile-id (:id prof)
|
|
:total-chunks 4})]
|
|
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :restriction (-> out :error ex-data :type)))
|
|
(t/is (= :max-quote-reached (-> out :error ex-data :code)))
|
|
(t/is (= "upload-chunks-per-session" (-> out :error ex-data :target))))))
|
|
|
|
(t/deftest chunked-upload-invalid-chunk-index
|
|
;; Both a negative index and an index >= total-chunks must be
|
|
;; rejected with a :validation / :invalid-chunk-index error.
|
|
(let [prof (th/create-profile* 1)
|
|
session-id (create-session! prof 2)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}]
|
|
|
|
;; index == total-chunks (out of range)
|
|
(let [out (th/command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 2
|
|
:content mfile})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :validation (-> out :error ex-data :type)))
|
|
(t/is (= :invalid-chunk-index (-> out :error ex-data :code))))
|
|
|
|
;; negative index
|
|
(let [out (th/command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index -1
|
|
:content mfile})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :validation (-> out :error ex-data :type)))
|
|
(t/is (= :invalid-chunk-index (-> out :error ex-data :code))))))
|
|
|
|
(t/deftest chunked-upload-sessions-per-profile-quota
|
|
;; With the session limit set to 2, creating a third session for the
|
|
;; same profile must fail with :restriction / :max-quote-reached.
|
|
;; The :quotes flag is already enabled by the test fixture.
|
|
(with-mocks [mock {:target 'app.config/get
|
|
:return (th/config-get-mock
|
|
{:quotes-upload-sessions-per-profile 2})}]
|
|
(let [prof (th/create-profile* 1)]
|
|
|
|
;; First two sessions succeed
|
|
(create-session! prof 1)
|
|
(create-session! prof 1)
|
|
|
|
;; Third session must be rejected
|
|
(let [out (th/command! {::th/type :create-upload-session
|
|
::rpc/profile-id (:id prof)
|
|
:total-chunks 1})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :restriction (-> out :error ex-data :type)))
|
|
(t/is (= :max-quote-reached (-> out :error ex-data :code)))))))
|