mirror of
https://github.com/penpot/penpot.git
synced 2026-11-12 13:00:37 +00:00
4874da450f
* 🐛 Escape SVG values and sanitize every SVG upload The SVG sanitizer removed dangerous elements and attributes by name and then wrote the tree back with clojure.xml/emit, which escapes nothing. A payload hidden in an attribute value or in a text node became live markup again: an id of "x' onload='alert(1)'" was stored as a real onload handler, and an entity encoded script tag was stored as a real script element. Write the filtered tree with a serializer that escapes text and attribute values, and route every path that stores an uploaded SVG through it. The thumbnail writers (create-file-object-thumbnail and create-file-thumbnail) and the organization logo stored the upload untouched, so they could plant live markup in a public bucket. The filter only knows about SVG elements, so it also accepted documents that are not SVG at all: a page with an iframe or a meta refresh was stored as an image. The sanitizer now rejects any document whose root is not an svg element in the SVG namespace, and it compares the scheme of an url the way the browser does, so a scheme split with a tab or a line break no longer slips through the javascript: filter. The attributes are written sorted by name, because the parser hands them over in the reverse order of the document and the output changed on every sanitization. The profile picture of an identity provider is not stored either when it is an SVG: the avatar is always rendered as an image, so an SVG would only add the risk of keeping live markup in the storage. The dimensions of an SVG are now read from its local name, so an SVG with a namespace prefix can be uploaded like any other. The logo had no size limit and the sanitizer reads the whole file, so it now honors the media max file size. The output shape changes with it: no line breaks, double quotes, and the single quote of an attribute value escaped as an entity. The hash of every stored SVG object changes with it. The media tests use cuerdas.core instead of clojure.string, as the rest of the codebase does. Relates to GHSA-qwhj-rchj-58mg AI-assisted-by: minimax-m3.1-flash * 🐛 Drop non-SVG namespace subtrees in the SVG sanitizer The sanitizer only checked the namespace of the root, so a child could rebind xmlns to XHTML and smuggle live HTML (iframe/srcdoc/meta) past the blocklist. The serializer escapes attribute values, but the browser decodes them again, so srcdoc stays live. Track the namespace bindings through the recursion and drop any element that is not in the SVG namespace, with its subtree. The root is validated up front with the same model, so a document that is not an svg document is rejected with an honest hint instead of a misleading one. The default namespace is stored under a namespaced key, because a prefix literally named `default` collided with the internal key and re-opened the same hole through a prefixed binding. The attribute filter now keeps `xmlns`/`xmlns:*` declarations, which are not event handlers, so a prefix named like one does not end up undeclared in the output. The blocklist also drops HTML-only elements written in the svg namespace (iframe, meta, embed, link, base). Add regression tests for the child rebinding, for the prefixed variants, for the `default` prefix, for the root local name, for the create-file-media-object-from-url path (mocked download), and for the HTML-only tags. Assert double-quoted attributes in the clean content test. Relates to GHSA-qwhj-rchj-58mg AI-assisted-by: longcat-2.5-free
1413 lines
69 KiB
Clojure
1413 lines
69 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
|
|
|
(ns backend-tests.rpc-media-test
|
|
(:require
|
|
[app.common.uuid :as uuid]
|
|
[app.db :as db]
|
|
[app.http.client :as http]
|
|
[app.media :as media]
|
|
[app.media.svg :as svg]
|
|
[app.rpc :as-alias rpc]
|
|
[app.storage :as sto]
|
|
[backend-tests.helpers :as th]
|
|
[clojure.test :as t]
|
|
[cuerdas.core :as str]
|
|
[datoteka.fs :as fs]
|
|
[datoteka.io :as io]
|
|
[mockery.core :refer [with-mocks]])
|
|
(:import
|
|
java.io.RandomAccessFile))
|
|
|
|
(t/use-fixtures :once th/state-init)
|
|
(t/use-fixtures :each th/database-reset)
|
|
|
|
(def ^:private evil-svg
|
|
"An SVG that smuggles an event handler inside an attribute value."
|
|
"<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"64\" height=\"64\" id=\"x' onload='alert(1)\"><rect width=\"10\" height=\"10\"/></svg>")
|
|
|
|
(def ^:private evil-text-svg
|
|
"An SVG that smuggles a script element inside a text node."
|
|
"<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"64\" height=\"64\"><text></text><script>alert(1)</script></text></svg>")
|
|
|
|
(t/deftest media-object-from-url
|
|
(let [prof (th/create-profile* 1)
|
|
proj (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
url "https://raw.githubusercontent.com/uxbox/uxbox/develop/sample_media/images/unsplash/anna-pelzer.jpg"
|
|
params {::th/type :create-file-media-object-from-url
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:url url}
|
|
out (th/command! params)]
|
|
|
|
;; (th/print-result! out)
|
|
(t/is (nil? (:error out)))
|
|
(let [{:keys [media-id thumbnail-id] :as result} (:result out)]
|
|
(t/is (= (:id file) (:file-id result)))
|
|
(t/is (= 1024 (:width result)))
|
|
(t/is (= 683 (:height result)))
|
|
(t/is (= "image/jpeg" (:mtype result)))
|
|
(t/is (uuid? media-id))
|
|
(t/is (uuid? thumbnail-id))
|
|
(let [storage (:app.storage/storage th/*system*)
|
|
mobj1 (sto/get-object storage media-id)
|
|
mobj2 (sto/get-object storage thumbnail-id)]
|
|
(t/is (sto/object? mobj1))
|
|
(t/is (sto/object? mobj2))
|
|
(t/is (= 122785 (:size mobj1)))
|
|
(t/is (= 3297 (:size mobj2)))))))
|
|
|
|
(t/deftest media-object-upload
|
|
(let [prof (th/create-profile* 1)
|
|
proj (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
mfile {:filename "sample.jpg"
|
|
:path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
|
|
params {::th/type :upload-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "testfile"
|
|
:content mfile}
|
|
out (th/multipart-command! params)]
|
|
|
|
;; (th/print-result! out)
|
|
(t/is (nil? (:error out)))
|
|
(let [{:keys [media-id thumbnail-id] :as result} (:result out)]
|
|
(t/is (= (:id file) (:file-id result)))
|
|
(t/is (uuid? (:id result)))
|
|
(t/is (= 800 (:width result)))
|
|
(t/is (= 800 (:height result)))
|
|
(t/is (= "image/jpeg" (:mtype result)))
|
|
(t/is (uuid? media-id))
|
|
(t/is (uuid? thumbnail-id))
|
|
(let [storage (:app.storage/storage th/*system*)
|
|
mobj1 (sto/get-object storage media-id)
|
|
mobj2 (sto/get-object storage thumbnail-id)]
|
|
(t/is (sto/object? mobj1))
|
|
(t/is (sto/object? mobj2))
|
|
(t/is (= 312043 (:size mobj1)))
|
|
(t/is (= 3890 (:size mobj2)))))))
|
|
|
|
|
|
(t/deftest upload-file-media-object-sanitizes-svg
|
|
(let [prof (th/create-profile* 1)
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})]
|
|
(th/with-temp-file [path evil-svg]
|
|
(let [params {::th/type :upload-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "evil.svg"
|
|
:content {:filename "evil.svg"
|
|
:path path
|
|
:mtype "image/svg+xml"
|
|
:size (fs/size path)}}
|
|
out (th/multipart-command! params)]
|
|
|
|
(t/is (nil? (:error out)))
|
|
|
|
(let [{:keys [media-id]} (:result out)
|
|
storage (:app.storage/storage th/*system*)
|
|
sobject (sto/get-object storage media-id)
|
|
content (slurp (sto/get-object-data storage sobject))
|
|
parsed (svg/parse-svg content)]
|
|
(t/is (not (contains? (:attrs parsed) :onload)))
|
|
(t/is (= "x' onload='alert(1)" (get (:attrs parsed) :id)))
|
|
(t/is (str/includes? content "'")))))))
|
|
|
|
(t/deftest upload-file-media-object-sanitizes-svg-text
|
|
(let [prof (th/create-profile* 1)
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})]
|
|
(th/with-temp-file [path evil-text-svg]
|
|
(let [params {::th/type :upload-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "evil.svg"
|
|
:content {:filename "evil.svg"
|
|
:path path
|
|
:mtype "image/svg+xml"
|
|
:size (fs/size path)}}
|
|
out (th/multipart-command! params)]
|
|
|
|
(t/is (nil? (:error out)))
|
|
|
|
(let [{:keys [media-id]} (:result out)
|
|
storage (:app.storage/storage th/*system*)
|
|
sobject (sto/get-object storage media-id)
|
|
content (slurp (sto/get-object-data storage sobject))
|
|
parsed (svg/parse-svg content)]
|
|
(t/is (not (str/includes? content "<script")))
|
|
(t/is (= "</text><script>alert(1)</script>"
|
|
(get-in parsed [:content 0 :content 0]))))))))
|
|
|
|
(t/deftest create-file-media-object-from-url-sanitizes-svg
|
|
(let [prof (th/create-profile* 1)
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
body (.getBytes ^String evil-svg "UTF-8")]
|
|
(with-mocks [http-mock {:target 'app.http.client/req-with-redirects
|
|
:return {:status 200
|
|
:headers {"content-type" "image/svg+xml"
|
|
"content-length" (str (alength ^bytes body))}
|
|
:body (java.io.ByteArrayInputStream. ^bytes body)}}]
|
|
(let [params {::th/type :create-file-media-object-from-url
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:url "https://example.com/evil.svg"}
|
|
out (th/command! params)]
|
|
(t/is (nil? (:error out)))
|
|
|
|
(let [{:keys [media-id]} (:result out)
|
|
storage (:app.storage/storage th/*system*)
|
|
sobject (sto/get-object storage media-id)
|
|
content (slurp (sto/get-object-data storage sobject))
|
|
parsed (svg/parse-svg content)]
|
|
(t/is (not (contains? (:attrs parsed) :onload)))
|
|
(t/is (not (str/includes? content "<script>")))
|
|
(t/is (= "x' onload='alert(1)" (get (:attrs parsed) :id)))
|
|
(t/is (str/includes? content "'")))))))
|
|
|
|
(def ^:private xhtml-child-svg
|
|
"An SVG whose root is in the SVG namespace but whose child rebinds the
|
|
default namespace to XHTML and embeds live HTML."
|
|
"<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"64\" height=\"64\"><g xmlns=\"http://www.w3.org/1999/xhtml\"><iframe srcdoc=\"<script>alert(1)</script>\"/><meta http-equiv=\"refresh\" content=\"0;url=javascript:alert(1)\"/></g><rect width=\"10\" height=\"10\"/></svg>")
|
|
|
|
(t/deftest create-file-media-object-from-url-sanitizes-xhtml-child
|
|
(let [prof (th/create-profile* 1)
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
body (.getBytes ^String xhtml-child-svg "UTF-8")]
|
|
(with-mocks [http-mock {:target 'app.http.client/req-with-redirects
|
|
:return {:status 200
|
|
:headers {"content-type" "image/svg+xml"
|
|
"content-length" (str (alength ^bytes body))}
|
|
:body (java.io.ByteArrayInputStream. ^bytes body)}}]
|
|
(let [params {::th/type :create-file-media-object-from-url
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:url "https://example.com/evil.svg"}
|
|
out (th/command! params)]
|
|
(t/is (nil? (:error out)))
|
|
|
|
(let [{:keys [media-id]} (:result out)
|
|
storage (:app.storage/storage th/*system*)
|
|
sobject (sto/get-object storage media-id)
|
|
content (slurp (sto/get-object-data storage sobject))]
|
|
(t/is (not (str/includes? content "iframe")))
|
|
(t/is (not (str/includes? content "srcdoc")))
|
|
(t/is (not (str/includes? content "http-equiv"))))))))
|
|
|
|
(t/deftest media-object-from-url-command
|
|
(let [prof (th/create-profile* 1)
|
|
proj (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
url "https://raw.githubusercontent.com/uxbox/uxbox/develop/sample_media/images/unsplash/anna-pelzer.jpg"
|
|
params {::th/type :create-file-media-object-from-url
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:url url}
|
|
out (th/command! params)]
|
|
|
|
;; (th/print-result! out)
|
|
(t/is (nil? (:error out)))
|
|
(let [{:keys [media-id thumbnail-id] :as result} (:result out)]
|
|
(t/is (= (:id file) (:file-id result)))
|
|
(t/is (= 1024 (:width result)))
|
|
(t/is (= 683 (:height result)))
|
|
(t/is (= "image/jpeg" (:mtype result)))
|
|
(t/is (uuid? media-id))
|
|
(t/is (uuid? thumbnail-id))
|
|
(let [storage (:app.storage/storage th/*system*)
|
|
mobj1 (sto/get-object storage media-id)
|
|
mobj2 (sto/get-object storage thumbnail-id)]
|
|
(t/is (sto/object? mobj1))
|
|
(t/is (sto/object? mobj2))
|
|
(t/is (= 122785 (:size mobj1)))
|
|
(t/is (= 3297 (:size mobj2)))))))
|
|
|
|
(t/deftest media-object-upload-command
|
|
(let [prof (th/create-profile* 1)
|
|
proj (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
mfile {:filename "sample.jpg"
|
|
:path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
|
|
params {::th/type :upload-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "testfile"
|
|
:content mfile}
|
|
out (th/multipart-command! params)]
|
|
|
|
;; (th/print-result! out)
|
|
(t/is (nil? (:error out)))
|
|
(let [{:keys [media-id thumbnail-id] :as result} (:result out)]
|
|
(t/is (= (:id file) (:file-id result)))
|
|
(t/is (uuid? (:id result)))
|
|
(t/is (= 800 (:width result)))
|
|
(t/is (= 800 (:height result)))
|
|
(t/is (= "image/jpeg" (:mtype result)))
|
|
(t/is (uuid? media-id))
|
|
(t/is (uuid? thumbnail-id))
|
|
(let [storage (:app.storage/storage th/*system*)
|
|
mobj1 (sto/get-object storage media-id)
|
|
mobj2 (sto/get-object storage thumbnail-id)]
|
|
(t/is (sto/object? mobj1))
|
|
(t/is (sto/object? mobj2))
|
|
(t/is (= 312043 (:size mobj1)))
|
|
(t/is (= 3890 (:size mobj2)))))))
|
|
|
|
|
|
(t/deftest media-object-upload-command-when-file-is-deleted
|
|
(let [prof (th/create-profile* 1)
|
|
proj (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
|
|
_ (th/db-update! :file
|
|
{:deleted-at (app.common.time/now)}
|
|
{:id (:id file)})
|
|
|
|
mfile {:filename "sample.jpg"
|
|
:path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
|
|
params {::th/type :upload-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "testfile"
|
|
:content mfile}
|
|
|
|
out (th/multipart-command! params)]
|
|
|
|
(let [error (:error out)
|
|
error-data (ex-data error)]
|
|
(t/is (th/ex-info? error))
|
|
(t/is (= (:type error-data) :not-found)))))
|
|
|
|
|
|
(t/deftest download-image-connection-error
|
|
(t/testing "connection refused raises validation error"
|
|
(with-mocks [http-mock {:target 'app.http.client/req-with-redirects
|
|
:throw (java.net.ConnectException. "Connection refused")}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "https://example.com/image.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :validation (:type (ex-data err))))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err)))))))
|
|
|
|
(t/testing "connection timeout raises validation error"
|
|
(with-mocks [http-mock {:target 'app.http.client/req-with-redirects
|
|
:throw (java.net.http.HttpConnectTimeoutException. "Connect timed out")}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "https://example.com/image.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :validation (:type (ex-data err))))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err)))))))
|
|
|
|
(t/testing "request timeout raises validation error"
|
|
(with-mocks [http-mock {:target 'app.http.client/req-with-redirects
|
|
:throw (java.net.http.HttpTimeoutException. "Request timed out")}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "https://example.com/image.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :validation (:type (ex-data err))))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err)))))))
|
|
|
|
(t/testing "I/O error raises validation error"
|
|
(with-mocks [http-mock {:target 'app.http.client/req-with-redirects
|
|
:throw (java.io.IOException. "Stream closed")}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "https://example.com/image.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :validation (:type (ex-data err))))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err))))))))
|
|
|
|
|
|
(t/deftest download-image-status-code-error
|
|
(t/testing "404 status raises validation error"
|
|
(with-mocks [http-mock {:target 'app.http.client/req-with-redirects
|
|
:return {:status 404
|
|
:headers {"content-type" "text/html"
|
|
"content-length" "0"}
|
|
:body nil}}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "https://example.com/not-found.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :validation (:type (ex-data err))))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err)))))))
|
|
|
|
(t/testing "500 status raises validation error"
|
|
(with-mocks [http-mock {:target 'app.http.client/req-with-redirects
|
|
:return {:status 500
|
|
:headers {"content-type" "text/html"
|
|
"content-length" "0"}
|
|
:body nil}}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "https://example.com/server-error.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :validation (:type (ex-data err))))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err)))))))
|
|
|
|
(t/testing "302 status raises validation error"
|
|
(with-mocks [http-mock {:target 'app.http.client/req-with-redirects
|
|
:return {:status 302
|
|
:headers {"content-type" "text/html"
|
|
"content-length" "0"}
|
|
:body nil}}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "https://example.com/redirect.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :validation (:type (ex-data err))))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err))))))))
|
|
|
|
|
|
(t/deftest download-image-closes-stream
|
|
(t/testing "response body stream is closed on success"
|
|
(let [closed? (atom false)
|
|
;; Minimal valid PNG (1x1 pixel, red)
|
|
png-data (byte-array [0x89 0x50 0x4E 0x47 0x0D 0x0A 0x1A 0x0A 0x00 0x00 0x00 0x0D 0x49 0x48 0x44 0x52 0x00 0x00 0x00 0x01 0x00 0x00 0x00 0x01 0x08 0x02 0x00 0x00 0x00 0x90 0x77 0x53 0xDE 0x00 0x00 0x00 0x0C 0x49 0x44 0x41 0x54 0x08 0xD7 0x63 0xF8 0xCF 0xC0 0x00 0x00 0x00 0x02 0x00 0x01 0xE2 0x21 0xBC 0x33 0x00 0x00 0x00 0x00 0x49 0x45 0x4E 0x44 0xAE 0x42 0x60 0x82])
|
|
body (proxy [java.io.ByteArrayInputStream] [png-data]
|
|
(close [] (reset! closed? true)))]
|
|
(with-mocks [http-mock {:target 'app.http.client/req-with-redirects
|
|
:return {:status 200
|
|
:headers {"content-type" "image/png"
|
|
"content-length" (str (alength png-data))}
|
|
:body body}}]
|
|
(let [cfg {::http/client :mock-client}
|
|
result (media/download-image cfg "https://example.com/image.png")]
|
|
(t/is (some? result))
|
|
(t/is @closed? "body stream should be closed after successful download")))))
|
|
|
|
(t/testing "response body stream is closed on validation error"
|
|
(let [closed? (atom false)
|
|
body (proxy [java.io.ByteArrayInputStream] [(byte-array 100)]
|
|
(close [] (reset! closed? true)))]
|
|
(with-mocks [http-mock {:target 'app.http.client/req-with-redirects
|
|
:return {:status 404
|
|
:headers {"content-type" "text/html"
|
|
"content-length" "100"}
|
|
:body body}}]
|
|
(let [cfg {::http/client :mock-client}
|
|
err (try
|
|
(media/download-image cfg "https://example.com/not-found.png")
|
|
nil
|
|
(catch clojure.lang.ExceptionInfo e e))]
|
|
(t/is (some? err))
|
|
(t/is (= :unable-to-download-image (:code (ex-data err))))
|
|
(t/is @closed? "body stream should be closed even on validation error"))))))
|
|
;; --------------------------------------------------------------------
|
|
|
|
(defn- split-file-into-chunks
|
|
"Splits the file at `path` into byte-array chunks of at most
|
|
`chunk-size` bytes. Returns a vector of byte arrays."
|
|
[path chunk-size]
|
|
(let [file (RandomAccessFile. (str path) "r")
|
|
length (.length file)]
|
|
(try
|
|
(loop [offset 0 chunks []]
|
|
(if (>= offset length)
|
|
chunks
|
|
(let [remaining (- length offset)
|
|
size (min chunk-size remaining)
|
|
buf (byte-array size)]
|
|
(.seek file offset)
|
|
(.readFully file buf)
|
|
(recur (+ offset size) (conj chunks buf)))))
|
|
(finally
|
|
(.close file)))))
|
|
|
|
(defn- make-chunk-mfile
|
|
"Writes `data` (byte array) to a tempfile and returns a map
|
|
compatible with `media/schema:upload`."
|
|
[data mtype]
|
|
(let [tmp (fs/create-tempfile :dir "/tmp/penpot" :prefix "test-chunk-")]
|
|
(io/write* tmp data)
|
|
{:filename "chunk"
|
|
:path tmp
|
|
:mtype mtype
|
|
:size (alength data)}))
|
|
|
|
;; --------------------------------------------------------------------
|
|
;; Chunked-upload tests
|
|
;; --------------------------------------------------------------------
|
|
|
|
(defn- create-session!
|
|
"Creates an upload session for `prof` with `total-chunks`. Returns the session-id UUID."
|
|
[prof total-chunks]
|
|
(let [out (th/command! {::th/type :create-upload-session
|
|
::rpc/profile-id (:id prof)
|
|
:total-chunks total-chunks})]
|
|
(t/is (nil? (:error out)))
|
|
(:session-id (:result out))))
|
|
|
|
(t/deftest chunked-upload-happy-path
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
chunks (split-file-into-chunks source-path 110000) ; ~107 KB each
|
|
mtype "image/jpeg"
|
|
total-size (reduce + (map alength chunks))
|
|
session-id (create-session! prof (count chunks))]
|
|
|
|
(t/is (= 3 (count chunks)))
|
|
|
|
;; --- 1. Upload chunks ---
|
|
(doseq [[idx chunk-data] (map-indexed vector chunks)]
|
|
(let [mfile (make-chunk-mfile chunk-data mtype)
|
|
out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index idx
|
|
:content mfile})]
|
|
(t/is (nil? (:error out)))
|
|
(t/is (= session-id (:session-id (:result out))))
|
|
(t/is (= idx (:index (:result out))))))
|
|
|
|
;; --- 2. Assemble ---
|
|
(let [assemble-out (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "assembled-image"
|
|
:mtype mtype})]
|
|
|
|
(t/is (nil? (:error assemble-out)))
|
|
(let [{:keys [media-id thumbnail-id] :as result} (:result assemble-out)]
|
|
(t/is (= (:id file) (:file-id result)))
|
|
(t/is (= 800 (:width result)))
|
|
(t/is (= 800 (:height result)))
|
|
(t/is (= mtype (:mtype result)))
|
|
(t/is (uuid? media-id))
|
|
(t/is (uuid? thumbnail-id))
|
|
|
|
(let [storage (:app.storage/storage th/*system*)
|
|
mobj1 (sto/get-object storage media-id)
|
|
mobj2 (sto/get-object storage thumbnail-id)]
|
|
(t/is (sto/object? mobj1))
|
|
(t/is (sto/object? mobj2))
|
|
(t/is (= total-size (:size mobj1))))))))
|
|
|
|
(t/deftest chunked-upload-idempotency
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
chunks (split-file-into-chunks source-path 312043) ; single chunk = whole file
|
|
mtype "image/jpeg"
|
|
mfile (make-chunk-mfile (first chunks) mtype)
|
|
session-id (create-session! prof 1)]
|
|
|
|
(th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})
|
|
|
|
;; First assemble succeeds; session row is marked as consumed afterwards
|
|
(let [out1 (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "sample"
|
|
:mtype mtype})]
|
|
(t/is (nil? (:error out1)))
|
|
(t/is (uuid? (:id (:result out1)))))
|
|
|
|
;; Second assemble with the same session-id must fail because the
|
|
;; session row has been marked as consumed after the first assembly
|
|
(let [out2 (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "sample"
|
|
:mtype mtype})]
|
|
(t/is (some? (:error out2)))
|
|
(t/is (= :not-found (-> out2 :error ex-data :type)))
|
|
(t/is (= :object-not-found (-> out2 :error ex-data :code))))))
|
|
|
|
(t/deftest chunked-upload-no-permission
|
|
;; A second profile must not be able to upload chunks into a session
|
|
;; that belongs to another profile: the DB lookup includes profile-id,
|
|
;; so the session will not be found.
|
|
(let [prof1 (th/create-profile* 1)
|
|
prof2 (th/create-profile* 2)
|
|
session-id (create-session! prof1 1)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
|
|
;; prof2 tries to upload a chunk into prof1's session
|
|
out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof2)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})]
|
|
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :not-found (-> out :error ex-data :type)))))
|
|
|
|
(t/deftest chunked-upload-other-profile-cannot-assemble
|
|
;; assemble-chunks must scope the session lookup to the requesting
|
|
;; profile so that a different profile cannot assemble chunks from
|
|
;; a session they do not own (BOLA / CWE-639).
|
|
(let [prof1 (th/create-profile* 1)
|
|
prof2 (th/create-profile* 2)
|
|
session-id (create-session! prof1 1)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}]
|
|
|
|
;; prof1 uploads a chunk into their own session
|
|
(let [out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof1)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})]
|
|
(t/is (nil? (:error out))))
|
|
|
|
;; prof2 tries to assemble prof1's session via create-font-variant
|
|
;; (which calls assemble-chunks without ownership check)
|
|
(let [out (th/command! {::th/type :create-font-variant
|
|
::rpc/profile-id (:id prof2)
|
|
:team-id (:default-team-id prof2)
|
|
:font-id (uuid/next)
|
|
:font-family "TestFont"
|
|
:font-weight 400
|
|
:font-style "normal"
|
|
:uploads {"font/ttf" session-id}})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :not-found (-> out :error ex-data :type)))
|
|
(t/is (= :object-not-found (-> out :error ex-data :code))))))
|
|
|
|
(t/deftest chunked-upload-invalid-media-type
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
session-id (create-session! prof 1)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}]
|
|
|
|
(th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})
|
|
|
|
;; Assemble with a wrong mtype should fail validation
|
|
(let [out (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "bad-type"
|
|
:mtype "application/octet-stream"})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :validation (-> out :error ex-data :type))))))
|
|
|
|
(t/deftest chunked-upload-missing-chunks
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
;; Session expects 3 chunks
|
|
session-id (create-session! prof 3)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}]
|
|
|
|
;; Upload only 1 chunk
|
|
(th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})
|
|
|
|
;; Assemble: session says 3 expected, only 1 stored → :missing-chunks
|
|
(let [out (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "incomplete"
|
|
:mtype "image/jpeg"})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :validation (-> out :error ex-data :type)))
|
|
(t/is (= :missing-chunks (-> out :error ex-data :code))))))
|
|
|
|
(t/deftest chunked-upload-duplicate-then-assemble
|
|
;; A rejected duplicate must leave the first chunk intact: upload 0,
|
|
;; re-upload 0 (rejected), then assemble succeeds with the original size.
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
session-id (create-session! prof 1)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
chunks (split-file-into-chunks source-path 312043)
|
|
mtype "image/jpeg"
|
|
size (alength (first chunks))]
|
|
|
|
(let [out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content (make-chunk-mfile (first chunks) mtype)})]
|
|
(t/is (nil? (:error out))))
|
|
|
|
(let [out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content (make-chunk-mfile (first chunks) mtype)})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :validation (-> out :error ex-data :type)))
|
|
(t/is (= :chunk-already-exists (-> out :error ex-data :code))))
|
|
|
|
(let [out (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "after-dupe"
|
|
:mtype mtype})]
|
|
(t/is (nil? (:error out)))
|
|
(let [storage (:app.storage/storage th/*system*)
|
|
mobj (sto/get-object storage (:media-id (:result out)))]
|
|
(t/is (= size (:size mobj)))))))
|
|
|
|
(t/deftest chunked-upload-rejected-duplicate-keeps-session-usable
|
|
;; Rejecting a duplicate must not poison the session: the remaining
|
|
;; distinct indices still accumulate and assemble normally.
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
session-id (create-session! prof 2)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
chunks (split-file-into-chunks source-path 110000)
|
|
mtype "image/jpeg"]
|
|
|
|
(t/is (= 3 (count chunks)))
|
|
|
|
(let [out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content (make-chunk-mfile (nth chunks 0) mtype)})]
|
|
(t/is (nil? (:error out))))
|
|
|
|
(let [out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content (make-chunk-mfile (nth chunks 0) mtype)})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :validation (-> out :error ex-data :type)))
|
|
(t/is (= :chunk-already-exists (-> out :error ex-data :code))))
|
|
|
|
(let [out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 1
|
|
:content (make-chunk-mfile (nth chunks 1) mtype)})]
|
|
(t/is (nil? (:error out))))
|
|
|
|
;; The mapping table holds exactly the two distinct indices: the
|
|
;; rejected duplicate stored nothing.
|
|
(let [rows (th/db-exec! ["SELECT chunk_index FROM upload_session_chunk WHERE session_id = ? ORDER BY chunk_index"
|
|
session-id])]
|
|
(t/is (= [0 1] (mapv :chunk-index rows))))))
|
|
|
|
(t/deftest chunked-upload-session-not-found
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
bogus-id (uuid/next)]
|
|
|
|
;; Assemble with a session-id that was never created
|
|
(let [out (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id bogus-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "ghost"
|
|
:mtype "image/jpeg"})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :not-found (-> out :error ex-data :type)))
|
|
(t/is (= :object-not-found (-> out :error ex-data :code))))))
|
|
|
|
(t/deftest chunked-upload-over-chunk-limit
|
|
;; Verify that requesting more chunks than the configured maximum
|
|
;; (quotes-upload-chunks-per-session) raises a :restriction error.
|
|
(with-mocks [mock {:target 'app.config/get
|
|
:return (th/config-get-mock
|
|
{:quotes-upload-chunks-per-session 3})}]
|
|
(let [prof (th/create-profile* 1)
|
|
out (th/command! {::th/type :create-upload-session
|
|
::rpc/profile-id (:id prof)
|
|
:total-chunks 4})]
|
|
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :restriction (-> out :error ex-data :type)))
|
|
(t/is (= :max-quote-reached (-> out :error ex-data :code)))
|
|
(t/is (= "upload-chunks-per-session" (-> out :error ex-data :target))))))
|
|
|
|
(t/deftest chunked-upload-consumed-session-frees-quota
|
|
;; Consumed sessions must not count against the sessions-per-profile
|
|
;; quota: with the limit set to 1, assembling a session frees the slot
|
|
;; for a new one.
|
|
(with-mocks [mock {:target 'app.config/get
|
|
:return (th/config-get-mock
|
|
{:quotes-upload-sessions-per-profile 1})}]
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
session-id (create-session! prof 1)
|
|
upload-out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})]
|
|
(t/is (nil? (:error upload-out)))
|
|
|
|
(let [assemble-out (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "assembled-image"
|
|
:mtype "image/jpeg"})]
|
|
(t/is (nil? (:error assemble-out))))
|
|
|
|
;; the consumed session frees the quota slot
|
|
(let [out (th/command! {::th/type :create-upload-session
|
|
::rpc/profile-id (:id prof)
|
|
:total-chunks 1})]
|
|
(t/is (nil? (:error out)))
|
|
(t/is (uuid? (:session-id (:result out))))))))
|
|
|
|
(t/deftest chunked-upload-invalid-total-chunks
|
|
;; total-chunks must be at least 1; zero and negative values are rejected
|
|
;; with a :validation error.
|
|
(let [prof (th/create-profile* 1)]
|
|
;; zero total-chunks
|
|
(let [out (th/command! {::th/type :create-upload-session
|
|
::rpc/profile-id (:id prof)
|
|
:total-chunks 0})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :validation (-> out :error ex-data :type))))
|
|
|
|
;; negative total-chunks
|
|
(let [out (th/command! {::th/type :create-upload-session
|
|
::rpc/profile-id (:id prof)
|
|
:total-chunks -1})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :validation (-> out :error ex-data :type))))))
|
|
|
|
(t/deftest chunked-upload-invalid-chunk-index
|
|
;; Both a negative index and an index >= total-chunks must be
|
|
;; rejected with a :validation / :invalid-chunk-index error.
|
|
(let [prof (th/create-profile* 1)
|
|
session-id (create-session! prof 2)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}]
|
|
|
|
;; index == total-chunks (out of range)
|
|
(let [out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 2
|
|
:content mfile})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :validation (-> out :error ex-data :type)))
|
|
(t/is (= :invalid-chunk-index (-> out :error ex-data :code))))
|
|
|
|
;; negative index
|
|
(let [out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index -1
|
|
:content mfile})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :validation (-> out :error ex-data :type)))
|
|
(t/is (= :invalid-chunk-index (-> out :error ex-data :code))))))
|
|
|
|
(t/deftest chunked-upload-chunk-too-large
|
|
;; Chunks larger than the configured cap must be rejected with
|
|
;; :validation / :chunk-too-large before anything is stored, while a
|
|
;; chunk exactly at the cap still uploads fine.
|
|
(with-mocks [mock {:target 'app.config/get
|
|
:return (th/config-get-mock
|
|
{:upload-max-chunk-size 1024})}]
|
|
(let [prof (th/create-profile* 1)
|
|
session-id (create-session! prof 1)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
chunks (split-file-into-chunks source-path 312043)
|
|
mtype "image/jpeg"]
|
|
|
|
;; 312043 bytes exceeds the mocked 1024-byte cap: rejected
|
|
(let [out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content (make-chunk-mfile (first chunks) mtype)})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :validation (-> out :error ex-data :type)))
|
|
(t/is (= :chunk-too-large (-> out :error ex-data :code))))
|
|
|
|
;; Nothing stored for the rejected chunk
|
|
(t/is (= 0 (:count (th/db-exec-one! ["SELECT count(*) FROM upload_session_chunk WHERE session_id = ?"
|
|
session-id]))))
|
|
|
|
;; A chunk exactly at the cap still uploads fine
|
|
(let [out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content (make-chunk-mfile (byte-array 1024 (byte 1)) mtype)})]
|
|
(t/is (nil? (:error out)))))))
|
|
|
|
(t/deftest chunked-upload-sessions-per-profile-quota
|
|
;; With the session limit set to 2, creating a third session for the
|
|
;; same profile must fail with :restriction / :max-quote-reached.
|
|
;; The :quotes flag is already enabled by the test fixture.
|
|
(with-mocks [mock {:target 'app.config/get
|
|
:return (th/config-get-mock
|
|
{:quotes-upload-sessions-per-profile 2})}]
|
|
(let [prof (th/create-profile* 1)]
|
|
|
|
;; First two sessions succeed
|
|
(create-session! prof 1)
|
|
(create-session! prof 1)
|
|
|
|
;; Third session must be rejected
|
|
(let [out (th/command! {::th/type :create-upload-session
|
|
::rpc/profile-id (:id prof)
|
|
:total-chunks 1})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :restriction (-> out :error ex-data :type)))
|
|
(t/is (= :max-quote-reached (-> out :error ex-data :code)))))))
|
|
|
|
;; --- upload_session_chunk mapping tests ---
|
|
|
|
(t/deftest chunked-upload-creates-chunk-mapping
|
|
;; Uploading a chunk creates a row in upload_session_chunk pointing to the
|
|
;; storage object, and the object itself carries no session metadata.
|
|
(let [prof (th/create-profile* 1)
|
|
session-id (create-session! prof 1)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})]
|
|
(t/is (nil? (:error out)))
|
|
|
|
(let [row (th/db-exec-one! ["select session_id, object_id, chunk_index from upload_session_chunk where session_id = ?"
|
|
session-id])]
|
|
(t/is (= session-id (:session-id row)))
|
|
(t/is (= 0 (:chunk-index row)))
|
|
|
|
(let [storage (:app.storage/storage th/*system*)
|
|
obj (sto/get-object storage (:object-id row))]
|
|
(t/is (sto/object? obj))
|
|
(t/is (= "upload-session" (-> obj meta :bucket)))
|
|
(t/is (nil? (-> obj meta :upload-id)))
|
|
(t/is (nil? (-> obj meta :chunk-index)))))))
|
|
|
|
(t/deftest chunked-upload-duplicate-index-fails
|
|
;; Re-uploading an already stored index fails with
|
|
;; :validation/:chunk-already-exists and creates no new storage object.
|
|
(let [prof (th/create-profile* 1)
|
|
session-id (create-session! prof 1)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
out1 (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})]
|
|
(t/is (nil? (:error out1)))
|
|
|
|
(let [before (:count (th/db-exec-one! ["select count(*) from storage_object"]))
|
|
out2 (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})]
|
|
(t/is (some? (:error out2)))
|
|
(t/is (= :validation (-> out2 :error ex-data :type)))
|
|
(t/is (= :chunk-already-exists (-> out2 :error ex-data :code)))
|
|
(t/is (= before (:count (th/db-exec-one! ["select count(*) from storage_object"])))))))
|
|
|
|
(t/deftest chunked-upload-null-reservation-blocks-retry
|
|
;; A reserved slot with NULL object_id (an upload that died between the
|
|
;; reserve and the link) counts as occupied: retrying the index in the
|
|
;; same session fails with :validation/:chunk-already-exists and stores
|
|
;; nothing, so the client must start a new session.
|
|
(let [prof (th/create-profile* 1)
|
|
session-id (create-session! prof 1)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}]
|
|
(th/db-exec! ["insert into upload_session_chunk (session_id, chunk_index, object_id) values (?, ?, null)"
|
|
session-id 0])
|
|
(let [before (:count (th/db-exec-one! ["select count(*) from storage_object"]))
|
|
out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :validation (-> out :error ex-data :type)))
|
|
(t/is (= :chunk-already-exists (-> out :error ex-data :code)))
|
|
(t/is (= before (:count (th/db-exec-one! ["select count(*) from storage_object"])))))))
|
|
|
|
(t/deftest chunked-upload-link-failure-releases-slot
|
|
;; When the link UPDATE fails after a successful blob write, the
|
|
;; reservation is removed so the client can retry the index in the same
|
|
;; session; the orphaned blob stays touched for touched-gc.
|
|
(let [prof (th/create-profile* 1)
|
|
session-id (create-session! prof 1)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
orig @#'app.rpc.commands.media/link-upload-session-chunk!
|
|
failed? (atom false)]
|
|
(with-mocks [mock {:target 'app.rpc.commands.media/link-upload-session-chunk!
|
|
:return (fn [pool object-id session-id index]
|
|
(if (compare-and-set! failed? false true)
|
|
(throw (ex-info "link boom" {}))
|
|
(orig pool object-id session-id index)))}]
|
|
(let [out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})]
|
|
(t/is (some? (:error out))))
|
|
;; the failed link left no reservation behind
|
|
(t/is (= 0 (:count (th/db-exec-one! ["select count(*) from upload_session_chunk where session_id = ?"
|
|
session-id]))))
|
|
;; retrying the same index in the same session succeeds
|
|
(let [out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})]
|
|
(t/is (nil? (:error out)))
|
|
(t/is (= 1 (:count (th/db-exec-one! ["select count(*) from upload_session_chunk where session_id = ?"
|
|
session-id]))))))))
|
|
|
|
(t/deftest chunked-upload-to-consumed-session-fails
|
|
;; Once assembled, the session is consumed: uploading another chunk fails
|
|
;; with :not-found and the session row stays, marked with deleted_at.
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
session-id (create-session! prof 1)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
out1 (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})]
|
|
(t/is (nil? (:error out1)))
|
|
|
|
(let [assemble-out (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "assembled-image"
|
|
:mtype "image/jpeg"})]
|
|
(t/is (nil? (:error assemble-out))))
|
|
|
|
;; chunk mappings stay until objects-gc purges them, session row
|
|
;; stays marked as consumed
|
|
(t/is (= 1 (:count (th/db-exec-one! ["select count(*) from upload_session_chunk where session_id = ?"
|
|
session-id]))))
|
|
(t/is (some? (:deleted-at (th/db-exec-one! ["select deleted_at from upload_session where id = ?"
|
|
session-id]))))
|
|
|
|
;; uploading to the consumed session fails without creating an object
|
|
(let [before (:count (th/db-exec-one! ["select count(*) from storage_object"]))
|
|
out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})]
|
|
(t/is (some? (:error out)))
|
|
(t/is (= :not-found (-> out :error ex-data :type)))
|
|
(t/is (= :object-not-found (-> out :error ex-data :code)))
|
|
(t/is (= before (:count (th/db-exec-one! ["select count(*) from storage_object"])))))))
|
|
|
|
(defn- sql-state-of
|
|
"Runs thunk (a db statement) and returns the SQLState of the raised
|
|
SQLException, or nil when no error is raised."
|
|
[thunk]
|
|
(try
|
|
(thunk)
|
|
nil
|
|
(catch java.sql.SQLException cause
|
|
(.getSQLState cause))))
|
|
|
|
(t/deftest upload-session-chunk-restrict-blocks-direct-deletes
|
|
;; With a live mapping row, deleting the storage object or the session
|
|
;; directly violates the RESTRICT foreign keys (SQLState 23503).
|
|
(let [prof (th/create-profile* 1)
|
|
session-id (create-session! prof 1)
|
|
source-path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
mfile {:filename "sample.jpg"
|
|
:path source-path
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
out (th/multipart-command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id session-id
|
|
:index 0
|
|
:content mfile})]
|
|
(t/is (nil? (:error out)))
|
|
|
|
(let [object-id (:object-id (th/db-exec-one! ["select object_id from upload_session_chunk where session_id = ?"
|
|
session-id]))]
|
|
(t/is (= "23503" (sql-state-of #(th/db-exec! ["delete from storage_object where id = ?"
|
|
object-id]))))
|
|
(t/is (= "23503" (sql-state-of #(th/db-exec! ["delete from upload_session where id = ?"
|
|
session-id]))))
|
|
;; the profile cannot disappear either while its session is live
|
|
;; (profile_id FK is NO ACTION DEFERRABLE; purge goes through
|
|
;; objects-gc). The deletion_protection rule is disabled here so the
|
|
;; statement reaches the FK check.
|
|
(t/is (= "23503" (sql-state-of #(db/transact! th/*pool*
|
|
(fn [conn]
|
|
(db/exec-one! conn ["SET LOCAL rules.deletion_protection TO off"])
|
|
(db/exec! conn ["delete from profile where id = ?"
|
|
(:id prof)])))))))))
|
|
|
|
;; --- Clone File Media Object BOLA tests ---
|
|
|
|
(defn- create-storage-object!
|
|
[content content-type]
|
|
(let [storage (:app.storage/storage th/*system*)]
|
|
(sto/put-object! storage {::sto/content (sto/content content)
|
|
:content-type content-type})))
|
|
|
|
(t/deftest clone-file-media-object-success
|
|
(let [prof1 (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof1)
|
|
:team-id (:default-team-id prof1)})
|
|
file1 (th/create-file* 1 {:profile-id (:id prof1)
|
|
:project-id (:default-project-id prof1)
|
|
:is-shared false})
|
|
sobj (create-storage-object! "image-content" "image/png")
|
|
mobj (th/create-file-media-object* {:file-id (:id file1)
|
|
:name "test-media"
|
|
:width 100
|
|
:height 100
|
|
:mtype "image/png"
|
|
:media-id (:id sobj)})
|
|
file2 (th/create-file* 2 {:profile-id (:id prof1)
|
|
:project-id (:default-project-id prof1)
|
|
:is-shared false})
|
|
params {::th/type :clone-file-media-object
|
|
::rpc/profile-id (:id prof1)
|
|
:file-id (:id file2)
|
|
:is-local true
|
|
:id (:id mobj)}
|
|
out (th/command! params)]
|
|
|
|
(t/is (nil? (:error out)))
|
|
(let [result (:result out)]
|
|
(t/is (= (:id file2) (:file-id result)))
|
|
(t/is (= (:name mobj) (:name result)))
|
|
(t/is (= (:media-id mobj) (:media-id result)))
|
|
(t/is (uuid? (:id result)))
|
|
(t/is (not= (:id mobj) (:id result))))))
|
|
|
|
(t/deftest clone-file-media-object-no-read-access
|
|
(let [prof1 (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof1)
|
|
:team-id (:default-team-id prof1)})
|
|
file1 (th/create-file* 1 {:profile-id (:id prof1)
|
|
:project-id (:default-project-id prof1)
|
|
:is-shared false})
|
|
sobj (create-storage-object! "private-content" "image/png")
|
|
mobj (th/create-file-media-object* {:file-id (:id file1)
|
|
:name "private-media"
|
|
:width 100
|
|
:height 100
|
|
:mtype "image/png"
|
|
:media-id (:id sobj)})
|
|
|
|
prof2 (th/create-profile* 2)
|
|
_ (th/create-project* 2 {:profile-id (:id prof2)
|
|
:team-id (:default-team-id prof2)})
|
|
file2 (th/create-file* 2 {:profile-id (:id prof2)
|
|
:project-id (:default-project-id prof2)
|
|
:is-shared false})
|
|
|
|
params {::th/type :clone-file-media-object
|
|
::rpc/profile-id (:id prof2)
|
|
:file-id (:id file2)
|
|
:is-local true
|
|
:id (:id mobj)}
|
|
out (th/command! params)]
|
|
|
|
(let [error (:error out)
|
|
error-data (ex-data error)]
|
|
(t/is (th/ex-info? error))
|
|
(t/is (= :not-found (:type error-data)))
|
|
(t/is (= :object-not-found (:code error-data))))))
|
|
|
|
(t/deftest clone-file-media-object-source-not-found
|
|
(let [prof (th/create-profile* 1)
|
|
_ (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:default-team-id prof)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
params {::th/type :clone-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:id (uuid/random)}
|
|
out (th/command! params)]
|
|
|
|
(let [error (:error out)
|
|
error-data (ex-data error)]
|
|
(t/is (th/ex-info? error))
|
|
(t/is (= :not-found (:type error-data)))
|
|
(t/is (= :object-not-found (:code error-data))))))
|
|
|
|
(t/deftest upload-file-media-object-rejects-client-id
|
|
(let [prof (th/create-profile* 1)
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
mfile {:filename "sample.jpg"
|
|
:path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
sent-id (uuid/next)
|
|
out (th/multipart-command! {::th/type :upload-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "testfile"
|
|
:content mfile
|
|
:id sent-id})]
|
|
(t/is (th/ex-info? (:error out)))
|
|
(t/is (th/ex-of-type? (:error out) :validation))
|
|
(t/is (th/ex-of-code? (:error out) :params-validation))))
|
|
|
|
(t/deftest create-file-media-object-from-url-rejects-client-id
|
|
(let [prof (th/create-profile* 1)
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
sent-id (uuid/next)
|
|
out (th/command! {::th/type :create-file-media-object-from-url
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:url "https://example.com/sample.jpg"
|
|
:id sent-id})]
|
|
(t/is (th/ex-info? (:error out)))
|
|
(t/is (th/ex-of-type? (:error out) :validation))
|
|
(t/is (th/ex-of-code? (:error out) :params-validation))))
|
|
|
|
(t/deftest assemble-file-media-object-rejects-client-id
|
|
(let [prof (th/create-profile* 1)
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
sent-id (uuid/next)
|
|
out (th/command! {::th/type :assemble-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:session-id (uuid/next)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "assembled-image"
|
|
:mtype "image/jpeg"
|
|
:id sent-id})]
|
|
(t/is (th/ex-info? (:error out)))
|
|
(t/is (th/ex-of-type? (:error out) :validation))
|
|
(t/is (th/ex-of-code? (:error out) :params-validation))))
|
|
|
|
(t/deftest media-audit-event-carries-team-id
|
|
;; The media event carries the file id, so the resolver could find the team;
|
|
;; the command already resolved the team for the quota check and sets it.
|
|
(let [prof (th/create-profile* 1)
|
|
team (th/create-team* 1 {:profile-id (:id prof)})
|
|
project (th/create-project* 1 {:profile-id (:id prof)
|
|
:team-id (:id team)})
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:id project)
|
|
:is-shared false})
|
|
mfile {:filename "sample.jpg"
|
|
:path (th/tempfile "backend_tests/test_files/sample.jpg")
|
|
:mtype "image/jpeg"
|
|
:size 312043}
|
|
out (th/multipart-command! {::th/type :upload-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "testfile"
|
|
:content mfile})]
|
|
(t/is (th/success? out))
|
|
(t/is (= (:id team)
|
|
(get-in (meta (:result out)) [:app.loggers.audit/replace-props :team-id])))))
|
|
|
|
(t/deftest upload-file-media-object-rejects-forged-content
|
|
;; A transit body like {"~:content": {"~:path": ["~#path", "/etc/passwd"]}}
|
|
;; decodes to this exact map. Without a multipart request it must fail at
|
|
;; params validation, before any file is read.
|
|
(let [prof (th/create-profile* 1)
|
|
file (th/create-file* 1 {:profile-id (:id prof)
|
|
:project-id (:default-project-id prof)
|
|
:is-shared false})
|
|
out (th/command! {::th/type :upload-file-media-object
|
|
::rpc/profile-id (:id prof)
|
|
:file-id (:id file)
|
|
:is-local true
|
|
:name "testfile"
|
|
:content {:filename "evil.jpg"
|
|
:size 123
|
|
:path (fs/path "/etc/passwd")
|
|
:mtype "image/jpeg"}})]
|
|
(t/is (th/ex-of-code? (:error out) :params-validation))))
|
|
|
|
(t/deftest upload-chunk-rejects-forged-content
|
|
;; Same transit-forgery vector through the chunk primitive: without a
|
|
;; multipart request the wrapper must reject it at params validation.
|
|
(let [prof (th/create-profile* 1)
|
|
out (th/command! {::th/type :upload-chunk
|
|
::rpc/profile-id (:id prof)
|
|
:session-id (uuid/random)
|
|
:index 0
|
|
:content {:filename "chunk"
|
|
:size 123
|
|
:path (fs/path "/etc/passwd")
|
|
:mtype "image/jpeg"}})]
|
|
(t/is (th/ex-of-code? (:error out) :params-validation))))
|