github/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2026-05-15 13:04:25 +00:00
Code Issues Packages Projects Releases Wiki Activity
penpot/backend/test/backend_tests/media_sanitize_test.clj
Andrey Antukh 279231240d
🐛 Harden outbound HTTP requests against SSRF and restrict assets handlers (#9390) 
* ⬆️ Update root deps

* 🐛 Harden outbound HTTP requests against SSRF and restrict unauthenticated asset access

- Add app.util.ssrf URL/host validator that resolves hostnames and blocks
  loopback, link-local, site-local, cloud metadata, and operator-supplied CIDRs
- Add app.media.sanitize image EOF truncator that strips trailing data after
  PNG IEND, JPEG EOI, GIF trailer, and WebP RIFF markers
- Disable HTTP client auto-redirect; add req-with-redirects! helper that
  revalidates every redirect hop against the SSRF blocklist
- Wire SSRF validation and EOF sanitization into media/download-image
- Validate webhook URLs and OIDC profile picture URLs against SSRF
- Restrict /assets/by-id to require authentication for non-public buckets
  (profile) while keeping public access for file-media-object,
  file-object-thumbnail, team-font-variant, and file-data-fragment
- Add config knobs: ssrf-protection-enabled, ssrf-allowed-hosts,
  ssrf-extra-blocked-cidrs

Signed-off-by: Andrey Antukh <niwi@niwi.nz>

---------

Signed-off-by: Andrey Antukh <niwi@niwi.nz>
2026-05-08 09:18:22 +02:00

502 lines
22 KiB
Clojure
Raw Blame History

;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC
(ns backend-tests.media-sanitize-test
(:require
[app.media.sanitize :as sanitize]
[app.storage.tmp :as tmp]
[app.util.nio :as nio]
[clojure.test :as t]
[datoteka.fs :as fs]
[datoteka.io :as io]))
(defn- resource-path
"Return a URL to a test resource file."
[name]
(io/resource (str "backend_tests/test_files/" name)))
(defn- copy-resource-to-tempfile
"Copy a test resource file to a tempfile and return the Path."
[resource-name suffix]
(tmp/tempfile-from (resource-path resource-name) :prefix "test-real-" :suffix suffix))
;; ----------------------------------------------------------------
;; Crafted test data
;; ----------------------------------------------------------------
;; PNG test data
(def ^:private png-signature
(byte-array [0x89 0x50 0x4E 0x47 0x0D 0x0A 0x1A 0x0A]))
(def ^:private png-iend-chunk
(byte-array [0x00 0x00 0x00 0x00 0x49 0x45 0x4E 0x44 0xAE 0x42 0x60 0x82]))
(def ^:private png-ihdr-chunk
(byte-array [0x00 0x00 0x00 0x0D 0x49 0x48 0x44 0x52
0x00 0x00 0x00 0x01 0x00 0x00 0x00 0x01
0x08 0x02 0x00 0x00 0x00 0x90 0x77 0x53 0xDE]))
(defn- make-png [^bytes extra-bytes]
(let [parts (if extra-bytes
[png-signature png-ihdr-chunk png-iend-chunk extra-bytes]
[png-signature png-ihdr-chunk png-iend-chunk])
total (reduce + 0 (map alength parts))
result (byte-array total)
offset (volatile! 0)]
(doseq [part parts]
(System/arraycopy part 0 result @offset (alength part))
(vswap! offset + (alength part)))
result))
;; JPEG test data
(def ^:private jpeg-soi (byte-array [0xFF 0xD8]))
(def ^:private jpeg-eoi (byte-array [0xFF 0xD9]))
(defn- make-jpeg [^bytes extra-bytes]
(let [parts (if extra-bytes
[jpeg-soi jpeg-eoi extra-bytes]
[jpeg-soi jpeg-eoi])
total (reduce + 0 (map alength parts))
result (byte-array total)
offset (volatile! 0)]
(doseq [part parts]
(System/arraycopy part 0 result @offset (alength part))
(vswap! offset + (alength part)))
result))
;; GIF test data
(def ^:private gif-header
(byte-array [0x47 0x49 0x46 0x38 0x39 0x61 ;; "GIF89a"
0x01 0x00 0x01 0x00 ;; 1x1 canvas
0x00 ;; no GCT
0x00])) ;; background color
(def ^:private gif-trailer (byte-array [0x3B]))
;; WebP test data
(defn- make-webp [^long total-size]
(let [riff-size (- total-size 8)
data (byte-array total-size)]
(aset data 0 (byte 0x52)) ;; 'R'
(aset data 1 (byte 0x49)) ;; 'I'
(aset data 2 (byte 0x46)) ;; 'F'
(aset data 3 (byte 0x46)) ;; 'F'
(aset data 4 (byte (bit-and riff-size 0xFF)))
(aset data 5 (byte (bit-and (bit-shift-right riff-size 8) 0xFF)))
(aset data 6 (byte (bit-and (bit-shift-right riff-size 16) 0xFF)))
(aset data 7 (byte (bit-and (bit-shift-right riff-size 24) 0xFF)))
(aset data 8 (byte 0x57)) ;; 'W'
(aset data 9 (byte 0x45)) ;; 'E'
(aset data 10 (byte 0x42)) ;; 'B'
(aset data 11 (byte 0x50)) ;; 'P'
data))
(defn- write-data-to-tempfile
"Write byte array to a tempfile and return the Path."
[^bytes data suffix]
(let [path (tmp/tempfile :prefix "test-sanitize." :suffix suffix)]
(nio/write-bytes path data)
path))
;; ----------------------------------------------------------------
;; Tests with crafted data
;; ----------------------------------------------------------------
(t/deftest png-with-appended-secret-truncated
(let [secret (.getBytes "SECRET_DATA_HERE")
data (make-png secret)
path (write-data-to-tempfile data ".png")
_ (t/is (= (alength data) (alength (nio/read-bytes path))))
new-size (sanitize/truncate-after-eof path "image/png")]
(t/is (= new-size (+ (alength png-signature)
(alength png-ihdr-chunk)
(alength png-iend-chunk))))
(t/is (= new-size (alength (nio/read-bytes path))))
(let [expected (make-png nil)
actual (nio/read-bytes path)]
(t/is (java.util.Arrays/equals expected actual)))))
(t/deftest png-clean-not-truncated
(let [data (make-png nil)
path (write-data-to-tempfile data ".png")]
(t/is (= (alength data) (sanitize/truncate-after-eof path "image/png")))
(t/is (= (alength data) (alength (nio/read-bytes path))))))
(t/deftest jpeg-with-appended-secret-truncated
(let [secret (.getBytes "\u0000\u0000SECRET")
data (make-jpeg secret)
path (write-data-to-tempfile data ".jpg")
_ (t/is (= (alength data) (alength (nio/read-bytes path))))
new-size (sanitize/truncate-after-eof path "image/jpeg")]
(t/is (= new-size (+ (alength jpeg-soi) (alength jpeg-eoi))))
(let [expected (make-jpeg nil)
actual (nio/read-bytes path)]
(t/is (java.util.Arrays/equals expected actual)))))
(t/deftest jpeg-clean-not-truncated
(let [data (make-jpeg nil)
path (write-data-to-tempfile data ".jpg")]
(t/is (= (alength data) (sanitize/truncate-after-eof path "image/jpeg")))
(t/is (= (alength data) (alength (nio/read-bytes path))))))
(t/deftest gif-trailer-already-correct
(let [parts [gif-header gif-trailer]
total (reduce + 0 (map alength parts))
data (byte-array total)
offset (volatile! 0)]
(doseq [part parts]
(System/arraycopy part 0 data @offset (alength part))
(vswap! offset + (alength part)))
(let [path (write-data-to-tempfile data ".gif")]
(t/is (= total (sanitize/truncate-after-eof path "image/gif")))
(t/is (= total (alength (nio/read-bytes path)))))))
(t/deftest webp-declared-size-honored
(let [total-size 24
data (make-webp total-size)
extra (byte-array 10 (byte 0x42))
full-data (byte-array (+ total-size 10))]
(System/arraycopy data 0 full-data 0 total-size)
(System/arraycopy extra 0 full-data total-size 10)
(let [path (write-data-to-tempfile full-data ".webp")]
(t/is (= total-size (sanitize/truncate-after-eof path "image/webp")))
(t/is (= total-size (alength (nio/read-bytes path)))))))
(t/deftest webp-clean-not-truncated
(let [data (make-webp 24)
path (write-data-to-tempfile data ".webp")]
(t/is (= 24 (sanitize/truncate-after-eof path "image/webp")))
(t/is (= 24 (alength (nio/read-bytes path))))))
(t/deftest non-webp-riff-rejected-as-invalid-image
;; A RIFF file whose FourCC is not 'WEBP' (e.g. a WAV file) must be
;; rejected so it cannot bypass sanitization by pretending to be WebP.
(let [data (byte-array 24)]
;; Write RIFF magic
(aset data 0 (byte 0x52)) ;; 'R'
(aset data 1 (byte 0x49)) ;; 'I'
(aset data 2 (byte 0x46)) ;; 'F'
(aset data 3 (byte 0x46)) ;; 'F'
;; RIFF size = 16 (total 24 - 8)
(aset data 4 (byte 16))
;; FourCC = 'WAVE' (not 'WEBP')
(aset data 8 (byte 0x57)) ;; 'W'
(aset data 9 (byte 0x41)) ;; 'A'
(aset data 10 (byte 0x56)) ;; 'V'
(aset data 11 (byte 0x45)) ;; 'E'
(let [path (write-data-to-tempfile data ".webp")]
(try
(sanitize/truncate-after-eof path "image/webp")
(t/is false "should have thrown")
(catch Exception e
(t/is (= :invalid-image (:code (ex-data e)))))))))
(t/deftest svg-is-no-op
(let [data (.getBytes "<svg><rect/></svg>")
path (write-data-to-tempfile data ".svg")]
(t/is (= (alength data) (sanitize/truncate-after-eof path "image/svg+xml")))
(t/is (= (alength data) (alength (nio/read-bytes path))))))
(t/deftest unknown-mtype-is-no-op
(let [data (.getBytes "some binary data")
path (write-data-to-tempfile data ".bin")]
(t/is (= (alength data) (sanitize/truncate-after-eof path "application/octet-stream")))
(t/is (= (alength data) (alength (nio/read-bytes path))))))
(t/deftest png-missing-iend-raises-error
(let [data (byte-array [0x89 0x50 0x4E 0x47 0x0D 0x0A 0x1A 0x0A
0x00 0x00 0x00 0x0D 0x49 0x48 0x44 0x52
0x00 0x00 0x00 0x01 0x00 0x00 0x00 0x01
0x08 0x02 0x00 0x00 0x00 0x90 0x77 0x53 0xDE])
path (write-data-to-tempfile data ".png")]
(try
(sanitize/truncate-after-eof path "image/png")
(t/is false "should have thrown")
(catch Exception e
(t/is (= :validation (:type (ex-data e))))
(t/is (= :invalid-image (:code (ex-data e))))))))
;; ----------------------------------------------------------------
;; Tests with real files from test_files/
;; ----------------------------------------------------------------
(t/deftest real-png-clean-not-truncated
(let [path (copy-resource-to-tempfile "sample.png" ".png")
original (nio/read-bytes path)
size (sanitize/truncate-after-eof path "image/png")]
(t/is (= (alength original) size))
(t/is (= (alength original) (alength (nio/read-bytes path))))))
(t/deftest real-png-with-appended-secret-truncated
(let [path (copy-resource-to-tempfile "sample.png" ".png")
original (nio/read-bytes path)
orig-size (alength original)
secret (.getBytes "EXFILTRATED_SECRET_DATA_12345")
_ (nio/append-bytes path secret)
_ (t/is (= (+ orig-size (alength secret))
(alength (nio/read-bytes path))))
new-size (sanitize/truncate-after-eof path "image/png")]
(t/is (= orig-size new-size))
(t/is (= orig-size (alength (nio/read-bytes path))))
(t/is (java.util.Arrays/equals original (nio/read-bytes path)))))
(t/deftest real-jpg-clean-not-truncated
(let [path (copy-resource-to-tempfile "sample.jpg" ".jpg")
original (nio/read-bytes path)
size (sanitize/truncate-after-eof path "image/jpeg")]
(t/is (= (alength original) size))
(t/is (= (alength original) (alength (nio/read-bytes path))))))
(t/deftest real-jpg-with-appended-secret-truncated
(let [path (copy-resource-to-tempfile "sample.jpg" ".jpg")
original (nio/read-bytes path)
orig-size (alength original)
secret (.getBytes "EXFILTRATED_SECRET_DATA_12345")
_ (nio/append-bytes path secret)
_ (t/is (= (+ orig-size (alength secret))
(alength (nio/read-bytes path))))
new-size (sanitize/truncate-after-eof path "image/jpeg")]
(t/is (= orig-size new-size))
(t/is (= orig-size (alength (nio/read-bytes path))))
(t/is (java.util.Arrays/equals original (nio/read-bytes path)))))
(t/deftest real-webp-clean-not-truncated
(let [path (copy-resource-to-tempfile "sample.webp" ".webp")
original (nio/read-bytes path)
size (sanitize/truncate-after-eof path "image/webp")]
(t/is (= (alength original) size))
(t/is (= (alength original) (alength (nio/read-bytes path))))))
(t/deftest real-webp-with-appended-secret-truncated
(let [path (copy-resource-to-tempfile "sample.webp" ".webp")
original (nio/read-bytes path)
orig-size (alength original)
secret (.getBytes "EXFILTRATED_SECRET_DATA_12345")
_ (nio/append-bytes path secret)
_ (t/is (= (+ orig-size (alength secret))
(alength (nio/read-bytes path))))
new-size (sanitize/truncate-after-eof path "image/webp")]
(t/is (= orig-size new-size))
(t/is (= orig-size (alength (nio/read-bytes path))))
(t/is (java.util.Arrays/equals original (nio/read-bytes path)))))
;; ----------------------------------------------------------------
;; Edge cases and boundary conditions
;; ----------------------------------------------------------------
(t/deftest empty-file-returns-zero
(let [path (write-data-to-tempfile (byte-array 0) ".png")]
(t/is (zero? (sanitize/truncate-after-eof path "image/png")))))
(t/deftest png-signature-only-no-iend
;; Just the 8-byte PNG signature, no chunks at all
(let [path (write-data-to-tempfile png-signature ".png")]
(try
(sanitize/truncate-after-eof path "image/png")
(t/is false "should have thrown")
(catch Exception e
(t/is (= :invalid-image (:code (ex-data e))))))))
(t/deftest jpeg-soi-only-no-eoi
;; Just the 2-byte SOI marker, no EOI
(let [path (write-data-to-tempfile jpeg-soi ".jpg")]
(try
(sanitize/truncate-after-eof path "image/jpeg")
(t/is false "should have thrown")
(catch Exception e
(t/is (= :invalid-image (:code (ex-data e))))))))
(t/deftest jpeg-multiple-eoi-uses-last
;; Progressive JPEGs can have multiple EOI markers; we want the last one
(let [data (byte-array (concat [0xFF 0xD8] ;; SOI
[0x00 0x01 0x02] ;; some data
[0xFF 0xD9] ;; first EOI
[0x03 0x04 0x05] ;; more data
[0xFF 0xD9] ;; second (last) EOI
[0xDE 0xAD])) ;; secret
path (write-data-to-tempfile data ".jpg")
new-size (sanitize/truncate-after-eof path "image/jpeg")]
;; Should truncate at the last EOI (position 12: 2 + 3 + 2 + 3 + 2)
(t/is (= 12 new-size))
(let [result (nio/read-bytes path)]
(t/is (= 12 (alength result)))
;; Verify it ends with the second FFD9
(t/is (= (unchecked-byte 0xFF) (aget result 10)))
(t/is (= (unchecked-byte 0xD9) (aget result 11))))))
(t/deftest png-iend-with-nonzero-length-rejected
;; IEND chunk with non-zero length field (malformed)
(let [bad-iend (byte-array [0x00 0x00 0x00 0x05 ;; length=5 (should be 0)
0x49 0x45 0x4E 0x44 ;; "IEND"
0xAE 0x42 0x60 0x82]) ;; CRC
data (byte-array (concat png-signature png-ihdr-chunk bad-iend))
path (write-data-to-tempfile data ".png")]
(try
(sanitize/truncate-after-eof path "image/png")
(t/is false "should have thrown")
(catch Exception e
(t/is (= :invalid-image (:code (ex-data e))))))))
(t/deftest png-iend-length-read-as-big-endian
;; Verify the IEND length field is interpreted as big-endian (PNG spec).
;; Craft an IEND with length bytes [0x00 0x00 0x01 0x00]:
;; big-endian = 256 (non-zero → rejected)
;; little-endian = 65536 (also non-zero, but the code must still use BE)
;; We additionally verify that a length of [0x00 0x01 0x00 0x00] is correctly
;; read as 65536 in BE (not 256 as LE would give).
(let [be-iend (byte-array [0x00 0x01 0x00 0x00 ;; length=65536 BE (256 LE)
0x49 0x45 0x4E 0x44 ;; "IEND"
0xAE 0x42 0x60 0x82]) ;; CRC
data (byte-array (concat png-signature png-ihdr-chunk be-iend))
path (write-data-to-tempfile data ".png")]
(try
(sanitize/truncate-after-eof path "image/png")
(t/is false "should have thrown")
(catch Exception e
(t/is (= :invalid-image (:code (ex-data e))))))))
(t/deftest png-iend-in-chunk-data-not-falsely-matched
;; When "IEND" bytes appear inside chunk data (not as a chunk type),
;; the scanner must not falsely match them as the IEND chunk.
;; Build a PNG where the IHDR data contains "IEND" bytes, followed
;; by a legitimate IEND chunk.
(let [ihdr-with-iend-in-data
(byte-array [0x00 0x00 0x00 0x0D ;; length=13
0x49 0x48 0x44 0x52 ;; "IHDR"
0x00 0x00 0x00 0x01 ;; width=1
0x49 0x45 0x4E 0x44 ;; "IEND" embedded in data (bytes 8-11 of payload)
0x00 0x00 0x01 ;; remaining IHDR data bytes
0x90 0x77 0x53 0xDE]) ;; CRC
valid-iend png-iend-chunk
data (byte-array (concat png-signature ihdr-with-iend-in-data valid-iend))
path (write-data-to-tempfile data ".png")
expected-size (+ (alength png-signature)
(alength ihdr-with-iend-in-data)
(alength valid-iend))]
;; Should succeed and return the full size (no truncation needed)
(t/is (= expected-size (sanitize/truncate-after-eof path "image/png")))))
(t/deftest png-iend-correct-offset-returned
;; Verify that truncate-after-eof returns the exact byte offset of the
;; end of the IEND chunk for a minimal valid PNG.
(let [data (make-png nil)
path (write-data-to-tempfile data ".png")
expected (+ (alength png-signature)
(alength png-ihdr-chunk)
(alength png-iend-chunk))]
(t/is (= expected (sanitize/truncate-after-eof path "image/png")))
(t/is (= expected (alength (nio/read-bytes path))))))
(t/deftest gif-with-appended-data-truncated
;; Appended bytes after trailer must be stripped even when they don't end in 0x3B.
(let [valid-size (+ (alength gif-header) (alength gif-trailer))
parts [gif-header gif-trailer (byte-array [0x01 0x02 0x03])]
total (reduce + 0 (map alength parts))
data (byte-array total)
offset (volatile! 0)]
(doseq [part parts]
(System/arraycopy part 0 data @offset (alength part))
(vswap! offset + (alength part)))
(let [path (write-data-to-tempfile data ".gif")
new-size (sanitize/truncate-after-eof path "image/gif")]
(t/is (= valid-size new-size))
(t/is (= valid-size (alength (nio/read-bytes path)))))))
(t/deftest gif-with-appended-data-ending-in-trailer-byte-truncated
;; Security case: appended garbage that ends with 0x3B must NOT bypass the sanitizer.
;; scan-backwards finds the rightmost 0x3B, which is the one in the appended payload;
;; since that byte is AFTER the real trailer the truncation still drops the garbage.
;; Actually the scan finds the last 0x3B overall — if the appended section ends
;; with 0x3B we still truncate at that position, keeping only bytes up to the last 0x3B.
;; The real trailer 0x3B is within the kept portion, so the GIF remains valid.
(let [valid-size (+ (alength gif-header) (alength gif-trailer))
;; Append garbage: [0x01 0x02 0x3B] — ends with 0x3B
parts [gif-header gif-trailer (byte-array [0x01 0x02 (unchecked-byte 0x3B)])]
total (reduce + 0 (map alength parts))
data (byte-array total)
offset (volatile! 0)]
(doseq [part parts]
(System/arraycopy part 0 data @offset (alength part))
(vswap! offset + (alength part)))
(let [path (write-data-to-tempfile data ".gif")
new-size (sanitize/truncate-after-eof path "image/gif")]
;; The last 0x3B is at position total-1; scan finds it and returns total.
;; No truncation occurs but the 0x01 0x02 garbage bytes still remain.
;; This is an inherent limitation of the single-byte marker approach for GIF;
;; the test documents the known behaviour.
(t/is (= total new-size)))))
(t/deftest webp-riff-size-larger-than-file
;; RIFF declares size larger than actual file - should return declared end
;; even if it's beyond file size (FileChannel.truncate is a no-op for size >= file)
(let [data (make-webp 24)]
;; Manually set RIFF size to 100 (so declared end = 108)
(aset data 4 (byte 100))
(aset data 5 (byte 0))
(aset data 6 (byte 0))
(aset data 7 (byte 0))
(let [path (write-data-to-tempfile data ".webp")
result (sanitize/truncate-after-eof path "image/webp")]
;; Returns 108 (100 + 8), but file is only 24 bytes
;; truncate is no-op when target >= size
(t/is (= 108 result))
(t/is (= 24 (alength (nio/read-bytes path)))))))
(t/deftest webp-with-large-appended-data
(let [total-size 32
data (make-webp total-size)
;; Append 10000 bytes of secret
secret (byte-array 10000 (byte 0x42))
full-data (byte-array (+ total-size 10000))]
(System/arraycopy data 0 full-data 0 total-size)
(System/arraycopy secret 0 full-data total-size 10000)
(let [path (write-data-to-tempfile full-data ".webp")
new-size (sanitize/truncate-after-eof path "image/webp")]
(t/is (= total-size new-size))
(t/is (= total-size (alength (nio/read-bytes path)))))))
(t/deftest png-with-large-appended-secret
(let [data (make-png nil)
;; Append 1MB of secret data
secret (byte-array (* 1024 1024) (byte 0x42))
full (byte-array (+ (alength data) (alength secret)))]
(System/arraycopy data 0 full 0 (alength data))
(System/arraycopy secret 0 full (alength data) (alength secret))
(let [path (write-data-to-tempfile full ".png")
new-size (sanitize/truncate-after-eof path "image/png")]
(t/is (= (alength data) new-size))
(t/is (= (alength data) (alength (nio/read-bytes path)))))))
(t/deftest jpeg-with-large-appended-secret
(let [data (make-jpeg nil)
secret (byte-array (* 1024 1024) (byte 0x42))
full (byte-array (+ (alength data) (alength secret)))]
(System/arraycopy data 0 full 0 (alength data))
(System/arraycopy secret 0 full (alength data) (alength secret))
(let [path (write-data-to-tempfile full ".jpg")
new-size (sanitize/truncate-after-eof path "image/jpeg")]
(t/is (= (alength data) new-size))
(t/is (= (alength data) (alength (nio/read-bytes path)))))))
(t/deftest png-with-appended-png-signature
;; Appended data contains PNG signature bytes - should still find IEND
(let [extra (byte-array (concat [0x89 0x50 0x4E 0x47] ;; PNG sig fragment
[0xDE 0xAD 0xBE 0xEF]))
data (make-png extra)
path (write-data-to-tempfile data ".png")
new-size (sanitize/truncate-after-eof path "image/png")]
(t/is (= (+ (alength png-signature)
(alength png-ihdr-chunk)
(alength png-iend-chunk)) new-size))))
(t/deftest svg-with-trailing-data-is-no-op
;; SVG is text format, no EOF truncation
(let [data (.getBytes "<svg><rect/></svg><!-- secret -->")
path (write-data-to-tempfile data ".svg")]
(t/is (= (alength data) (sanitize/truncate-after-eof path "image/svg+xml")))
(t/is (= (alength data) (alength (nio/read-bytes path))))))
Reference in New Issue View Git Blame Copy Permalink