;; This Source Code Form is subject to the terms of the Mozilla Public ;; License, v. 2.0. If a copy of the MPL was not distributed with this ;; file, You can obtain one at http://mozilla.org/MPL/2.0/. ;; ;; Copyright (c) KALEIDOS SUBSIDIARY SL (ns app.storage "Objects storage abstraction layer." (:refer-clojure :exclude [resolve]) (:require [app.common.data :as d] [app.common.data.macros :as dm] [app.common.logging :as l] [app.common.schema :as sm] [app.common.time :as ct] [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] [app.metrics :as mtx] [app.storage.fs :as sfs] [app.storage.impl :as impl] [app.storage.s3 :as ss3] [app.storage.schema :as stsch] [cuerdas.core :as str] [datoteka.fs :as fs] [integrant.core :as ig]) (:import java.io.InputStream)) (defn get-legacy-backend [] (when-let [name (cf/get :assets-storage-backend)] (l/wrn :hint "using deprecated configuration, please read 2.11 release notes" :href "https://github.com/penpot/penpot/releases/tag/2.11.0") (case name :assets-fs :fs :assets-s3 :s3 nil))) (def default-bucket stsch/default-bucket) (def tempfile-bucket "Bucket name for temporary file uploads (10-minute expiry)." stsch/tempfile-bucket) (def upload-session-bucket "Bucket name for chunked-upload chunks." stsch/upload-session-bucket) (def job-resource-bucket "Bucket for storage objects referenced by job.resource_id (the unified jobs substrate): transient artifacts owned by their job row, reclaimed by storage-gc-touched (via the jobs GC touch) once no job row references them anymore." stsch/job-resource-bucket) (def valid-buckets stsch/metadata-buckets) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Storage Module State ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (def ^:private schema:backends [:map-of :keyword [:maybe [:or ::ss3/backend ::sfs/backend]]]) (def ^:private valid-backends? (sm/validator schema:backends)) (def ^:private schema:storage [:map {:title "storage"} [::backends schema:backends] [::backend [:enum :s3 :fs]] [::mtx/metrics ::mtx/metrics] ::db/pool]) (def valid-storage? (sm/validator schema:storage)) (sm/register! ::storage schema:storage) (defmethod ig/assert-key ::storage [_ params] (assert (db/pool? (::db/pool params)) "expected valid database pool") (assert (valid-backends? (::backends params)) "expected valid backends map")) (defmethod ig/init-key ::storage [_ {:keys [::backends ::db/pool] :as cfg}] (let [backend (or (get-legacy-backend) (cf/get :objects-storage-backend) :fs) backends (d/without-nils backends)] (l/dbg :hint "initialize" :default (d/name backend) :available (str/join "," (map d/name (keys backends)))) (-> (d/without-nils cfg) (assoc ::backends backends) (assoc ::backend backend) (assoc ::db/pool pool)))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Database Objects ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (defn get-metadata [params] (reduce-kv (fn [res k _] (if (qualified-keyword? k) (dissoc res k) res)) params params)) ;; Matches both metadata encodings while Transit rows still exist. The ;; UNION ALL keeps each branch indexable: migration 0068 covers the legacy ;; "~:" keys and 0156 covers the plain JSON keys. Phase 3 drops the legacy ;; branch once no "~:" rows remain. (def ^:private sql:get-database-object-by-hash "( select * from storage_object where (metadata->>'~:hash') = ? and (metadata->>'~:bucket') = ? and backend = ? and deleted_at is null and status = 'valid' limit 1 ) union all ( select * from storage_object where (metadata->>'hash') = ? and (metadata->>'bucket') = ? and backend = ? and deleted_at is null and status = 'valid' limit 1 ) limit 1") ;; NOTE: metadata is left encoded; row->storage-object is responsible for ;; decoding it. (defn- get-database-object-by-hash [connectable backend bucket hash] (let [backend (name backend)] (db/exec-one! connectable [sql:get-database-object-by-hash hash bucket backend hash bucket backend]))) (defn- promote-object! [storage object] (let [ds (db/get-connectable storage) res (-> (db/update! ds :storage-object {:status "valid"} {:id (:id object)} {::db/return-keys false}) (db/get-update-count))] (when-not (pos? res) ;; The pending row disappeared while the blob was being written ;; (e.g. reclaimed by :storage-pending-gc); make it observable. (l/wrn :hint "unable to promote storage object, pending row not found" :id (str (:id object)))) res)) (defn row->storage-object [res] (let [mdata (or (some-> (:metadata res) (stsch/decode-metadata)) {})] (impl/storage-object (:id res) (:size res) (:created-at res) (:deleted-at res) (:touched-at res) (keyword (:backend res)) mdata))) (def ^:private sql:get-storage-object "SELECT * FROM storage_object WHERE id = ? AND (deleted_at IS NULL) AND status = 'valid'") (defn- get-database-object [conn id] (some-> (db/exec-one! conn [sql:get-storage-object id]) (row->storage-object))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; API ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (defn object->relative-path [{:keys [id] :as obj}] (impl/id->path id)) (defn file-url->path [url] (when url (fs/path (java.net.URI. (str url))))) (dm/export impl/content) (dm/export impl/wrap-with-hash) (dm/export impl/object?) (defn- emit-op! "Record a logical storage operation. Recording never fails: metrics must not change storage behavior." ([storage op bucket] (emit-op! storage op bucket nil)) ([storage op bucket object] (mtx/run! (::mtx/metrics storage) :id :storage-operations :inc 1 :labels [op (mtx/label bucket "unknown") (mtx/label (or (some-> object :backend) (::backend storage)) "unknown")]))) (defn- emit-dedup! "Record a deduplication outcome. Recording never fails." [storage result bucket] (mtx/run! (::mtx/metrics storage) :id :storage-dedup :inc 1 :labels [(name result) (mtx/label bucket "unknown")])) (defn get-object [storage id] (assert (valid-storage? storage)) (let [ds (db/get-connectable storage)] (get-database-object ds id))) (defn put-object! "Creates a new object with the provided content." [{:keys [::backend ::db/pool] :as storage} {:keys [::content ::expired-at ::touched-at ::touch] :as params}] (assert (valid-storage? storage)) (assert (impl/content? content) "expected an instance of content") (let [id (or (::id params) (uuid/random)) mdata (cond-> (get-metadata params) (satisfies? impl/IContentHash content) (assoc :hash (impl/get-hash content))) touched-at (if touch (or touched-at (ct/now)) touched-at) backend' (impl/resolve-backend storage backend) bucket (:bucket mdata) ;; `job-resource` objects belong to one profile, so they are ;; never shared: a dedup hit would hand the artifact of one ;; profile to the job of another. dedupable? (and (::deduplicate? params) (:hash mdata) (some? bucket) (not= tempfile-bucket bucket) (not= upload-session-bucket bucket) (not= job-resource-bucket bucket)) hit (when dedupable? (get-database-object-by-hash pool backend bucket (:hash mdata)))] ;; NOTE: for now we don't reuse the deleted objects, but in futute ;; we can consider reusing deleted objects if we found a duplicated ;; one and is marked for deletion but still not deleted. ;; PHASE 1: deduplication lookup (see `dedupable?` and `hit` above). (if-some [hit hit] ;; PHASE 2: an existing reference is found: reuse or repair it. ;; The `exists` op is emitted only after a successful probe so every ;; count stays paired with its `hit`/`repair` outcome. (if (impl/exists-object? backend' hit) ;; PHASE 2a: healthy reference. Optionally refresh touched_at ;; and reuse the object as it is. (do (emit-op! storage "exists" bucket) (emit-dedup! storage :hit bucket) (when touch (db/update! pool :storage-object {:touched-at touched-at} {:id (:id hit)} {::db/return-keys false})) (row->storage-object (cond-> hit touch (assoc :touched-at touched-at)))) ;; PHASE 2b: the referenced blob is missing (a stale/broken row). ;; Repair the reference in place: rewrite the incoming content ;; under the same id, restoring the blob for all existing ;; references to it. If the write fails, the exception propagates ;; and the row stays live and valid, so a later matching upload ;; retries the heal. (let [object (row->storage-object hit)] (l/wrn :hint "blob not found on reusing storage object" :id (:id object) :backend (name backend)) (impl/put-object backend' object content) (emit-op! storage "exists" bucket) (emit-op! storage "repair" bucket) (emit-dedup! storage :repair bucket) (promote-object! storage object) object)) ;; PHASE 3: no dedup hit: create a fresh object. The row is ;; inserted in 'pending' state so it is not visible to the normal ;; lifecycle (dedup, gc, reads) until the blob has been written ;; and the object promoted to 'valid'. (let [row (db/insert! pool :storage-object {:id id :size (impl/get-size content) :backend (name backend) :metadata (stsch/encode-metadata mdata) :deleted-at expired-at :touched-at touched-at :status "pending"}) object (row->storage-object row)] (impl/put-object backend' object content) (emit-op! storage "put" bucket) (emit-dedup! storage (if dedupable? :miss :skip) bucket) (promote-object! storage object) object)))) (defn touch-object! "Mark object as touched. Takes the object id (UUID). The metric labels come from the updated row itself (RETURNING); no row, no metric." [storage id] (assert (valid-storage? storage)) (let [ds (db/get-connectable storage) res (db/update! ds :storage-object {:touched-at (ct/now)} {:id id} {::db/return-keys [:id :backend :metadata]})] (when-some [object (some-> res row->storage-object)] (emit-op! storage "touch" (-> object meta :bucket) object)) (some? res))) (defn get-object-data "Return an input stream instance of the object content." ^InputStream [storage object] (assert (valid-storage? storage)) (when (or (nil? (:expired-at object)) (ct/is-after? (:expired-at object) (ct/now))) (let [result (-> (impl/resolve-backend storage (:backend object)) (impl/get-object-data object))] (emit-op! storage "get-data" (-> object meta :bucket) object) result))) (defn get-object-bytes "Returns a byte array of object content." [storage object] (assert (valid-storage? storage)) (when (or (nil? (:expired-at object)) (ct/is-after? (:expired-at object) (ct/now))) (let [result (-> (impl/resolve-backend storage (:backend object)) (impl/get-object-bytes object))] (emit-op! storage "get-bytes" (-> object meta :bucket) object) result))) (defn get-object-url ([storage object] (get-object-url storage object nil)) ([storage object options] (assert (valid-storage? storage)) (when (or (nil? (:expired-at object)) (ct/is-after? (:expired-at object) (ct/now))) (-> (impl/resolve-backend storage (:backend object)) (impl/get-object-url object options))))) (defn get-object-path "Get the Path to the object. Only works with `:fs` type of storages." [storage object] (assert (valid-storage? storage)) (let [backend (impl/resolve-backend storage (:backend object))] (when (and (= :fs (::type backend)) (or (nil? (:expired-at object)) (ct/is-after? (:expired-at object) (ct/now)))) (-> (impl/get-object-url backend object nil) file-url->path)))) (defn del-object! "Mark the object as deleted (soft delete: the backend content is removed by the GC). Takes the object id (UUID). Only a live row (deleted_at IS NULL) is deleted, so a repeated call is a no-op that returns false. The metric labels come from the updated row itself (RETURNING); no row, no metric." [storage id] (assert (valid-storage? storage)) (let [ds (db/get-connectable storage) res (db/update! ds :storage-object {:deleted-at (ct/now)} ["id = ? AND deleted_at IS NULL" id] {::db/return-keys [:id :backend :metadata]})] (when-some [object (some-> res row->storage-object)] (emit-op! storage "del" (-> object meta :bucket) object)) (some? res))) (dm/export impl/calculate-hash) (dm/export impl/get-hash) (dm/export impl/get-size) (defn configure [storage connection] (assert (db/connection? connection)) (assert (valid-storage? storage)) (assoc storage ::db/conn connection)) (defn resolve "Resolves the storage instance with preconfigured backend. You can specify to reuse the database connection from provided cfg/system (default false)." [cfg & {:as opts}] (let [storage (::storage cfg)] (if (::db/reuse-conn opts false) (configure storage (db/get-connection cfg)) storage)))