mirror of
https://github.com/penpot/penpot.git
synced 2026-10-29 22:10:34 +00:00
42f6c952e1
The media-storage-bytes-per-team quota ran a full SUM over storage_object on every upload, taking seconds on large teams. A new team_storage_usage table (migration 0158) caches the counter: a third audit-log consumer (storage-usage) selectively queues the events that change media storage, marks their teams dirty, and a scheduler turns the marks into per-team recompute jobs on a dedicated :storage queue. The quota reads the counter, always answers one row, and fails open when the row is missing; the three commands that write media bytes pay it. The dirty-team stream runs in one read-only transaction and submits each 50-team chunk through its own writable one (db/own-transaction), so a killed run keeps what it already enqueued and the next tick resumes the rest. The recompute only clears marks older than its read, and the query also returns the rows nobody refreshed for a week, so a lost mark heals at a bounded rate instead of never. Producers resolve team ids at emission with one batched lookup, so the drain never joins production tables: file-gc carries its team, the delete cascade reports flat and nested ids, moves carry explicit source/destination teams (mixed origins rejected), restores carry theirs, and purges carry only their entity key. Selective queueing is gated per consumer flag on both ingest paths, and telemetry shadows never queue the admin-console consumer. An audit event that cannot be stored now fails loudly instead of leaving a consumer stale. The delete cascade owns library relations: the RPC only marks the file deleted, and the job captures dependents, removes the relations and absorbs them under a savepoint, only for survivors. Jobs queued by the previous release still absorb through a deprecated :file-ids snapshot. Closes #12462 AI-assisted-by: muse-spark-1.3-contributor
109 lines
5.3 KiB
Clojure
109 lines
5.3 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
|
|
|
(ns backend-tests.delete-object-test
|
|
(:require
|
|
[app.common.time :as ct]
|
|
[app.common.uuid :as uuid]
|
|
[app.config :as cf]
|
|
[app.db :as db]
|
|
[backend-tests.helpers :as th]
|
|
[clojure.test :as t]
|
|
[mockery.core :refer [with-mocks]]))
|
|
|
|
(t/use-fixtures :once th/state-init)
|
|
(t/use-fixtures :each th/database-reset)
|
|
|
|
(defn- submitted-events
|
|
[audit-mock]
|
|
(->> (:call-args-list @audit-mock)
|
|
(map second)))
|
|
|
|
(defn- cascade-events
|
|
[audit-mock]
|
|
(filter #(= "delete-object-with-cascade" (:name %))
|
|
(submitted-events audit-mock)))
|
|
|
|
(t/deftest delete-object-team-cascade-emits-event
|
|
;; The cascade marks the team, its projects and their files; the event
|
|
;; carries every affected id so a projection can follow without reading
|
|
;; production tables.
|
|
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
|
|
(with-redefs [cf/flags (conj cf/flags :audit-log)]
|
|
(let [profile (th/create-profile* 1 {:is-active true})
|
|
team (th/create-team* 91 {:profile-id (:id profile)})
|
|
project (th/create-project* 91 {:profile-id (:id profile)
|
|
:team-id (:id team)})
|
|
file (th/create-file* 91 {:profile-id (:id profile)
|
|
:project-id (:id project)})
|
|
deleted-at (ct/now)]
|
|
(th/run-task! :delete-object {:object :team
|
|
:id (:id team)
|
|
:deleted-at deleted-at})
|
|
(t/is (some? (:deleted-at (th/db-get :team {:id (:id team)}
|
|
{::db/remove-deleted false}))))
|
|
(t/is (some? (:deleted-at (th/db-get :project {:id (:id project)}
|
|
{::db/remove-deleted false}))))
|
|
(t/is (some? (:deleted-at (th/db-get :file {:id (:id file)}
|
|
{::db/remove-deleted false}))))
|
|
(let [events (cascade-events audit-mock)]
|
|
(t/is (= 1 (count events)))
|
|
(t/is (= "action" (:type (first events))))
|
|
(let [props (:props (first events))]
|
|
(t/is (= "team" (name (:object props))))
|
|
(t/is (= (:id team) (:id props)))
|
|
(t/is (= deleted-at (:deleted-at props)))
|
|
;; Flat team ids for team-keyed consumers, next to the
|
|
;; per-entity cascade breakdown.
|
|
(t/is (= #{(:id team)} (set (:team-ids props))))
|
|
(t/is (= #{(:id team)} (set (get-in props [:cascade :team-ids]))))
|
|
(t/is (= #{(:default-project-id team) (:id project)}
|
|
(set (get-in props [:cascade :project-ids]))))
|
|
(t/is (= #{(:id file)} (set (get-in props [:cascade :file-ids]))))))))))
|
|
|
|
(t/deftest delete-object-file-cascade-emits-event
|
|
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
|
|
(with-redefs [cf/flags (conj cf/flags :audit-log)]
|
|
(let [profile (th/create-profile* 1 {:is-active true})
|
|
team (th/create-team* 92 {:profile-id (:id profile)})
|
|
project (th/create-project* 92 {:profile-id (:id profile)
|
|
:team-id (:id team)})
|
|
file (th/create-file* 92 {:profile-id (:id profile)
|
|
:project-id (:id project)})
|
|
deleted-at (ct/now)]
|
|
(th/run-task! :delete-object {:object :file
|
|
:id (:id file)
|
|
:deleted-at deleted-at})
|
|
(let [events (cascade-events audit-mock)]
|
|
(t/is (= 1 (count events)))
|
|
(t/is (= "action" (:type (first events))))
|
|
(let [props (:props (first events))]
|
|
(t/is (= "file" (name (:object props))))
|
|
(t/is (= (:id file) (:id props)))
|
|
(t/is (= [] (get-in props [:cascade :team-ids])))
|
|
(t/is (= [] (get-in props [:cascade :project-ids])))
|
|
(t/is (= [(:id file)] (get-in props [:cascade :file-ids])))
|
|
(t/is (= #{(:id team)} (set (:team-ids props))))))))))
|
|
|
|
(t/deftest delete-object-missing-row-emits-nothing
|
|
;; A cascade over an unknown id marks nothing, so there is nothing to report.
|
|
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
|
|
(with-redefs [cf/flags (conj cf/flags :audit-log)]
|
|
(th/run-task! :delete-object {:object :team
|
|
:id (uuid/next)
|
|
:deleted-at (ct/now)})
|
|
(t/is (empty? (:call-args-list @audit-mock))))))
|
|
|
|
(t/deftest delete-object-snapshot-emits-nothing
|
|
;; Snapshots have no projection table; their cascade stays silent.
|
|
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
|
|
(with-redefs [cf/flags (conj cf/flags :audit-log)]
|
|
(th/run-task! :delete-object {:object :snapshot
|
|
:id (uuid/next)
|
|
:file-id (uuid/next)
|
|
:deleted-at (ct/now)})
|
|
(t/is (empty? (:call-args-list @audit-mock))))))
|