diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 34751179b7..42d1fab650 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -526,7 +526,10 @@ :fn (mg/resource "app/migrations/sql/0155-add-idx-to-file-change.sql")} {:name "0156-reset-pg-storage-defaults" - :fn (mg/resource "app/migrations/sql/0156-reset-pg-storage-defaults.sql")}]) + :fn (mg/resource "app/migrations/sql/0156-reset-pg-storage-defaults.sql")} + + {:name "0157-migrate-legacy-file-data" + :fn (mg/resource "app/migrations/sql/0157-migrate-legacy-file-data.sql")}]) (defn apply-migrations! [pool name migrations] diff --git a/backend/src/app/migrations/sql/0157-migrate-legacy-file-data.sql b/backend/src/app/migrations/sql/0157-migrate-legacy-file-data.sql new file mode 100644 index 0000000000..c7cd6d1fe4 --- /dev/null +++ b/backend/src/app/migrations/sql/0157-migrate-legacy-file-data.sql @@ -0,0 +1,55 @@ +-- Migrate legacy payloads to file_data: labelled file_change.data as +-- snapshots, file.data as mains (backend 'db', deleted_at copied so the +-- GC keeps collecting them), then delete orphan file_change rows with +-- data but no label (no read path reaches them). +-- +-- Idempotent: INSERTs ignore conflicts, UPDATEs only null the legacy +-- column where the file_data row exists. Big instances pre-migrate by +-- hand (operations/2.19.0-migrate-legacy-file-data.md) and fake this +-- step instead: +-- +-- INSERT INTO migrations (module, step) +-- VALUES ('main', '0157-migrate-legacy-file-data'); + +INSERT INTO file_data (file_id, id, created_at, modified_at, deleted_at, + type, backend, metadata, data) +SELECT file_id, id, created_at, updated_at, deleted_at, + 'snapshot', 'db', NULL, data + FROM file_change + WHERE data IS NOT NULL + AND label IS NOT NULL +ON CONFLICT (file_id, id) DO NOTHING; + +UPDATE file_change AS fc + SET data = NULL + WHERE fc.data IS NOT NULL + AND fc.label IS NOT NULL + AND EXISTS (SELECT 1 + FROM file_data AS fd + WHERE fd.file_id = fc.file_id + AND fd.id = fc.id + AND fd.type = 'snapshot'); + +INSERT INTO file_data (file_id, id, created_at, modified_at, deleted_at, + type, backend, metadata, data) +SELECT id, id, created_at, modified_at, deleted_at, + 'main', 'db', NULL, data + FROM file + WHERE data IS NOT NULL +ON CONFLICT (file_id, id) DO NOTHING; + +UPDATE file AS f + SET data = NULL + WHERE f.data IS NOT NULL + AND EXISTS (SELECT 1 + FROM file_data AS fd + WHERE fd.file_id = f.id + AND fd.id = f.id + AND fd.type = 'main'); + +-- Orphan revisions: legacy payloads without label. No file_data row +-- corresponds to them (snapshots are told apart by label), so there is +-- nothing to copy; the row itself is garbage. Delete it. +DELETE FROM file_change + WHERE data IS NOT NULL + AND label IS NULL; diff --git a/backend/test/backend_tests/migrations_legacy_file_data_test.clj b/backend/test/backend_tests/migrations_legacy_file_data_test.clj new file mode 100644 index 0000000000..70c4c72925 --- /dev/null +++ b/backend/test/backend_tests/migrations_legacy_file_data_test.clj @@ -0,0 +1,142 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns backend-tests.migrations-legacy-file-data-test + "Covers the 0157 data migration: legacy file/file_change payloads move + to file_data, keeping timestamps and deleted_at, nulling the legacy + column; unlabelled file_change payloads (orphan revisions) are + deleted." + (:require + [app.common.time :as ct] + [app.common.uuid :as uuid] + [app.db :as db] + [app.util.blob :as blob] + [backend-tests.helpers :as th] + [clojure.java.io :as io] + [clojure.test :as t] + [next.jdbc :as jdbc])) + +(t/use-fixtures :once th/state-init) +(t/use-fixtures :each th/database-reset) + +(defn- run-migration + [] + (let [sql (slurp (io/resource "app/migrations/sql/0157-migrate-legacy-file-data.sql"))] + (jdbc/execute! th/*pool* [sql]))) + +(defn- get-file-data + [file-id id] + (th/db-get :file-data {:file-id file-id :id id} {::db/remove-deleted false})) + +(t/deftest migrates-legacy-file-and-snapshot + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + fid (:id file) + + ;; Simulate an unmigrated main: move the payload back to + ;; file.data and drop the file_data row. + main-data (:data (get-file-data fid fid)) + _ (t/is (some? main-data) "expected file_data main row from create-file*") + file-row (th/db-get :file {:id fid} {::db/remove-deleted false}) + _ (th/db-delete! :file-data {:file-id fid :id fid}) + _ (th/db-update! :file {:data main-data} {:id fid}) + + ;; Simulate an unmigrated snapshot (labelled) with its own + ;; timestamps and a deletion date. + snap-id (uuid/next) + snap-data (blob/encode {:snapshot 1}) + created-at (ct/in-past "2h") + updated-at (ct/in-past "1h") + deleted-at (ct/in-past "30m") + _ (th/db-insert! :file-change + {:id snap-id + :file-id fid + :revn 1 + :data snap-data + :label "v1" + :created-at created-at + :updated-at updated-at + :deleted-at deleted-at}) + + ;; Orphan revision (data, no label): deleted by the migration. + rev-id (uuid/next) + rev-data (blob/encode {:rev 1}) + _ (th/db-insert! :file-change + {:id rev-id + :file-id fid + :revn 2 + :data rev-data}) + + ;; Live revision (changes, no data, no label): untouched. + live-id (uuid/next) + _ (th/db-insert! :file-change + {:id live-id + :file-id fid + :revn 3 + :changes (blob/encode [{:type :dummy}])})] + + (run-migration) + + (t/testing "main payload moves to file_data as type main" + (let [fd (get-file-data fid fid)] + (t/is (some? fd)) + (t/is (= "main" (:type fd))) + (t/is (= "db" (:backend fd))) + (t/is (nil? (:metadata fd))) + (t/is (= (seq ^bytes main-data) (seq ^bytes (:data fd)))) + (t/is (= (:created-at file-row) (:created-at fd))) + (t/is (= (:modified-at file-row) (:modified-at fd))) + (t/is (= (:deleted-at file-row) (:deleted-at fd))))) + + (t/testing "snapshot payload moves to file_data as type snapshot" + (let [fd (get-file-data fid snap-id)] + (t/is (some? fd)) + (t/is (= "snapshot" (:type fd))) + (t/is (= "db" (:backend fd))) + (t/is (nil? (:metadata fd))) + (t/is (= (seq ^bytes snap-data) (seq ^bytes (:data fd)))) + ;; NOTE: timestamptz stores microsecond precision while the + ;; test instants carry nanoseconds, so compare at millis. + (t/is (= (inst-ms created-at) (inst-ms (:created-at fd)))) + (t/is (= (inst-ms updated-at) (inst-ms (:modified-at fd)))) + (t/is (= (inst-ms deleted-at) (inst-ms (:deleted-at fd)))))) + + (t/testing "legacy columns are nulled" + (t/is (nil? (:data (th/db-get :file {:id fid} {::db/remove-deleted false})))) + (t/is (nil? (:data (th/db-get :file-change {:id snap-id} {::db/remove-deleted false}))))) + + (t/testing "unlabelled payload row is deleted, live revision stays" + (t/is (nil? (th/db-get :file-change {:id rev-id} {::db/remove-deleted false}))) + (let [row (th/db-get :file-change {:id live-id} {::db/remove-deleted false})] + (t/is (some? row)) + (t/is (some? (:changes row))))) + + (t/testing "second run is a no-op" + (run-migration) + (t/is (some? (get-file-data fid fid))) + (t/is (some? (get-file-data fid snap-id)))))) + +(t/deftest keeps-preexisting-file-data-row + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + fid (:id file) + + ;; file_data already holds the current payload; the legacy + ;; column keeps a stale copy that must not overwrite it. + current (:data (get-file-data fid fid)) + stale (blob/encode {:stale true}) + _ (t/is (some? current)) + _ (t/is (not= (seq ^bytes current) (seq ^bytes stale))) + _ (th/db-update! :file {:data stale} {:id fid})] + + (run-migration) + + (t/testing "file_data keeps its data and legacy is nulled" + (let [fd (get-file-data fid fid)] + (t/is (= (seq ^bytes current) (seq ^bytes (:data fd))))) + (t/is (nil? (:data (th/db-get :file {:id fid} {::db/remove-deleted false})))))))