From dda6a96407b220ce231417ee213e54f032649d5b Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 21 Sep 2020 15:58:56 +0200 Subject: [PATCH] :recycle: Refactor migrations. --- backend/src/app/migrations.clj | 114 +++++++----------- .../migrations/sql}/0001-add-extensions.sql | 0 .../sql}/0002-add-profile-tables.sql | 0 .../sql}/0003-add-project-tables.sql | 0 .../migrations/sql}/0004-add-tasks-tables.sql | 0 .../sql}/0005-add-libraries-tables.sql | 0 .../sql}/0006-add-presence-tables.sql | 0 ...007-drop-version-field-from-page-table.sql | 0 .../sql}/0008-add-generic-token-table.sql | 0 .../sql}/0009-drop-profile-email-table.sql | 0 .../sql}/0010-add-http-session-table.sql | 0 ...-session-id-field-to-page-change-table.sql | 0 .../0012-make-libraries-linked-to-a-file.sql | 0 .../sql}/0013-mark-files-shareable.sql | 0 .../sql}/0014-refactor-media-storage.sql | 0 .../sql}/0015-improve-tasks-tables.sql | 0 .../0016-truncate-and-alter-tokens-table.sql | 0 .../sql}/0017-link-files-to-libraries.sql | 0 .../sql}/0018-add-file-trimming-triggers.sql | 0 .../0019-add-improved-scheduled-tasks.sql | 0 .../sql}/0020-minor-fixes-to-media-object.sql | 0 .../sql}/0021-http-session-improvements.sql | 0 .../sql}/0022-page-file-refactor.sql | 0 backend/src/app/util/migrations.clj | 2 +- docs/03-Backend-Developer-Guide.md | 29 +++++ 25 files changed, 75 insertions(+), 70 deletions(-) rename backend/{resources/migrations => src/app/migrations/sql}/0001-add-extensions.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0002-add-profile-tables.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0003-add-project-tables.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0004-add-tasks-tables.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0005-add-libraries-tables.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0006-add-presence-tables.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0007-drop-version-field-from-page-table.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0008-add-generic-token-table.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0009-drop-profile-email-table.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0010-add-http-session-table.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0011-add-session-id-field-to-page-change-table.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0012-make-libraries-linked-to-a-file.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0013-mark-files-shareable.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0014-refactor-media-storage.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0015-improve-tasks-tables.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0016-truncate-and-alter-tokens-table.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0017-link-files-to-libraries.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0018-add-file-trimming-triggers.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0019-add-improved-scheduled-tasks.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0020-minor-fixes-to-media-object.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0021-http-session-improvements.sql (100%) rename backend/{resources/migrations => src/app/migrations/sql}/0022-page-file-refactor.sql (100%) diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index 4d9d13edf9..8598d9a020 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -18,97 +18,73 @@ (def +migrations+ {:name "uxbox-main" :steps - [{:desc "Add initial extensions and functions." - :name "0001-add-extensions" - :fn (mg/resource "migrations/0001-add-extensions.sql")} + [{:name "0001-add-extensions" + :fn (mg/resource "app/migrations/sql/0001-add-extensions.sql")} - {:desc "Add profile related tables" - :name "0002-add-profile-tables" - :fn (mg/resource "migrations/0002-add-profile-tables.sql")} + {:name "0002-add-profile-tables" + :fn (mg/resource "app/migrations/sql/0002-add-profile-tables.sql")} - {:desc "Add project related tables" - :name "0003-add-project-tables" - :fn (mg/resource "migrations/0003-add-project-tables.sql")} + {:name "0003-add-project-tables" + :fn (mg/resource "app/migrations/sql/0003-add-project-tables.sql")} - {:desc "Add tasks related tables" - :name "0004-add-tasks-tables" - :fn (mg/resource "migrations/0004-add-tasks-tables.sql")} + {:name "0004-add-tasks-tables" + :fn (mg/resource "app/migrations/sql/0004-add-tasks-tables.sql")} - {:desc "Add libraries related tables" - :name "0005-add-libraries-tables" - :fn (mg/resource "migrations/0005-add-libraries-tables.sql")} + {:name "0005-add-libraries-tables" + :fn (mg/resource "app/migrations/sql/0005-add-libraries-tables.sql")} - {:desc "Add presence related tables" - :name "0006-add-presence-tables" - :fn (mg/resource "migrations/0006-add-presence-tables.sql")} + {:name "0006-add-presence-tables" + :fn (mg/resource "app/migrations/sql/0006-add-presence-tables.sql")} - {:desc "Drop version field from page table." - :name "0007-drop-version-field-from-page-table" - :fn (mg/resource "migrations/0007-drop-version-field-from-page-table.sql")} + {:name "0007-drop-version-field-from-page-table" + :fn (mg/resource "app/migrations/sql/0007-drop-version-field-from-page-table.sql")} - {:desc "Add generic token related tables." - :name "0008-add-generic-token-table" - :fn (mg/resource "migrations/0008-add-generic-token-table.sql")} + {:name "0008-add-generic-token-table" + :fn (mg/resource "app/migrations/sql/0008-add-generic-token-table.sql")} - {:desc "Drop the profile_email table" - :name "0009-drop-profile-email-table" - :fn (mg/resource "migrations/0009-drop-profile-email-table.sql")} + {:name "0009-drop-profile-email-table" + :fn (mg/resource "app/migrations/sql/0009-drop-profile-email-table.sql")} - {:desc "Add new HTTP session table" - :name "0010-add-http-session-table" - :fn (mg/resource "migrations/0010-add-http-session-table.sql")} + {:name "0010-add-http-session-table" + :fn (mg/resource "app/migrations/sql/0010-add-http-session-table.sql")} - {:desc "Add session_id field to page_change table" - :name "0011-add-session-id-field-to-page-change-table" - :fn (mg/resource "migrations/0011-add-session-id-field-to-page-change-table.sql")} + {:name "0011-add-session-id-field-to-page-change-table" + :fn (mg/resource "app/migrations/sql/0011-add-session-id-field-to-page-change-table.sql")} - {:desc "Make libraries linked to a file" - :name "0012-make-libraries-linked-to-a-file" - :fn (mg/resource "migrations/0012-make-libraries-linked-to-a-file.sql")} + {:name "0012-make-libraries-linked-to-a-file" + :fn (mg/resource "app/migrations/sql/0012-make-libraries-linked-to-a-file.sql")} - {:desc "Mark files shareable" - :name "0013-mark-files-shareable" - :fn (mg/resource "migrations/0013-mark-files-shareable.sql")} + {:name "0013-mark-files-shareable" + :fn (mg/resource "app/migrations/sql/0013-mark-files-shareable.sql")} - {:desc "Refactor media storage" - :name "0014-refactor-media-storage.sql" - :fn (mg/resource "migrations/0014-refactor-media-storage.sql")} + {:name "0014-refactor-media-storage.sql" + :fn (mg/resource "app/migrations/sql/0014-refactor-media-storage.sql")} - {:desc "Improve and partition task related tables" - :name "0015-improve-tasks-tables" - :fn (mg/resource "migrations/0015-improve-tasks-tables.sql")} + {:name "0015-improve-tasks-tables" + :fn (mg/resource "app/migrations/sql/0015-improve-tasks-tables.sql")} - {:desc "Truncate & alter tokens tables" - :name "0016-truncate-and-alter-tokens-table" - :fn (mg/resource "migrations/0016-truncate-and-alter-tokens-table.sql")} + {:name "0016-truncate-and-alter-tokens-table" + :fn (mg/resource "app/migrations/sql/0016-truncate-and-alter-tokens-table.sql")} - {:desc "Link files to libraries" - :name "0017-link-files-to-libraries" - :fn (mg/resource "migrations/0017-link-files-to-libraries.sql")} + {:name "0017-link-files-to-libraries" + :fn (mg/resource "app/migrations/sql/0017-link-files-to-libraries.sql")} - {:desc "Add file triming triggers" - :name "0018-add-file-trimming-triggers" - :fn (mg/resource "migrations/0018-add-file-trimming-triggers.sql")} + {:name "0018-add-file-trimming-triggers" + :fn (mg/resource "app/migrations/sql/0018-add-file-trimming-triggers.sql")} - {:desc "Improve scheduled task tables" - :name "0019-add-improved-scheduled-tasks" - :fn (mg/resource "migrations/0019-add-improved-scheduled-tasks.sql")} + {:name "0019-add-improved-scheduled-tasks" + :fn (mg/resource "app/migrations/sql/0019-add-improved-scheduled-tasks.sql")} - {:desc "Minor fixes to media object" - :name "0020-minor-fixes-to-media-object" - :fn (mg/resource "migrations/0020-minor-fixes-to-media-object.sql")} + {:name "0020-minor-fixes-to-media-object" + :fn (mg/resource "app/migrations/sql/0020-minor-fixes-to-media-object.sql")} - {:desc "Improve http session tables" - :name "0021-http-session-improvements" - :fn (mg/resource "migrations/0021-http-session-improvements.sql")} + {:name "0021-http-session-improvements" + :fn (mg/resource "app/migrations/sql/0021-http-session-improvements.sql")} - {:desc "Refactor pages and files" - :name "0022-page-file-refactor" - :fn (mg/resource "migrations/0022-page-file-refactor.sql")} + {:name "0022-page-file-refactor" + :fn (mg/resource "app/migrations/sql/0022-page-file-refactor.sql")} - - {:desc "Adapt old pages and files to new format" - :name "0023-adapt-old-pages-and-files" + {:name "0023-adapt-old-pages-and-files" :fn mg0023/migrate} ]}) diff --git a/backend/resources/migrations/0001-add-extensions.sql b/backend/src/app/migrations/sql/0001-add-extensions.sql similarity index 100% rename from backend/resources/migrations/0001-add-extensions.sql rename to backend/src/app/migrations/sql/0001-add-extensions.sql diff --git a/backend/resources/migrations/0002-add-profile-tables.sql b/backend/src/app/migrations/sql/0002-add-profile-tables.sql similarity index 100% rename from backend/resources/migrations/0002-add-profile-tables.sql rename to backend/src/app/migrations/sql/0002-add-profile-tables.sql diff --git a/backend/resources/migrations/0003-add-project-tables.sql b/backend/src/app/migrations/sql/0003-add-project-tables.sql similarity index 100% rename from backend/resources/migrations/0003-add-project-tables.sql rename to backend/src/app/migrations/sql/0003-add-project-tables.sql diff --git a/backend/resources/migrations/0004-add-tasks-tables.sql b/backend/src/app/migrations/sql/0004-add-tasks-tables.sql similarity index 100% rename from backend/resources/migrations/0004-add-tasks-tables.sql rename to backend/src/app/migrations/sql/0004-add-tasks-tables.sql diff --git a/backend/resources/migrations/0005-add-libraries-tables.sql b/backend/src/app/migrations/sql/0005-add-libraries-tables.sql similarity index 100% rename from backend/resources/migrations/0005-add-libraries-tables.sql rename to backend/src/app/migrations/sql/0005-add-libraries-tables.sql diff --git a/backend/resources/migrations/0006-add-presence-tables.sql b/backend/src/app/migrations/sql/0006-add-presence-tables.sql similarity index 100% rename from backend/resources/migrations/0006-add-presence-tables.sql rename to backend/src/app/migrations/sql/0006-add-presence-tables.sql diff --git a/backend/resources/migrations/0007-drop-version-field-from-page-table.sql b/backend/src/app/migrations/sql/0007-drop-version-field-from-page-table.sql similarity index 100% rename from backend/resources/migrations/0007-drop-version-field-from-page-table.sql rename to backend/src/app/migrations/sql/0007-drop-version-field-from-page-table.sql diff --git a/backend/resources/migrations/0008-add-generic-token-table.sql b/backend/src/app/migrations/sql/0008-add-generic-token-table.sql similarity index 100% rename from backend/resources/migrations/0008-add-generic-token-table.sql rename to backend/src/app/migrations/sql/0008-add-generic-token-table.sql diff --git a/backend/resources/migrations/0009-drop-profile-email-table.sql b/backend/src/app/migrations/sql/0009-drop-profile-email-table.sql similarity index 100% rename from backend/resources/migrations/0009-drop-profile-email-table.sql rename to backend/src/app/migrations/sql/0009-drop-profile-email-table.sql diff --git a/backend/resources/migrations/0010-add-http-session-table.sql b/backend/src/app/migrations/sql/0010-add-http-session-table.sql similarity index 100% rename from backend/resources/migrations/0010-add-http-session-table.sql rename to backend/src/app/migrations/sql/0010-add-http-session-table.sql diff --git a/backend/resources/migrations/0011-add-session-id-field-to-page-change-table.sql b/backend/src/app/migrations/sql/0011-add-session-id-field-to-page-change-table.sql similarity index 100% rename from backend/resources/migrations/0011-add-session-id-field-to-page-change-table.sql rename to backend/src/app/migrations/sql/0011-add-session-id-field-to-page-change-table.sql diff --git a/backend/resources/migrations/0012-make-libraries-linked-to-a-file.sql b/backend/src/app/migrations/sql/0012-make-libraries-linked-to-a-file.sql similarity index 100% rename from backend/resources/migrations/0012-make-libraries-linked-to-a-file.sql rename to backend/src/app/migrations/sql/0012-make-libraries-linked-to-a-file.sql diff --git a/backend/resources/migrations/0013-mark-files-shareable.sql b/backend/src/app/migrations/sql/0013-mark-files-shareable.sql similarity index 100% rename from backend/resources/migrations/0013-mark-files-shareable.sql rename to backend/src/app/migrations/sql/0013-mark-files-shareable.sql diff --git a/backend/resources/migrations/0014-refactor-media-storage.sql b/backend/src/app/migrations/sql/0014-refactor-media-storage.sql similarity index 100% rename from backend/resources/migrations/0014-refactor-media-storage.sql rename to backend/src/app/migrations/sql/0014-refactor-media-storage.sql diff --git a/backend/resources/migrations/0015-improve-tasks-tables.sql b/backend/src/app/migrations/sql/0015-improve-tasks-tables.sql similarity index 100% rename from backend/resources/migrations/0015-improve-tasks-tables.sql rename to backend/src/app/migrations/sql/0015-improve-tasks-tables.sql diff --git a/backend/resources/migrations/0016-truncate-and-alter-tokens-table.sql b/backend/src/app/migrations/sql/0016-truncate-and-alter-tokens-table.sql similarity index 100% rename from backend/resources/migrations/0016-truncate-and-alter-tokens-table.sql rename to backend/src/app/migrations/sql/0016-truncate-and-alter-tokens-table.sql diff --git a/backend/resources/migrations/0017-link-files-to-libraries.sql b/backend/src/app/migrations/sql/0017-link-files-to-libraries.sql similarity index 100% rename from backend/resources/migrations/0017-link-files-to-libraries.sql rename to backend/src/app/migrations/sql/0017-link-files-to-libraries.sql diff --git a/backend/resources/migrations/0018-add-file-trimming-triggers.sql b/backend/src/app/migrations/sql/0018-add-file-trimming-triggers.sql similarity index 100% rename from backend/resources/migrations/0018-add-file-trimming-triggers.sql rename to backend/src/app/migrations/sql/0018-add-file-trimming-triggers.sql diff --git a/backend/resources/migrations/0019-add-improved-scheduled-tasks.sql b/backend/src/app/migrations/sql/0019-add-improved-scheduled-tasks.sql similarity index 100% rename from backend/resources/migrations/0019-add-improved-scheduled-tasks.sql rename to backend/src/app/migrations/sql/0019-add-improved-scheduled-tasks.sql diff --git a/backend/resources/migrations/0020-minor-fixes-to-media-object.sql b/backend/src/app/migrations/sql/0020-minor-fixes-to-media-object.sql similarity index 100% rename from backend/resources/migrations/0020-minor-fixes-to-media-object.sql rename to backend/src/app/migrations/sql/0020-minor-fixes-to-media-object.sql diff --git a/backend/resources/migrations/0021-http-session-improvements.sql b/backend/src/app/migrations/sql/0021-http-session-improvements.sql similarity index 100% rename from backend/resources/migrations/0021-http-session-improvements.sql rename to backend/src/app/migrations/sql/0021-http-session-improvements.sql diff --git a/backend/resources/migrations/0022-page-file-refactor.sql b/backend/src/app/migrations/sql/0022-page-file-refactor.sql similarity index 100% rename from backend/resources/migrations/0022-page-file-refactor.sql rename to backend/src/app/migrations/sql/0022-page-file-refactor.sql diff --git a/backend/src/app/util/migrations.clj b/backend/src/app/util/migrations.clj index ec7f108383..479ffb7a2d 100644 --- a/backend/src/app/util/migrations.clj +++ b/backend/src/app/util/migrations.clj @@ -13,7 +13,7 @@ [next.jdbc :as jdbc])) (s/def ::name string?) -(s/def ::step (s/keys :req-un [::name ::desc ::fn])) +(s/def ::step (s/keys :req-un [::name ::fn])) (s/def ::steps (s/every ::step :kind vector?)) (s/def ::migrations (s/keys :req-un [::name ::steps])) diff --git a/docs/03-Backend-Developer-Guide.md b/docs/03-Backend-Developer-Guide.md index 2f45110e6d..9900f13c50 100644 --- a/docs/03-Backend-Developer-Guide.md +++ b/docs/03-Backend-Developer-Guide.md @@ -45,3 +45,32 @@ command: rlwrap netcat localhost 5555 ``` +## Migrations + +The database migrations are located in two directories: + +- `src/app/migrations` (contains migration scripts in clojure) +- `src/app/migrations/sql` (contains the pure SQL migrations) + +The SQL migration naming consists in the following: + +``` +XXXX--- +``` + +Examples: + +``` +0025-del-generic-tokens-table +0026-mod-profile-table-add-is-active-field +``` + +**NOTE**: if table name has more than one words, we still use `-` as a separator. + +If you need to have a global overview of the all schema of the database you can extract it +using postgresql: + +```bash +# (in the devenv environment) +pg_dump -h postgres -s > schema.sql +```