From a882d0bf6de9e5510c82f10d4863113865a4af10 Mon Sep 17 00:00:00 2001 From: Eva Date: Thu, 25 Jan 2024 10:27:25 +0100 Subject: [PATCH 1/7] :recycle: Update basic color palette --- .../styles/common/dependencies/colors.scss | 2 +- .../styles/common/refactor/color-defs.scss | 105 ++++++++++-------- .../common/refactor/common-refactor.scss | 4 +- .../styles/common/refactor/design-tokens.scss | 75 +++++++------ .../common/refactor/themes/default-theme.scss | 41 +++---- .../common/refactor/themes/light-theme.scss | 41 +++---- frontend/src/app/main/constants.cljs | 4 +- .../main/data/workspace/notifications.cljs | 14 +-- frontend/src/app/main/ui/auth.scss | 2 +- frontend/src/app/main/ui/auth/common.scss | 6 +- frontend/src/app/main/ui/auth/register.scss | 4 +- frontend/src/app/main/ui/comments.scss | 2 +- .../src/app/main/ui/components/forms.scss | 2 +- .../src/app/main/ui/components/select.scss | 2 +- frontend/src/app/main/ui/dashboard.scss | 2 +- .../src/app/main/ui/dashboard/comments.scss | 2 +- frontend/src/app/main/ui/dashboard/files.scss | 2 +- frontend/src/app/main/ui/dashboard/fonts.scss | 14 +-- frontend/src/app/main/ui/dashboard/grid.scss | 4 +- .../src/app/main/ui/dashboard/import.scss | 34 +++--- .../app/main/ui/dashboard/inline_edition.scss | 4 +- .../src/app/main/ui/dashboard/libraries.scss | 2 +- .../app/main/ui/dashboard/placeholder.scss | 4 +- .../src/app/main/ui/dashboard/projects.scss | 2 +- .../src/app/main/ui/dashboard/search.scss | 4 +- .../src/app/main/ui/dashboard/sidebar.scss | 25 ++--- frontend/src/app/main/ui/dashboard/team.scss | 18 +-- .../src/app/main/ui/dashboard/templates.scss | 4 +- .../app/main/ui/debug/components_preview.cljs | 2 +- frontend/src/app/main/ui/export.scss | 19 ++-- .../src/app/main/ui/flex_controls/common.cljs | 6 +- frontend/src/app/main/ui/measurements.cljs | 10 +- frontend/src/app/main/ui/messages.scss | 4 +- frontend/src/app/main/ui/settings.scss | 10 +- .../app/main/ui/settings/access_tokens.scss | 4 +- .../src/app/main/ui/settings/profile.scss | 6 +- .../src/app/main/ui/settings/sidebar.scss | 8 +- .../main/ui/shapes/grid_layout_viewer.cljs | 6 +- .../main/ui/viewer/inspect/left_sidebar.scss | 2 +- .../src/app/main/ui/workspace/libraries.scss | 2 +- .../app/main/ui/workspace/right_header.scss | 10 +- .../main/ui/workspace/shapes/path/common.cljs | 8 +- .../workspace/sidebar/debug_shape_info.scss | 2 +- .../options/menus/layout_container.scss | 2 +- .../main/ui/workspace/viewport/gradients.cljs | 8 +- .../viewport/grid_layout_editor.scss | 2 +- .../ui/workspace/viewport/interactions.cljs | 8 +- .../main/ui/workspace/viewport/presence.cljs | 4 +- .../main/ui/workspace/viewport/selection.cljs | 4 +- .../ui/workspace/viewport/snap_distances.cljs | 4 +- .../ui/workspace/viewport/snap_points.cljs | 2 +- 51 files changed, 287 insertions(+), 271 deletions(-) diff --git a/frontend/resources/styles/common/dependencies/colors.scss b/frontend/resources/styles/common/dependencies/colors.scss index 859c6af4a9..34bd2dccbb 100644 --- a/frontend/resources/styles/common/dependencies/colors.scss +++ b/frontend/resources/styles/common/dependencies/colors.scss @@ -8,7 +8,7 @@ $db-primary: #18181a; $db-secondary: #000000; $db-tertiary: #212426; -$db-cuaternary: #2e3434; +$db-quaternary: #2e3434; $df-primary: #ffffff; $df-secondary: #8f9da3; diff --git a/frontend/resources/styles/common/refactor/color-defs.scss b/frontend/resources/styles/common/refactor/color-defs.scss index 7f7538b91a..6c47a842f7 100644 --- a/frontend/resources/styles/common/refactor/color-defs.scss +++ b/frontend/resources/styles/common/refactor/color-defs.scss @@ -8,68 +8,85 @@ :root { // DARK - --dark-gray-1: #1d1f20; - --dark-gray-2: #000000; - --dark-gray-2-30: #{color.change(#000000, $alpha: 0.3)}; - --dark-gray-2-80: #{color.change(#000000, $alpha: 0.8)}; - --dark-gray-3: #292c2d; - --dark-gray-4: #34393b; - --white: #fff; - --off-white: #aab5ba; - --off-white-40: #{color.change(#aab5ba, $alpha: 0.4)}; - --green: #91fadb; - --green-30: rgba(145, 250, 219, 0.3); - --lilac: #bb97d8; - --pink: #ff6fe0; - --strong-green: #00d1b8; - --strong-green-10: #{color.change(#00d1b8, $alpha: 0.1)}; + // Dark background + --db-primary: #18181a; + --db-secondary: #000000; + --db-secondary-30: #{color.change(#000000, $alpha: 0.3)}; + --db-secondary-80: #{color.change(#000000, $alpha: 0.8)}; + --db-tertiary: #212426; + --db-quaternary: #2e3434; - // NOTIFICATION - --dark-ok-color: var(--strong-green); - --dark-warning-color: #ff6432; - --dark-pending-color: var(--lilac); - --dark-error-color: #ff3277; - --default-presence-color: #dee563; + //Dark foreground + --df-primary: #ffffff; + --df-secondary: #8f9da3; + --df-secondary-40: #{color.change(#8f9da3, $alpha: 0.4)}; // TODO: Check if needed + + //Dark accent + --da-primary: #7efff5; + --da-primary-muted: #426158; + --da-secondary: #bb97d8; + --da-tertiary: #00d1b8; + --da-tertiary-10: #{color.change(#00d1b8, $alpha: 0.1)}; + --da-quaternary: #ff6fe0; // LIGHT - --light-gray-1: #fff; - --light-gray-2: #e8eaee; - --light-gray-2-30: #{color.change(#e8eaee, $alpha: 0.3)}; - --light-gray-2-80: #{color.change(#e8eaee, $alpha: 0.8)}; - --light-gray-3: #f3f4f6; - --light-gray-4: #eef0f2; - --black: #000; - --off-black: #495e74; - --off-black-40: #{color.change(#495e74, $alpha: 0.4)}; - --purple: #6911d4; - --purple-30: rgba(105, 17, 212, 0.2); - --blue: #1345aa; - --strong-purple: #8c33eb; - --strong-purple-10: #{color.change(#8c33eb, $alpha: 0.1)}; + // Light background + --lb-primary: #ffffff; + --lb-secondary: #e8eaee; + --lb-secondary-30: #{color.change(#e8eaee, $alpha: 0.3)}; + --lb-secondary-80: #{color.change(#e8eaee, $alpha: 0.8)}; + --lb-tertiary: #f3f4f6; + --lb-quaternary: #eef0f2; - // NOTIFICATION WILL CHANGE - --light-ok-color: var(--strong-green); - --light-warning-color: #ff9b49; - --light-pending-color: var(--lilac); - --light-error-color: #ff4986; + //Light foreground + --lf-primary: #000; + --lf-secondary: #495e74; + --lf-secondary-40: #{color.change(#495e74, $alpha: 0.4)}; + + //Light accent + --la-primary: #6911d4; + --la-primary-muted: #e1d2f5; + --la-secondary: #1345aa; + --la-tertiary: #8c33eb; + --la-tertiary-10: #{color.change(#8c33eb, $alpha: 0.1)}; + --la-quaternary: #ff6fe0; + + // STATUS COLOR + --status-color-success-50: #f0f8ff; + --status-color-success-500: #2d9f8f; + --status-color-success-950: #0a2927; + --status-color-warning-50: #fff4ed; + --status-color-warning-500: #fe4811; + --status-color-warning-950: #440806; + --status-color-error-50: #fff0f3; + --status-color-error-500: #ff3277; + --status-color-error-950: #500124; + --status-color-info-50: #f0f8ff; + --status-color-info-500: #0e9be9; + --status-color-info-950: #082c49; + // Status color default will change with theme and will be defined on theme files //GENERIC --color-canvas: #e8e9ea; + // APP COLORS + --app-white: #ffffff; + --app-black: #000; + // SOCIAL LOGIN BUTTONS --google-login-background: #4285f4; --google-login-background-hover: #{color.adjust(#4285f4, $lightness: -15%)}; - --google-login-foreground: var(--white); + --google-login-foreground: var(--df-primary); --github-login-background: #4c4c4c; --github-login-background-hover: #{color.adjust(#4c4c4c, $lightness: -15%)}; - --github-login-foreground: var(--white); + --github-login-foreground: var(--app-white); --oidc-login-background: #b3b3b3; --oidc-login-background-hover: #{color.adjust(#b3b3b3, $lightness: -15%)}; - --oidc-login-foreground: var(--white); + --oidc-login-foreground: var(--app-white); --gitlab-login-background: #fc6d26; --gitlab-login-background-hover: #{color.adjust(#fc6d26, $lightness: -15%)}; - --gitlab-login-foreground: var(--white); + --gitlab-login-foreground: var(--app-white); } diff --git a/frontend/resources/styles/common/refactor/common-refactor.scss b/frontend/resources/styles/common/refactor/common-refactor.scss index c1033735c8..b06f8723e8 100644 --- a/frontend/resources/styles/common/refactor/common-refactor.scss +++ b/frontend/resources/styles/common/refactor/common-refactor.scss @@ -23,13 +23,13 @@ $db-primary: var(--color-background-primary); $db-secondary: var(--color-background-secondary); $db-tertiary: var(--color-background-tertiary); -$db-cuaternary: var(--color-background-quaternary); +$db-quaternary: var(--color-background-quaternary); $db-subtle: var(--color-background-subtle); $db-disabled: var(--color-background-disabled); $df-primary: var(--color-foreground-primary); $df-secondary: var(--color-foreground-secondary); -$df-tertiary: var(--color-foreground-tertiary); +$df-tertiary: var(--color-accent-quaternary); $da-primary: var(--color-accent-primary); $da-primary-muted: var(--color-accent-primary-muted); $da-secondary: var(--color-accent-secondary); diff --git a/frontend/resources/styles/common/refactor/design-tokens.scss b/frontend/resources/styles/common/refactor/design-tokens.scss index 184d86ba7a..fe73cd6aff 100644 --- a/frontend/resources/styles/common/refactor/design-tokens.scss +++ b/frontend/resources/styles/common/refactor/design-tokens.scss @@ -9,14 +9,13 @@ .default { // BASE COLORS --canvas-background-color: var(--color-background-primary); - --canvas-fill-color: var(--canvas-color); + --canvas-fill-color: var(--color-canvas); --scrollbar-background-color: var(--color-foreground-secondary); --panel-background-color: var(--color-background-primary); --app-background: var(--color-background-primary); --loader-background: var(--color-background-primary); --panel-title-background-color: var(--color-background-secondary); - --presence-color: var(--default-presence-color); // BUTTONS --button-foreground-hover: var(--color-accent-primary); @@ -76,8 +75,8 @@ --button-radio-border-color-focus: var(--color-accent-primary); --button-radio-foreground-color-focus: var(--color-foreground-secondary); - --button-warning-background-color-rest: var(--warning-color); - --button-warning-border-color-rest: var(--warning-color); + --button-warning-background-color-rest: var(--status-color-warning-500); + --button-warning-border-color-rest: var(--status-color-warning-500); --button-warning-foreground-color-rest: var(--color-background-secondary); --button-disabled-background-color-rest: var(--color-background-disabled); @@ -132,15 +131,16 @@ --palette-button-shadow-final: transparent; --palette-handler-background-color: var(--color-background-quaternary); - --color-bullet-background-color: var(--white); // We don't want this color to change with palette + --color-bullet-background-color: var(--app-white); // We don't want this color to change with palette --color-bullet-border-color: var(--color-background-quaternary); --color-bullet-border-color-selected: var(--color-accent-primary); // ICONS --icon-foreground: var(--color-foreground-secondary); + --main-icon-foreground: var(--color-foreground-primary); --icon-foreground-hover: var(--color-foreground-primary); - --icon-foreground-accept: var(--ok-color); - --icon-foreground-discard: var(--error-color); + --icon-foreground-accept: var(--status-color-success-500); + --icon-foreground-discard: var(--status-color-error-500); // INPUTS, SELECTS, DROPDOWNS @@ -164,7 +164,7 @@ --input-background-color-disabled: var(--color-background-primary); --input-foreground-color-disabled: var(--color-foreground-secondary); --input-border-color-disabled: var(--color-background-quaternary); - --input-border-color-error: var(--error-color); + --input-border-color-error: var(--status-color-error-500); --input-border-color-success: var(--color-accent-primary); --input-details-color: var(--color-background-primary); @@ -212,16 +212,16 @@ --assets-title-background-color: var(--color-background-primary); --assets-item-background-color: var(--color-background-tertiary); --assets-item-background-color-hover: var(--color-background-quaternary); - --assets-item-name-background-color: var(--dark-gray-2-80); // TODO: penpot file has a non-existing token + --assets-item-name-background-color: var(--db-secondary-80); // TODO: penpot file has a non-existing token --assets-item-name-foreground-color: var(--color-foreground-secondary); --assets-item-name-foreground-color-hover: var(--color-foreground-primary); --assets-item-name-foreground-color-disabled: var(--color-foreground-disabled); --assets-item-border-color: var(--color-accent-primary); --assets-item-background-color-drag: var(--color-accent-primary-muted); --assets-item-border-color-drag: var(--color-accent-tertiary); - --assets-component-background-color: var(--white); // We don't want this color to change with palette + --assets-component-background-color: var(--app-white); // We don't want this color to change with palette --assets-component-background-color-disabled: var( - --off-white + --df-secondary; ); // We don't want this color to change with palette --assets-component-border-color: var(--color-background-tertiary); --assets-component-border-selected: var(--color-accent-tertiary); @@ -236,6 +236,8 @@ --library-content-foreground-color: var(--color-foreground-secondary); --dropdown-background-color: var(--color-background-tertiary); + --dropdown-separator-color: var(--color-background-primary); + --profile-drowpdown-background-color: var(--color-background-primary); --not-found-background-color: var(--color-background-tertiary); --not-found-foreground-color: var(--color-foreground-secondary); @@ -271,42 +273,53 @@ --comment-bullet-foreground-color-resolved: var(--color-foreground-secondary); --comment-bullet-border-color-resolved: var(--color-background-quaternary); --comment-modal-background-color: var(--color-background-primary); + --comment-thread-background-color-hover: var(--color-background-primary); // GRID LAYOUT - --grid-editor-marker-color: var(--color-foreground-tertiary); - --grid-editor-marker-text: var(--color-foreground-tertiary); - --grid-editor-area-background: var(--color-foreground-tertiary); - --grid-editor-area-text: var(--color-foreground-tertiary); - --grid-editor-line-color: var(--color-foreground-tertiary); - --grid-editor-plus-btn-foreground: var(--white); - --grid-editor-plus-btn-background: var(--color-foreground-tertiary); + --grid-editor-marker-color: var(--color-accent-quaternary); + --grid-editor-marker-text: var(--color-accent-quaternary); + --grid-editor-area-background: var(--color-accent-quaternary); + --grid-editor-area-text: var(--color-accent-quaternary); + --grid-editor-line-color: var(--color-accent-quaternary); + --grid-editor-plus-btn-foreground: var(--app-white); + --grid-editor-plus-btn-background: var(--color-accent-quaternary); // MODALS --modal-background-color: var(--color-background-primary); --modal-title-foreground-color: var(--color-foreground-primary); --modal-text-foreground-color: var(--color-foreground-secondary); --modal-hint-border-color: var(--color-background-quaternary); - --modal-button-background-color-error: var(--error-color); + --modal-button-background-color-error: var(--status-color-error-500); --modal-button-foreground-color-error: var(--color-foreground-primary); --modal-link-foreground-color: var(--color-accent-primary); --modal-border-color: var(--color-background-quaternary); - // ALERTS & STATUS - --alert-background-color-ok: var(--ok-color); - --alert-foreground-color-ok: var(--dark-gray-2); // We don't want this color to change with theme - --alert-background-color-warning: var(--warning-color); - --alert-foreground-color-warning: var(--white); // We don't want this color to change with theme - --alert-background-color-error: var(--error-color); - --alert-foreground-color-error: var(--white); // We don't want this color to change with theme + // ALERTS NOTIFICATION TOAST & STATUS WIDGET + --alert-background-color-success: var(--status-color-success-500); + --alert-foreground-color-success: var(--db-secondary); // We don't want this color to change with theme + --alert-background-color-warning: var(--status-color-warning-500); + --alert-foreground-color-warning: var(--app-white); // We don't want this color to change with theme + --alert-background-color-error: var(--status-color-error-500); + --alert-foreground-color-error: var(--app-white); // We don't want this color to change with theme --alert-background-color-neutral: var(--color-background-quaternary); --alert-foreground-color-neutral: var(--color-foreground-secondary); --alert-foreground-color-neutral-active: var(--color-foreground-primary); - --status-ok-background-color: var(--ok-color); - --status-warning-background-color: var(--warning-color); - --status-pending-background-color: var(--pending-color); - --status-error-background-color: var(--error-color); - --status-icon-foreground-color: var(--color-background-primary); + --notification-background-color-success: var(); + --notification-foreground-color-success: var(); + --notification-border-color-success: var(); + --notification-foreground-color-default: var(--color-foreground-secondary); + + --status-widget-background-color-success: var(--status-color-success-500); + --status-widget-background-color-warning: var(--status-color-warning-500); + --status-widget-background-color-pending: var(--status-color-pending-500); + --status-widget-background-color-error: var(--status-color-error-500); + --status-widget-icon-foreground-color: var(--color-background-primary); // TODO review + + --element-foreground-success: var(--status-color-success-500); + --element-foreground-warning: var(--status-color-warning-500); + --element-foreground-pending: var(--status-color-info-500); + --element-foreground-error: var(--status-color-error-500); // INTERFACE ELEMENTS --search-bar-background-color: var(--color-background-primary); diff --git a/frontend/resources/styles/common/refactor/themes/default-theme.scss b/frontend/resources/styles/common/refactor/themes/default-theme.scss index fd5ebd4f3a..64dafa89b9 100644 --- a/frontend/resources/styles/common/refactor/themes/default-theme.scss +++ b/frontend/resources/styles/common/refactor/themes/default-theme.scss @@ -7,32 +7,27 @@ @use "sass:meta"; :root { - --color-background-primary: var(--dark-gray-1); - --color-background-secondary: var(--dark-gray-2); - --color-background-tertiary: var(--dark-gray-3); - --color-background-quaternary: var(--dark-gray-4); - --color-background-subtle: var(--dark-gray-2-30); - --color-background-disabled: var(--off-white); - --color-foreground-primary: var(--white); - --color-foreground-secondary: var(--off-white); - --color-foreground-tertiary: var(--pink); - --color-foreground-disabled: var(--off-white-40); - --color-accent-primary: var(--green); - --color-accent-primary-muted: var(--green-30); - --color-accent-secondary: var(--lilac); - --color-accent-tertiary: var(--strong-green); - --color-accent-tertiary-muted: var(--strong-green-10); - --color-component-highlight: var(--lilac); + --color-background-primary: var(--db-primary); + --color-background-secondary: var(--db-secondary); + --color-background-tertiary: var(--db-tertiary); + --color-background-quaternary: var(--db-quaternary); + --color-background-subtle: var(--db-secondary-30); + --color-background-disabled: var(--df-secondary); + --color-foreground-primary: var(--df-primary); + --color-foreground-secondary: var(--df-secondary); + --color-foreground-disabled: var(--df-secondary-40); + --color-accent-primary: var(--da-primary); + --color-accent-primary-muted: var(--da-primary-muted); + --color-accent-secondary: var(--da-secondary); + --color-accent-tertiary: var(--da-tertiary); + --color-accent-tertiary-muted: var(--da-tertiary-10); + --color-accent-quaternary: var(--da-quaternary); + --color-component-highlight: var(--da-secondary); --overlay-color: rgba(0, 0, 0, 0.4); - --ok-color: var(--dark-ok-color); - --warning-color: var(--dark-warning-color); - --pending-color: var(--dark-pending-color); - --error-color: var(--dark-error-color); - --canvas-color: var(--color-canvas); - --shadow-color: var(--dark-gray-2-30); - --radio-button-box-shadow: 0 0 0 1px var(--dark-gray-2-30) inset; + --shadow-color: var(--db-secondary-30); + --radio-button-box-shadow: 0 0 0 1px var(--db-secondary-30) inset; @include meta.load-css("hljs-dark-theme"); } diff --git a/frontend/resources/styles/common/refactor/themes/light-theme.scss b/frontend/resources/styles/common/refactor/themes/light-theme.scss index 9cee8c4270..ba5fcb7a7c 100644 --- a/frontend/resources/styles/common/refactor/themes/light-theme.scss +++ b/frontend/resources/styles/common/refactor/themes/light-theme.scss @@ -7,32 +7,27 @@ @use "sass:meta"; .light { - --color-background-primary: var(--light-gray-1); - --color-background-secondary: var(--light-gray-2); - --color-background-tertiary: var(--light-gray-3); - --color-background-quaternary: var(--light-gray-4); - --color-background-subtle: var(--light-gray-2-30); //Whatch this¡¡ - --color-background-disabled: var(--light-gray-4); - --color-foreground-primary: var(--black); - --color-foreground-secondary: var(--off-black); - --color-foreground-tertiary: var(--pink); - --color-foreground-disabled: var(--off-black-40); - --color-accent-primary: var(--purple); - --color-accent-primary-muted: var(--purple-30); - --color-accent-secondary: var(--blue); - --color-accent-tertiary: var(--strong-purple); - --color-accent-tertiary-muted: var(--strong-purple-10); - --color-component-highlight: var(--blue); + --color-background-primary: var(--lb-primary); + --color-background-secondary: var(--lb-secondary); + --color-background-tertiary: var(--lb-tertiary); + --color-background-quaternary: var(--lb-quaternary); + --color-background-subtle: var(--lb-secondary-30); //Whatch this¡¡ + --color-background-disabled: var(--lb-quaternary); + --color-foreground-primary: var(--lf-primary); + --color-foreground-secondary: var(--lf-secondary); + --color-foreground-disabled: var(--lf-secondary-40); + --color-accent-primary: var(--la-primary); + --color-accent-primary-muted: var(--la-primary-muted); + --color-accent-secondary: var(--la-secondary); + --color-accent-tertiary: var(--la-tertiary); + --color-accent-tertiary-muted: var(--la-tertiary-10); + --color-accent-quaternary: var(--la-quaternary); + --color-component-highlight: var(--la-secondary); --overlay-color: rgba(255, 255, 255, 0.4); - --ok-color: var(--light-ok-color); - --warning-color: var(--light-warning-color); - --pending-color: var(--light-pending-color); - --error-color: var(--light-error-color); - --canvas-color: var(--color-canvas); - --shadow-color: var(--off-black-40); - --radio-button-box-shadow: 0 0 0 1px var(--light-gray-2) inset; + --shadow-color: var(--lf-secondary-40); + --radio-button-box-shadow: 0 0 0 1px var(--lb-secondary) inset; @include meta.load-css("hljs-light-theme"); } diff --git a/frontend/src/app/main/constants.cljs b/frontend/src/app/main/constants.cljs index 75fb6b8c81..31bd4dfd63 100644 --- a/frontend/src/app/main/constants.cljs +++ b/frontend/src/app/main/constants.cljs @@ -19,9 +19,9 @@ "Default data for page metadata." {:grid-x-axis grid-x-axis :grid-y-axis grid-y-axis - :grid-color "var(--off-white)" + :grid-color "var(--df-secondary)" :grid-alignment true - :background "var(--white)"}) + :background "var(--app-white)"}) (def size-presets [{:name "APPLE"} diff --git a/frontend/src/app/main/data/workspace/notifications.cljs b/frontend/src/app/main/data/workspace/notifications.cljs index 0f5616fd2e..6efa574390 100644 --- a/frontend/src/app/main/data/workspace/notifications.cljs +++ b/frontend/src/app/main/data/workspace/notifications.cljs @@ -124,17 +124,15 @@ ;; --- Handle: Presence (def ^:private presence-palette - #{"#82e590" ; green - "#7ad7c5" ; blue-green + #{"#f49ef7" ; pink "#75cafc" ; blue - "#a9bdfa" ; blue-purple + "#fdcf79" ; gold + "#a9bdfa" ; indigo + "#faa6b7" ; red "#cbaaff" ; purple - "#f49ef7" ; pink - "#faa6b7" ; salmon "#f9b489" ; orange - "#fdcd79" ; soft-orange "#dee563" ; yellow -> default presence color - "#b1e96f" ; yellow-green + "#b1e96f" ; lemon }) (defn handle-presence @@ -145,7 +143,7 @@ (remove nil?)) used (into #{} xfm presence) avail (set/difference presence-palette used)] - (or (first avail) "var(--black)"))) + (or (first avail) "var(--app-black)"))) (update-color [color presence] (if (some? color) diff --git a/frontend/src/app/main/ui/auth.scss b/frontend/src/app/main/ui/auth.scss index ae955b0625..2e5d45fc70 100644 --- a/frontend/src/app/main/ui/auth.scss +++ b/frontend/src/app/main/ui/auth.scss @@ -8,7 +8,7 @@ .auth-section { align-items: center; - background: $db-primary; + background: var(--panel-background-color); display: grid; gap: $s-32; grid-template-columns: repeat(5, 1fr); diff --git a/frontend/src/app/main/ui/auth/common.scss b/frontend/src/app/main/ui/auth/common.scss index 9483f9ba5d..db99ba8fa6 100644 --- a/frontend/src/app/main/ui/auth/common.scss +++ b/frontend/src/app/main/ui/auth/common.scss @@ -19,7 +19,7 @@ } .separator { - border-color: $db-cuaternary; + border-color: $db-quaternary; margin: $s-24 0; } @@ -60,7 +60,7 @@ flex-direction: column; gap: $s-12; padding: $s-24 0; - border-top: $s-1 solid $db-cuaternary; + border-top: $s-1 solid $db-quaternary; span { text-align: center; @@ -111,7 +111,7 @@ } &:hover { - color: var(--white); + color: var(--app-white); } } diff --git a/frontend/src/app/main/ui/auth/register.scss b/frontend/src/app/main/ui/auth/register.scss index 7fab883e15..9cbc004574 100644 --- a/frontend/src/app/main/ui/auth/register.scss +++ b/frontend/src/app/main/ui/auth/register.scss @@ -20,7 +20,7 @@ } .notification-icon { - fill: $df-primary; + fill: var(--main-icon-foreground); display: flex; justify-content: center; margin-bottom: $s-32; @@ -33,6 +33,6 @@ .notification-text-email, .notification-text { font-size: $fs-16; - color: $df-secondary; + color: var(--notification-foreground-color-default); margin-bottom: $s-16; } diff --git a/frontend/src/app/main/ui/comments.scss b/frontend/src/app/main/ui/comments.scss index 874a229ad3..2c5e2275f5 100644 --- a/frontend/src/app/main/ui/comments.scss +++ b/frontend/src/app/main/ui/comments.scss @@ -14,7 +14,7 @@ padding: $s-8 $s-16; &:hover { - background: $db-primary; + background: var(--comment-thread-background-color-hover); } } diff --git a/frontend/src/app/main/ui/components/forms.scss b/frontend/src/app/main/ui/components/forms.scss index 512ba312ee..ccfc5cf0bc 100644 --- a/frontend/src/app/main/ui/components/forms.scss +++ b/frontend/src/app/main/ui/components/forms.scss @@ -325,7 +325,7 @@ } } &.invalid { - background-color: var(--status-error-background-color); + background-color: var(--status-widget-background-color-error); .text { color: var(--alert-foreground-color-error); } diff --git a/frontend/src/app/main/ui/components/select.scss b/frontend/src/app/main/ui/components/select.scss index 1f6621c914..0817253a6f 100644 --- a/frontend/src/app/main/ui/components/select.scss +++ b/frontend/src/app/main/ui/components/select.scss @@ -48,7 +48,7 @@ .separator { margin: 0; height: $s-12; - border-top: 1px solid $db-primary; + border-top: $s-1 solid var(--dropdown-separator-color); } } .checked-element { diff --git a/frontend/src/app/main/ui/dashboard.scss b/frontend/src/app/main/ui/dashboard.scss index fa287e7fad..fdf2d0d695 100644 --- a/frontend/src/app/main/ui/dashboard.scss +++ b/frontend/src/app/main/ui/dashboard.scss @@ -7,7 +7,7 @@ @use "refactor/common-refactor.scss" as *; .dashboard { - background-color: $db-primary; + background-color: var(--app-background); display: grid; grid-template-columns: $s-40 $s-256 1fr; grid-template-rows: $s-52 1fr; diff --git a/frontend/src/app/main/ui/dashboard/comments.scss b/frontend/src/app/main/ui/dashboard/comments.scss index 395d08005d..bdfcca1412 100644 --- a/frontend/src/app/main/ui/dashboard/comments.scss +++ b/frontend/src/app/main/ui/dashboard/comments.scss @@ -79,7 +79,7 @@ } &:hover { - background-color: $db-cuaternary; + background-color: $db-quaternary; svg { stroke: $da-primary; diff --git a/frontend/src/app/main/ui/dashboard/files.scss b/frontend/src/app/main/ui/dashboard/files.scss index 9173bac111..4473dab6a8 100644 --- a/frontend/src/app/main/ui/dashboard/files.scss +++ b/frontend/src/app/main/ui/dashboard/files.scss @@ -12,7 +12,7 @@ margin-right: $s-16; overflow-y: auto; width: 100%; - border-top: $s-1 solid $db-cuaternary; + border-top: $s-1 solid $db-quaternary; &.dashboard-projects { user-select: none; diff --git a/frontend/src/app/main/ui/dashboard/fonts.scss b/frontend/src/app/main/ui/dashboard/fonts.scss index 6c8817fda5..6543736c16 100644 --- a/frontend/src/app/main/ui/dashboard/fonts.scss +++ b/frontend/src/app/main/ui/dashboard/fonts.scss @@ -8,7 +8,7 @@ @use "common/refactor/common-dashboard"; .dashboard-fonts { - border-top: $s-1 solid $db-cuaternary; + border-top: $s-1 solid $db-quaternary; display: flex; flex-direction: column; padding-left: $s-120; @@ -161,7 +161,7 @@ .table-field { color: $df-primary; .variant { - background-color: $db-cuaternary; + background-color: $db-quaternary; border-radius: $br-8; margin-right: $s-4; padding-right: $s-4; @@ -196,7 +196,7 @@ &.failure { margin-right: $s-12; svg { - stroke: var(--warning-color); + stroke: var(--element-foreground-warning); } } @@ -262,7 +262,7 @@ background-color: $db-primary; border-radius: $br-12; - border: $s-1 solid $db-cuaternary; + border: $s-1 solid $db-quaternary; color: $df-primary; font-size: $fs-12; @@ -289,9 +289,9 @@ } } &.warning { - background-color: $db-cuaternary; + background-color: $db-quaternary; .icon svg { - stroke: var(--warning-color); + stroke: var(--element-foreground-warning); } } } @@ -304,7 +304,7 @@ .fonts-placeholder { align-items: center; border-radius: $br-8; - border: $s-1 solid $db-cuaternary; + border: $s-1 solid $db-quaternary; display: flex; flex-direction: column; height: $s-160; diff --git a/frontend/src/app/main/ui/dashboard/grid.scss b/frontend/src/app/main/ui/dashboard/grid.scss index b8521d9c24..98b10ac721 100644 --- a/frontend/src/app/main/ui/dashboard/grid.scss +++ b/frontend/src/app/main/ui/dashboard/grid.scss @@ -280,7 +280,7 @@ $thumbnail-default-height: $s-168; // Default width } :global(svg#loader-pencil) { - stroke: $db-cuaternary; + stroke: $db-quaternary; width: calc(var(--th-width, #{$thumbnail-default-width}) * 0.25); } } @@ -345,7 +345,7 @@ $thumbnail-default-height: $s-168; // Default width } svg { - background-color: var(--canvas-color); + background-color: var(--color-canvas); border-radius: $br-4; border: $s-2 solid transparent; height: $s-24; diff --git a/frontend/src/app/main/ui/dashboard/import.scss b/frontend/src/app/main/ui/dashboard/import.scss index fc726714e4..e4b3a7bb7b 100644 --- a/frontend/src/app/main/ui/dashboard/import.scss +++ b/frontend/src/app/main/ui/dashboard/import.scss @@ -39,8 +39,8 @@ width: 100%; margin-bottom: $s-24; border-radius: $br-8; - background-color: var(--alert-background-color-ok); - color: var(--alert-foreground-color-ok); + background-color: var(--alert-background-color-success); + color: var(--alert-foreground-color-success); .icon { @include flexCenter; @@ -48,7 +48,7 @@ width: $s-24; svg { @extend .button-icon; - stroke: var(--alert-foreground-color-ok); + stroke: var(--alert-foreground-color-success); } } .message { @@ -155,7 +155,7 @@ } &.error { svg { - stroke: var(--status-error-color); + stroke: var(--element-foreground-error); } } } @@ -163,46 +163,46 @@ &.loading { .file-name { - color: var(--status-pending-color); + color: var(--element-foreground-pending); .file-icon { :global(#loader-pencil) { - color: var(--status-pending-color); - stroke: var(--status-pending-color); - fill: var(--status-pending-color); + color: var(--element-foreground-pending); + stroke: var(--element-foreground-pending); + fill: var(--element-foreground-pending); } } } } &.warning { .file-name { - color: var(--status-warning-color); + color: var(--element-foreground-warning); .file-icon svg { - stroke: var(--status-warning-color); + stroke: var(--element-foreground-warning); } .file-icon.icon-fill svg { - fill: var(--status-warning-color); + fill: var(--element-foreground-warning); } } } &.success { .file-name { - color: var(--status-success-color); + color: var(--element-foreground-success); .file-icon svg { - stroke: var(--status-success-color); + stroke: var(--element-foreground-success); } .file-icon.icon-fill svg { - fill: var(--status-success-color); + fill: var(--element-foreground-success); } } } &.error { .file-name { - color: var(--status-error-color); + color: var(--element-foreground-error); .file-icon svg { - stroke: var(--status-error-color); + stroke: var(--element-foreground-error); } .file-icon.icon-fill svg { - fill: var(--status-error-color); + fill: var(--element-foreground-error); } } } diff --git a/frontend/src/app/main/ui/dashboard/inline_edition.scss b/frontend/src/app/main/ui/dashboard/inline_edition.scss index 0c84951f9c..b2d0276cd5 100644 --- a/frontend/src/app/main/ui/dashboard/inline_edition.scss +++ b/frontend/src/app/main/ui/dashboard/inline_edition.scss @@ -15,7 +15,7 @@ } input.element-title { - background-color: $db-primary; + background-color: var(--input-background-color-active); border-radius: $br-8; color: $df-primary; font-size: $fs-16; @@ -47,7 +47,7 @@ input.element-title { } &:hover { svg { - fill: var(--warning-color); + fill: var(--element-foreground-warning); } } } diff --git a/frontend/src/app/main/ui/dashboard/libraries.scss b/frontend/src/app/main/ui/dashboard/libraries.scss index eb7864c9e2..69660e4f0a 100644 --- a/frontend/src/app/main/ui/dashboard/libraries.scss +++ b/frontend/src/app/main/ui/dashboard/libraries.scss @@ -12,7 +12,7 @@ margin-right: $s-16; overflow-y: auto; width: 100%; - border-top: $s-1 solid $db-cuaternary; + border-top: $s-1 solid $db-quaternary; &.dashboard-projects { user-select: none; diff --git a/frontend/src/app/main/ui/dashboard/placeholder.scss b/frontend/src/app/main/ui/dashboard/placeholder.scss index 829ad1d530..6f05ba0006 100644 --- a/frontend/src/app/main/ui/dashboard/placeholder.scss +++ b/frontend/src/app/main/ui/dashboard/placeholder.scss @@ -34,7 +34,7 @@ 85% top; background-repeat: no-repeat; align-items: center; - border: $s-1 solid $db-cuaternary; + border: $s-1 solid $db-quaternary; border-radius: $br-4; display: flex; flex-direction: column; @@ -75,7 +75,7 @@ &:hover { border: $s-2 solid $da-tertiary; - background-color: $db-cuaternary; + background-color: $db-quaternary; color: $da-primary; svg { diff --git a/frontend/src/app/main/ui/dashboard/projects.scss b/frontend/src/app/main/ui/dashboard/projects.scss index 48652ee378..4700ad366d 100644 --- a/frontend/src/app/main/ui/dashboard/projects.scss +++ b/frontend/src/app/main/ui/dashboard/projects.scss @@ -12,7 +12,7 @@ margin-right: $s-16; overflow-y: auto; width: 100%; - border-top: $s-1 solid $db-cuaternary; + border-top: $s-1 solid $db-quaternary; &.dashboard-projects { user-select: none; diff --git a/frontend/src/app/main/ui/dashboard/search.scss b/frontend/src/app/main/ui/dashboard/search.scss index 9eb379706f..0922089ad7 100644 --- a/frontend/src/app/main/ui/dashboard/search.scss +++ b/frontend/src/app/main/ui/dashboard/search.scss @@ -13,7 +13,7 @@ margin-right: $s-16; overflow-y: auto; width: 100%; - border-top: $s-1 solid $db-cuaternary; + border-top: $s-1 solid $db-quaternary; &.dashboard-projects { user-select: none; @@ -35,7 +35,7 @@ flex-direction: column; height: $s-200; background: transparent; - border: $s-1 solid $db-cuaternary; + border: $s-1 solid $db-quaternary; border-radius: $br-8; .text { diff --git a/frontend/src/app/main/ui/dashboard/sidebar.scss b/frontend/src/app/main/ui/dashboard/sidebar.scss index 8c0520ea4c..27b31d569b 100644 --- a/frontend/src/app/main/ui/dashboard/sidebar.scss +++ b/frontend/src/app/main/ui/dashboard/sidebar.scss @@ -11,8 +11,8 @@ grid-row: 1 / span 2; grid-column: 1 / span 2; - background-color: $db-primary; - border-right: $s-1 solid $db-cuaternary; + background-color: var(--panel-background-color); + border-right: $s-1 solid $db-quaternary; margin: 0 $s-16 0 0; padding: $s-16 0 0 0; @@ -141,7 +141,7 @@ position: absolute; z-index: $z-index-4; background-color: $db-tertiary; - border: $s-1 solid $db-cuaternary; + border: $s-1 solid $db-quaternary; border-radius: $br-8; .separator { @@ -161,11 +161,11 @@ padding: $s-6 $s-16; .warning { - color: var(--dark-warning-color); + color: var(--element-foreground-warning); } &:hover { - background-color: $db-cuaternary; + background-color: $db-quaternary; } svg { height: $s-12; @@ -182,7 +182,7 @@ li { color: $df-primary; &.warning { - color: var(--dark-warning-color); + color: var(--element-foreground-warning); } } } @@ -191,7 +191,7 @@ .teams-dropdown { background-color: $db-tertiary; border-radius: $br-8; - border: $s-1 solid $db-cuaternary; + border: $s-1 solid $db-quaternary; min-width: $s-248; left: 0; @@ -212,7 +212,7 @@ } &:hover { - background-color: $db-cuaternary; + background-color: $db-quaternary; .team-icon { &.new-team { background-color: $da-primary; @@ -233,7 +233,7 @@ } .new-team { - background-color: $db-cuaternary; + background-color: $db-quaternary; } &.action { @@ -425,7 +425,7 @@ } &.current { - background-color: $db-cuaternary; + background-color: $db-quaternary; .element-title { color: $da-primary; } @@ -441,7 +441,7 @@ position: relative; background-color: $db-tertiary; - border-top: $s-1 solid $db-cuaternary; + border-top: $s-1 solid $db-quaternary; .profile { align-items: center; @@ -474,8 +474,7 @@ .dropdown { left: $s-16; bottom: $s-44; - - background-color: $db-primary; + background-color: var(--profile-drowpdown-background-color); border: $s-1 solid $db-tertiary; border-radius: $br-8; min-width: $s-252; diff --git a/frontend/src/app/main/ui/dashboard/team.scss b/frontend/src/app/main/ui/dashboard/team.scss index d593616b2d..32afa09a09 100644 --- a/frontend/src/app/main/ui/dashboard/team.scss +++ b/frontend/src/app/main/ui/dashboard/team.scss @@ -12,7 +12,7 @@ margin-right: $s-16; overflow-y: auto; width: 100%; - border-top: $s-1 solid $db-cuaternary; + border-top: $s-1 solid $db-quaternary; &.dashboard-projects { user-select: none; @@ -41,7 +41,7 @@ align-items: center; background-color: transparent; border-radius: $br-8; - border: $s-1 solid $db-cuaternary; + border: $s-1 solid $db-quaternary; color: $df-secondary; display: flex; flex-direction: column; @@ -110,7 +110,7 @@ .dropdown { background-color: $db-tertiary; border-radius: $br-8; - border: $s-1 solid $db-cuaternary; + border: $s-1 solid $db-quaternary; box-shadow: 0 $s-2 $s-8 rgba(0, 0, 0, 0.25); left: calc(-1 * $s-144); max-height: $s-480; @@ -138,7 +138,7 @@ color: $df-primary; &:hover { - background-color: $db-cuaternary; + background-color: $db-quaternary; } &.title { @@ -156,7 +156,7 @@ li { color: $df-primary; &.warning { - color: var(--warning-color); + color: var(--element-foreground-warning); } } } @@ -237,7 +237,7 @@ align-items: center; padding: $s-4 $s-8; font-size: $fs-14; - background-color: $db-cuaternary; + background-color: $db-quaternary; border-color: transparent; border-radius: $br-8; } @@ -271,7 +271,7 @@ text-transform: uppercase; &.pending { - background-color: var(--warning-color); + background-color: var(--status-color-warning-500); } &.expired { @@ -303,7 +303,7 @@ height: $s-16; } .failure svg { - fill: var(--warning-color); + fill: var(--element-foreground-warning); width: $s-16; height: $s-16; } @@ -422,7 +422,7 @@ align-items: center; flex-direction: column; margin-top: $s-16; - border: $s-1 solid $db-cuaternary; + border: $s-1 solid $db-quaternary; border-radius: $br-8; color: $df-secondary; } diff --git a/frontend/src/app/main/ui/dashboard/templates.scss b/frontend/src/app/main/ui/dashboard/templates.scss index bec721aa6b..f70618c45a 100644 --- a/frontend/src/app/main/ui/dashboard/templates.scss +++ b/frontend/src/app/main/ui/dashboard/templates.scss @@ -41,7 +41,7 @@ margin-right: $s-32; position: relative; z-index: 1; - background-color: $db-cuaternary; + background-color: $db-quaternary; span { display: inline-block; @@ -107,7 +107,7 @@ margin-left: $s-6; position: absolute; border-top-left-radius: $s-8; - background-color: $db-cuaternary; + background-color: $db-quaternary; .card-container { width: $s-276; diff --git a/frontend/src/app/main/ui/debug/components_preview.cljs b/frontend/src/app/main/ui/debug/components_preview.cljs index 5a841ac8e8..e13ad747eb 100644 --- a/frontend/src/app/main/ui/debug/components_preview.cljs +++ b/frontend/src/app/main/ui/debug/components_preview.cljs @@ -42,7 +42,7 @@ colors ["var(--color-background-primary)" "var(--color-background-secondary)" "var(--color-background-tertiary)" - "var(--color-background-cuaternary)" + "var(--color-background-quaternary)" "var(--color-foreground-primary)" "var(--color-foreground-secondary)" "var(--color-accent-primary)" diff --git a/frontend/src/app/main/ui/export.scss b/frontend/src/app/main/ui/export.scss index 847cf306f8..cf72b301d5 100644 --- a/frontend/src/app/main/ui/export.scss +++ b/frontend/src/app/main/ui/export.scss @@ -202,7 +202,7 @@ height: 100%; min-height: $s-32; min-width: $s-32; - background-color: var(--white); + background-color: var(--app-white); border-radius: $br-6; margin: auto 0; img, @@ -276,28 +276,27 @@ } &.loading { .file-name { - color: var(--pending-color); + color: var(--element-foreground-pending); .file-icon svg:global(#loader-pencil) { - color: var(--pending-color); - stroke: var(--pending-color); - fill: var(--pending-color); + color: var(--element-foreground-pending); + stroke: var(--element-foreground-pending); + fill: var(--element-foreground-pending); } } } &.error { .file-name { - color: var(--error-color); + color: var(--element-foreground-error); .file-icon svg { - stroke: var(--error-color); + stroke: var(--element-foreground-error); } } } - &.success { .file-name { - color: var(--ok-color); + color: var(--element-foreground-success); .file-icon svg { - stroke: var(--ok-color); + stroke: var(--element-foreground-success); } } } diff --git a/frontend/src/app/main/ui/flex_controls/common.cljs b/frontend/src/app/main/ui/flex_controls/common.cljs index 052e501ee2..032760688f 100644 --- a/frontend/src/app/main/ui/flex_controls/common.cljs +++ b/frontend/src/app/main/ui/flex_controls/common.cljs @@ -8,9 +8,9 @@ ;; ------------------------------------------------ (def font-size 11) -(def distance-color "var(--color-distance)") -(def distance-text-color "var(--color-white)") -(def warning-color "var(--color-warning)") +(def distance-color "var(--da-quaternary)") +(def distance-text-color "var(--app-white)") +(def warning-color "var(--status-color-warning-500)") (def flex-display-pill-width 40) (def flex-display-pill-height 20) (def flex-display-pill-border-radius 4) diff --git a/frontend/src/app/main/ui/measurements.cljs b/frontend/src/app/main/ui/measurements.cljs index 2719b97391..370830fdb8 100644 --- a/frontend/src/app/main/ui/measurements.cljs +++ b/frontend/src/app/main/ui/measurements.cljs @@ -27,17 +27,17 @@ (def select-guide-width 1) (def select-guide-dasharray 5) -(def hover-color "var(--color-foreground-tertiary)") +(def hover-color "var(--color-accent-quaternary)") -(def size-display-color "var(--white)") +(def size-display-color "var(--app-white)") (def size-display-opacity 0.7) -(def size-display-text-color "var(--black)") +(def size-display-text-color "var(--app-black)") (def size-display-width-min 50) (def size-display-width-max 75) (def size-display-height 16) -(def distance-color "var(--color-foreground-tertiary)") -(def distance-text-color "var(--white)") +(def distance-color "var(--color-accent-quaternary)") +(def distance-text-color "var(--app-white)") (def distance-border-radius 2) (def distance-pill-width 50) (def distance-pill-height 16) diff --git a/frontend/src/app/main/ui/messages.scss b/frontend/src/app/main/ui/messages.scss index 03ad9dfed9..67e68c875b 100644 --- a/frontend/src/app/main/ui/messages.scss +++ b/frontend/src/app/main/ui/messages.scss @@ -23,8 +23,8 @@ } .success { - --bg-color: var(--alert-background-color-ok); - --fg-color: var(--alert-foreground-color-ok); + --bg-color: var(--alert-background-color-success); + --fg-color: var(--alert-foreground-color-success); } .info { diff --git a/frontend/src/app/main/ui/settings.scss b/frontend/src/app/main/ui/settings.scss index 2fc327282a..6bae8499fa 100644 --- a/frontend/src/app/main/ui/settings.scss +++ b/frontend/src/app/main/ui/settings.scss @@ -8,7 +8,7 @@ @use "common/refactor/common-dashboard"; .dashboard { - background-color: $db-primary; + background-color: var(--app-background); display: grid; grid-template-rows: $s-48 1fr; grid-template-columns: $s-40 $s-256 1fr; @@ -28,7 +28,7 @@ margin-right: $s-16; overflow-y: auto; width: 100%; - border-top: $s-1 solid $db-cuaternary; + border-top: $s-1 solid $db-quaternary; &.dashboard-projects { user-select: none; @@ -107,8 +107,8 @@ } &.disabled { input { - background-color: $db-primary; - border-color: $db-cuaternary; + background-color: var(--input-background-color-disabled); + border-color: $db-quaternary; color: $df-secondary; } } @@ -161,7 +161,7 @@ color: $df-primary; &:hover { color: $da-primary; - background-color: $db-cuaternary; + background-color: $db-quaternary; } } hr { diff --git a/frontend/src/app/main/ui/settings/access_tokens.scss b/frontend/src/app/main/ui/settings/access_tokens.scss index 50bdef0af3..44bfaf9c0a 100644 --- a/frontend/src/app/main/ui/settings/access_tokens.scss +++ b/frontend/src/app/main/ui/settings/access_tokens.scss @@ -85,7 +85,7 @@ .content { padding: $s-2 $s-6; &.expired { - background-color: var(--warning-color); + background-color: var(--status-color-warning-500); border-radius: $br-4; color: $db-secondary; } @@ -149,7 +149,7 @@ .access-tokens-empty { align-items: center; border-radius: $br-8; - border: $s-1 solid $db-cuaternary; + border: $s-1 solid $db-quaternary; color: $df-secondary; display: flex; flex-direction: column; diff --git a/frontend/src/app/main/ui/settings/profile.scss b/frontend/src/app/main/ui/settings/profile.scss index 59e22acbbc..666d648e66 100644 --- a/frontend/src/app/main/ui/settings/profile.scss +++ b/frontend/src/app/main/ui/settings/profile.scss @@ -39,7 +39,7 @@ color: $df-primary; &:hover { color: $da-primary; - background-color: $db-cuaternary; + background-color: $db-quaternary; } } hr { @@ -110,8 +110,8 @@ } &.disabled { input { - background-color: $db-primary; - border-color: $db-cuaternary; + background-color: var(--input-background-color-disabled); + border-color: $db-quaternary; color: $df-secondary; } } diff --git a/frontend/src/app/main/ui/settings/sidebar.scss b/frontend/src/app/main/ui/settings/sidebar.scss index f0eb99029a..afb7cb38ff 100644 --- a/frontend/src/app/main/ui/settings/sidebar.scss +++ b/frontend/src/app/main/ui/settings/sidebar.scss @@ -7,8 +7,8 @@ @use "common/refactor/common-refactor.scss" as *; .dashboard-sidebar { - background-color: $db-primary; - border-right: $s-1 solid $db-cuaternary; + background-color: var(--panel-background-color); + border-right: $s-1 solid $db-quaternary; display: flex; flex-direction: column; grid-column: 1 / span 2; @@ -112,14 +112,14 @@ } &:hover { - background-color: $db-cuaternary; + background-color: $db-quaternary; &::before { background-color: $df-primary; } } &.current { - background-color: $db-cuaternary; + background-color: $db-quaternary; color: $da-primary; a { diff --git a/frontend/src/app/main/ui/shapes/grid_layout_viewer.cljs b/frontend/src/app/main/ui/shapes/grid_layout_viewer.cljs index b0f2b3482e..5cd437eada 100644 --- a/frontend/src/app/main/ui/shapes/grid_layout_viewer.cljs +++ b/frontend/src/app/main/ui/shapes/grid_layout_viewer.cljs @@ -38,11 +38,11 @@ :y area-y :width area-width :height area-height - :style {:fill "var(--color-foreground-tertiary)" + :style {:fill "var(--color-accent-quaternary)" :fill-opacity 0.3}}] [:text {:x area-text-x :y area-text-y - :style {:fill "var(--color-foreground-tertiary)" + :style {:fill "var(--color-accent-quaternary)" :font-family "worksans" :font-weight 600 :font-size 14 @@ -71,7 +71,7 @@ :y (:y cell-origin) :width cell-width :height cell-height - :style {:stroke "var(--color-foreground-tertiary)" + :style {:stroke "var(--color-accent-quaternary)" :stroke-width 1.5 :fill "none"}}] diff --git a/frontend/src/app/main/ui/viewer/inspect/left_sidebar.scss b/frontend/src/app/main/ui/viewer/inspect/left_sidebar.scss index 889d480ad1..503b80907e 100644 --- a/frontend/src/app/main/ui/viewer/inspect/left_sidebar.scss +++ b/frontend/src/app/main/ui/viewer/inspect/left_sidebar.scss @@ -7,7 +7,7 @@ @use "common/refactor/common-refactor.scss" as *; .settings-bar-left { - background-color: $db-primary; + background-color: var(--panel-background-color); height: 100%; width: $s-256; } diff --git a/frontend/src/app/main/ui/workspace/libraries.scss b/frontend/src/app/main/ui/workspace/libraries.scss index e91cedb36d..38e6836c8d 100644 --- a/frontend/src/app/main/ui/workspace/libraries.scss +++ b/frontend/src/app/main/ui/workspace/libraries.scss @@ -190,7 +190,7 @@ } & svg { - background-color: var(--canvas-color); + background-color: var(--color-canvas); border-radius: $br-4; border: $s-2 solid transparent; height: $s-24; diff --git a/frontend/src/app/main/ui/workspace/right_header.scss b/frontend/src/app/main/ui/workspace/right_header.scss index 9eafbcb7d9..a56f736296 100644 --- a/frontend/src/app/main/ui/workspace/right_header.scss +++ b/frontend/src/app/main/ui/workspace/right_header.scss @@ -188,16 +188,16 @@ @extend .button-icon; height: $s-12; width: $s-12; - stroke: var(--status-icon-foreground-color); + stroke: var(--status-widget-icon-foreground-color); } } .pending-status { - background-color: var(--status-warning-background-color); + background-color: var(--status-widget-background-color-warning); } .saving-status { - background-color: var(--status-pending-background-color); + background-color: var(--status-widget-background-color-pending); svg { animation: spin-animation 1s infinite; animation-timing-function: linear; @@ -205,11 +205,11 @@ } .saved-status { - background-color: var(--status-ok-background-color); + background-color: var(--status-widget-background-color-success); } .error-status { - background-color: var(--status-error-background-color); + background-color: var(--status-widget-background-color-error); } .viewer-btn { diff --git a/frontend/src/app/main/ui/workspace/shapes/path/common.cljs b/frontend/src/app/main/ui/workspace/shapes/path/common.cljs index 8c13c32450..b6d976a9e2 100644 --- a/frontend/src/app/main/ui/workspace/shapes/path/common.cljs +++ b/frontend/src/app/main/ui/workspace/shapes/path/common.cljs @@ -14,10 +14,10 @@ [rumext.v2 :as mf])) (def primary-color "var(--color-accent-tertiary)") -(def secondary-color "var(--color-foreground-tertiary)") -(def black-color "var(--black)") -(def white-color "var(--white)") -(def gray-color "var(--off-white)") +(def secondary-color "var(--color-accent-quaternary)") +(def black-color "var(--app-black)") +(def white-color "var(--app-white)") +(def gray-color "var(--df-secondary)") (def current-edit-path-ref (l/derived diff --git a/frontend/src/app/main/ui/workspace/sidebar/debug_shape_info.scss b/frontend/src/app/main/ui/workspace/sidebar/debug_shape_info.scss index 0e910e7f5c..59c2e3fc60 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/debug_shape_info.scss +++ b/frontend/src/app/main/ui/workspace/sidebar/debug_shape_info.scss @@ -53,7 +53,7 @@ .shape-title { font-size: $fs-14; padding-bottom: $s-4; - background: $db-cuaternary; + background: $db-quaternary; color: $df-primary; padding: $s-8; border-radius: $s-8; diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/layout_container.scss b/frontend/src/app/main/ui/workspace/sidebar/options/menus/layout_container.scss index cc51a740bd..d52e12d334 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/layout_container.scss +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/layout_container.scss @@ -359,7 +359,7 @@ border-radius: $br-6; &:hover { - background: $db-cuaternary; + background: $db-quaternary; } } } diff --git a/frontend/src/app/main/ui/workspace/viewport/gradients.cljs b/frontend/src/app/main/ui/workspace/viewport/gradients.cljs index 5840009800..e2bf517d1a 100644 --- a/frontend/src/app/main/ui/workspace/viewport/gradients.cljs +++ b/frontend/src/app/main/ui/workspace/viewport/gradients.cljs @@ -22,13 +22,13 @@ [rumext.v2 :as mf])) (def gradient-line-stroke-width 2) -(def gradient-line-stroke-color "var(--white)") +(def gradient-line-stroke-color "var(--app-white)") (def gradient-square-width 15) (def gradient-square-radius 2) (def gradient-square-stroke-width 2) (def gradient-width-handler-radius 5) -(def gradient-width-handler-color "var(--white)") -(def gradient-square-stroke-color "var(--white)") +(def gradient-width-handler-color "var(--app-white)") +(def gradient-square-stroke-color "var(--app-white)") (def gradient-square-stroke-color-selected "var(--color-accent-tertiary)") (mf/defc shadow [{:keys [id x y width height offset]}] @@ -109,7 +109,7 @@ :rx (/ gradient-square-radius zoom) :width (/ gradient-square-width zoom) :height (/ gradient-square-width zoom) - :stroke (if selected "var(--color-accent-tertiary)" "var(--white)") + :stroke (if selected "var(--color-accent-tertiary)" "var(--app-white)") :stroke-width (/ gradient-square-stroke-width zoom) :fill (:value color) :fill-opacity (:opacity color) diff --git a/frontend/src/app/main/ui/workspace/viewport/grid_layout_editor.scss b/frontend/src/app/main/ui/workspace/viewport/grid_layout_editor.scss index 9c50cc2f49..48c4186d42 100644 --- a/frontend/src/app/main/ui/workspace/viewport/grid_layout_editor.scss +++ b/frontend/src/app/main/ui/workspace/viewport/grid_layout_editor.scss @@ -11,7 +11,7 @@ fill: var(--grid-editor-marker-color); } .marker-text { - fill: var(--white); + fill: var(--app-white); font-size: calc($s-12 / var(--zoom)); font-family: worksans; } diff --git a/frontend/src/app/main/ui/workspace/viewport/interactions.cljs b/frontend/src/app/main/ui/workspace/viewport/interactions.cljs index f7a21f6dfd..4f46f4133b 100644 --- a/frontend/src/app/main/ui/workspace/viewport/interactions.cljs +++ b/frontend/src/app/main/ui/workspace/viewport/interactions.cljs @@ -131,7 +131,7 @@ (when icon-pdata [:path {:fill stroke :stroke-width 2 - :stroke "var(--white)" + :stroke "var(--app-white)" :d icon-pdata :transform (str "scale(" inv-zoom ", " inv-zoom ") " @@ -164,7 +164,7 @@ (if-not selected? [:g {:on-pointer-down #(on-pointer-down % index orig-shape)} - [:path {:stroke "var(--off-white)" + [:path {:stroke "var(--df-secondary)" :fill "none" :pointer-events "visible" :stroke-width (/ 2 zoom) @@ -173,7 +173,7 @@ [:& interaction-marker {:index index :x dest-x :y dest-y - :stroke "var(--off-white)" + :stroke "var(--df-secondary)" :action-type action-type :arrow-dir arrow-dir :zoom zoom}])] @@ -257,7 +257,7 @@ [:& (mf/provider embed/context) {:value false} [:& shape-wrapper {:shape dest-shape}]]]] [:path {:stroke "var(--color-accent-tertiary)" - :fill "var(--black)" + :fill "var(--app-black)" :fill-opacity 0.5 :stroke-width 1 :d (dm/str "M" marker-x " " marker-y " " diff --git a/frontend/src/app/main/ui/workspace/viewport/presence.cljs b/frontend/src/app/main/ui/workspace/viewport/presence.cljs index 0b4c7a091d..ffd4a59bfd 100644 --- a/frontend/src/app/main/ui/workspace/viewport/presence.cljs +++ b/frontend/src/app/main/ui/workspace/viewport/presence.cljs @@ -25,8 +25,8 @@ [{:keys [session profile] :as props}] (let [zoom (mf/deref refs/selected-zoom) point (:point session) - background-color (:color session "var(--black)") - text-color (:text-color session "var(--white)") + background-color (:color session "var(--app-black)") + text-color (:text-color session "var(--app-white)") transform (str/fmt "translate(%s, %s) scale(%s)" (:x point) (:y point) (/ 1 zoom)) shown-name (if (> (count (:fullname profile)) 16) (str (str/slice (:fullname profile) 0 12) "...") diff --git a/frontend/src/app/main/ui/workspace/viewport/selection.cljs b/frontend/src/app/main/ui/workspace/viewport/selection.cljs index a8202db7f0..ed91e2b6ca 100644 --- a/frontend/src/app/main/ui/workspace/viewport/selection.cljs +++ b/frontend/src/app/main/ui/workspace/viewport/selection.cljs @@ -213,7 +213,7 @@ :style {:fillOpacity "1" :strokeWidth "1px" :vectorEffect "non-scaling-stroke"} - :fill "var(--white)" + :fill "var(--app-white)" :stroke color :cx cx' :cy cy'}] @@ -279,7 +279,7 @@ :style {:fillOpacity 1 :stroke color :strokeWidth "1px" - :fill "var(--white)" + :fill "var(--app-white)" :vectorEffect "non-scaling-stroke"} :data-position (name position) :cx (+ x (/ length 2)) diff --git a/frontend/src/app/main/ui/workspace/viewport/snap_distances.cljs b/frontend/src/app/main/ui/workspace/viewport/snap_distances.cljs index 273813a93c..c903a19389 100644 --- a/frontend/src/app/main/ui/workspace/viewport/snap_distances.cljs +++ b/frontend/src/app/main/ui/workspace/viewport/snap_distances.cljs @@ -21,7 +21,7 @@ [cuerdas.core :as str] [rumext.v2 :as mf])) -(def ^:private line-color "var(--color-foreground-tertiary)") +(def ^:private line-color "var(--color-accent-quaternary)") (def ^:private segment-gap 2) (def ^:private segment-gap-side 5) @@ -85,7 +85,7 @@ [:text {:x (if (= coord :x) x (+ x (/ width 2))) :y (- (+ y (/ (/ pill-text-height zoom) 2) (- (/ 6 zoom))) (if (= coord :x) (/ 2 zoom) 0)) :font-size (/ pill-text-font-size zoom) - :fill "var(--white)" + :fill "var(--app-white)" :text-anchor "middle"} (fmt/format-number distance)]]) diff --git a/frontend/src/app/main/ui/workspace/viewport/snap_points.cljs b/frontend/src/app/main/ui/workspace/viewport/snap_points.cljs index 3e5a0c190b..d65ae80f06 100644 --- a/frontend/src/app/main/ui/workspace/viewport/snap_points.cljs +++ b/frontend/src/app/main/ui/workspace/viewport/snap_points.cljs @@ -16,7 +16,7 @@ [beicon.v2.core :as rx] [rumext.v2 :as mf])) -(def ^:private line-color "var(--color-foreground-tertiary)") +(def ^:private line-color "var(--color-accent-quaternary)") (def ^:private line-opacity 0.6) (def ^:private line-width 1) From 153bb752a4924cdd82fc36435acdc30d8b2ba35a Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 30 Jan 2024 10:40:28 +0100 Subject: [PATCH 2/7] :recycle: Add new exceptions for light theme --- .../styles/common/refactor/basic-rules.scss | 14 ++--- .../styles/common/refactor/color-defs.scss | 1 + .../styles/common/refactor/design-tokens.scss | 44 ++++++++++++--- .../common/refactor/themes/light-theme.scss | 3 +- .../main/ui/components/editable_label.scss | 6 +-- .../app/main/ui/components/radio_buttons.scss | 18 ++++--- .../app/main/ui/components/tab_container.scss | 8 +-- .../main/ui/components/tabs_container.cljs | 53 ------------------- .../src/app/main/ui/dashboard/comments.scss | 4 +- .../src/app/main/ui/dashboard/import.scss | 26 ++++----- .../src/app/main/ui/dashboard/pin_button.scss | 10 ++-- frontend/src/app/main/ui/export.scss | 16 +++--- .../src/app/main/ui/workspace/comments.scss | 7 +-- .../src/app/main/ui/workspace/palette.scss | 6 +-- .../app/main/ui/workspace/right_header.scss | 6 ++- .../src/app/main/ui/workspace/sidebar.scss | 5 +- .../app/main/ui/workspace/sidebar/assets.cljs | 3 +- .../app/main/ui/workspace/sidebar/assets.scss | 4 ++ .../workspace/sidebar/assets/components.scss | 8 +-- .../app/main/ui/workspace/sidebar/layers.cljs | 1 + .../app/main/ui/workspace/sidebar/layers.scss | 3 ++ .../sidebar/options/menus/grid_cell.scss | 2 +- .../sidebar/options/menus/interactions.scss | 5 +- .../options/menus/layout_container.scss | 10 +--- .../sidebar/options/menus/layout_item.scss | 6 +-- .../sidebar/options/menus/measures.cljs | 2 +- .../sidebar/options/menus/measures.scss | 10 +--- .../app/main/ui/workspace/top_toolbar.scss | 7 +-- 28 files changed, 131 insertions(+), 157 deletions(-) delete mode 100644 frontend/src/app/main/ui/components/tabs_container.cljs diff --git a/frontend/resources/styles/common/refactor/basic-rules.scss b/frontend/resources/styles/common/refactor/basic-rules.scss index 272577023d..638867c89e 100644 --- a/frontend/resources/styles/common/refactor/basic-rules.scss +++ b/frontend/resources/styles/common/refactor/basic-rules.scss @@ -127,7 +127,7 @@ border-radius: $br-8; color: var(--button-tertiary-foreground-color-rest); background-color: transparent; - border: $s-1 solid transparent; + border: $s-2 solid transparent; svg, span svg { stroke: var(--button-tertiary-foreground-color-rest); @@ -135,7 +135,7 @@ &:hover { background-color: var(--button-tertiary-background-color-hover); color: var(--button-tertiary-foreground-color-hover); - border: $s-1 solid var(--button-secondary-border-color-hover); + border: $s-2 solid var(--button-secondary-border-color-hover); svg, span svg { stroke: var(--button-tertiary-foreground-color-hover); @@ -143,7 +143,7 @@ } &:active { outline: none; - border: $s-1 solid transparent; + border: $s-2 solid transparent; background-color: var(--button-tertiary-background-color-active); color: var(--button-tertiary-foreground-color-active); svg, @@ -168,11 +168,11 @@ .button-icon-selected { outline: none; - border: $s-1 solid transparent; - background-color: var(--button-tertiary-background-color-hover); - color: var(--button-tertiary-foreground-color-active); + border: $s-2 solid var(--button-icon-border-color-selected); + background-color: var(--button-icon-background-color-selected); + color: var(--button-icon-foreground-color-selected); svg { - stroke: var(--button-tertiary-foreground-color-active); + stroke: var(--button-icon-foreground-color-selected); } } diff --git a/frontend/resources/styles/common/refactor/color-defs.scss b/frontend/resources/styles/common/refactor/color-defs.scss index 6c47a842f7..24b66209b9 100644 --- a/frontend/resources/styles/common/refactor/color-defs.scss +++ b/frontend/resources/styles/common/refactor/color-defs.scss @@ -42,6 +42,7 @@ --lf-primary: #000; --lf-secondary: #495e74; --lf-secondary-40: #{color.change(#495e74, $alpha: 0.4)}; + --lf-secondary-50: #{color.change(#495e74, $alpha: 0.5)}; //Light accent --la-primary: #6911d4; diff --git a/frontend/resources/styles/common/refactor/design-tokens.scss b/frontend/resources/styles/common/refactor/design-tokens.scss index fe73cd6aff..0e80da0588 100644 --- a/frontend/resources/styles/common/refactor/design-tokens.scss +++ b/frontend/resources/styles/common/refactor/design-tokens.scss @@ -63,6 +63,12 @@ --button-tertiary-border-color-focus: var(--color-accent-primary); --button-tertiary-foreground-color-focus: var(--color-foreground-primary); + --button-icon-foreground-color: var(--color-foreground-secondary); + --button-icon-foreground-color-hover: var(--color-foreground-secondary); + --button-icon-background-color-selected: var(--color-background-quaternary); + --button-icon-foreground-color-selected: var(--color-accent-primary); + --button-icon-border-color-selected: var(--color-background-quaternary); + --button-radio-background-color-rest: var(--color-background-tertiary); --button-radio-border-color-rest: var(--color-background-tertiary); --button-radio-foreground-color-rest: var(--color-foreground-secondary); @@ -92,11 +98,14 @@ --constraint-center-area-background-color: var(--color-background-primary); // TABS + --tabs-background-color: var(--color-background-secondary); --tab-background-color-hover: var(--color-background-primary); --tab-background-color-selected: var(--color-background-quaternary); --tab-foreground-color: var(--color-foreground-secondary); --tab-foreground-color-hover: var(--color-foreground-primary); --tab-foreground-color-selected: var(--color-accent-primary); + --tab-border-color: var(--color-background-secondary); + --tab-border-color-selected: var(--color-background-secondary); // SECTION TITLE --title-background-color: var(--color-background-primary); @@ -213,16 +222,14 @@ --assets-item-background-color: var(--color-background-tertiary); --assets-item-background-color-hover: var(--color-background-quaternary); --assets-item-name-background-color: var(--db-secondary-80); // TODO: penpot file has a non-existing token - --assets-item-name-foreground-color: var(--color-foreground-secondary); + --assets-item-name-foreground-color: var(--color-foreground-primary); --assets-item-name-foreground-color-hover: var(--color-foreground-primary); --assets-item-name-foreground-color-disabled: var(--color-foreground-disabled); --assets-item-border-color: var(--color-accent-primary); - --assets-item-background-color-drag: var(--color-accent-primary-muted); - --assets-item-border-color-drag: var(--color-accent-tertiary); - --assets-component-background-color: var(--app-white); // We don't want this color to change with palette - --assets-component-background-color-disabled: var( - --df-secondary; - ); // We don't want this color to change with palette + --assets-item-background-color-drag: transparent; + --assets-item-border-color-drag: var(--color-accent-primary-muted); + --assets-component-background-color: var(--app-white); // TODO: review this token + --assets-component-background-color-disabled: var(--df-secondary;); --assets-component-border-color: var(--color-background-tertiary); --assets-component-border-selected: var(--color-accent-tertiary); @@ -230,6 +237,7 @@ --radio-btn-background-color-selected: var(--color-background-quaternary); --radio-btn-foreground-color: var(--color-foreground-secondary); --radio-btn-foreground-color-selected: var(--color-accent-primary); + --radio-btn-border-color: var(--color-background-tertiary); --radio-btn-border-color-selected: var(--color-background-quaternary); --library-name-foreground-color: var(--color-foreground-primary); @@ -354,3 +362,25 @@ #app { background-color: var(--app-background); } + +.light { + --assets-component-background-color: var(--color-background-secondary); + + --tabs-background-color: var(--color-background-tertiary); + --tab-background-color-selected: var(--color-background-primary); + --tab-border-color: var(--color-background-tertiary); + --tab-border-color-selected: var(--color-background-secondary); + + --radio-btns-background-color: var(--color-background-tertiary); + --radio-btn-background-color-selected: var(--color-background-primary); + --radio-btn-foreground-color: var(--color-foreground-secondary); + --radio-btn-foreground-color-selected: var(--color-accent-primary); + --radio-btn-border-color: var(--color-background-tertiary); + --radio-btn-border-color-selected: var(--color-background-secondary); + + --button-icon-background-color-selected: var(--color-background-primary); + --button-icon-border-color-selected: var(--color-background-secondary); + + --assets-item-name-background-color: var(--color-background-primary); + --assets-item-name-foreground-color: var(--color-foreground-primary); +} diff --git a/frontend/resources/styles/common/refactor/themes/light-theme.scss b/frontend/resources/styles/common/refactor/themes/light-theme.scss index ba5fcb7a7c..e2fe1b643e 100644 --- a/frontend/resources/styles/common/refactor/themes/light-theme.scss +++ b/frontend/resources/styles/common/refactor/themes/light-theme.scss @@ -24,8 +24,7 @@ --color-accent-quaternary: var(--la-quaternary); --color-component-highlight: var(--la-secondary); - --overlay-color: rgba(255, 255, 255, 0.4); - + --overlay-color: var(--lf-secondary-50); --shadow-color: var(--lf-secondary-40); --radio-button-box-shadow: 0 0 0 1px var(--lb-secondary) inset; diff --git a/frontend/src/app/main/ui/components/editable_label.scss b/frontend/src/app/main/ui/components/editable_label.scss index b444a3be7e..168d73ad14 100644 --- a/frontend/src/app/main/ui/components/editable_label.scss +++ b/frontend/src/app/main/ui/components/editable_label.scss @@ -15,9 +15,9 @@ max-width: calc(var(--parent-size) - (var(--depth) * var(--layer-indentation-size))); margin: 0; padding-left: $s-6; - border-radius: $br-8; - border: $s-1 solid var(--input-border-color-focus); - color: var(--layer-row-foreground-color); + border-radius: $br-4; + border: $s-1 solid var(--input-border-color-active); + color: var(--input-foreground-color-active); } .editable-label { diff --git a/frontend/src/app/main/ui/components/radio_buttons.scss b/frontend/src/app/main/ui/components/radio_buttons.scss index 0c512f3e86..62e5c612f6 100644 --- a/frontend/src/app/main/ui/components/radio_buttons.scss +++ b/frontend/src/app/main/ui/components/radio_buttons.scss @@ -12,13 +12,16 @@ height: $s-32; background-color: var(--input-background-color); } + .radio-icon { - @extend .button-radio; + @include buttonStyle; + @include flexCenter; + @include focusRadio; height: $s-32; flex-grow: 1; border-radius: $s-8; - box-sizing: content-box; - border: none; + box-sizing: border-box; + border: $s-2 solid var(--radio-btn-border-color); input { display: none; } @@ -31,16 +34,14 @@ color: var(--radio-btn-foreground-color); } &:hover { - border: none; svg { - color: var(--radio-btn-foreground-color-selected); + stroke: var(--radio-btn-foreground-color-selected); } } &.checked { - border: none; background-color: var(--radio-btn-background-color-selected); - box-shadow: var(--radio-button-box-shadow); + border-color: var(--radio-btn-border-color-selected); svg { stroke: var(--radio-btn-foreground-color-selected); } @@ -52,6 +53,7 @@ &.disabled { cursor: default; background-color: transparent; + border: $s-2 solid transparent; svg { stroke: var(--button-foreground-color-disabled); } @@ -59,8 +61,8 @@ color: var(--button-foreground-color-disabled); } &:hover { - border: none; background-color: transparent; + border: $s-2 solid transparent; svg { stroke: var(--button-foreground-color-disabled); } diff --git a/frontend/src/app/main/ui/components/tab_container.scss b/frontend/src/app/main/ui/components/tab_container.scss index 2db5c9df33..d859680cfc 100644 --- a/frontend/src/app/main/ui/components/tab_container.scss +++ b/frontend/src/app/main/ui/components/tab_container.scss @@ -17,8 +17,7 @@ flex-direction: row; gap: $s-2; border-radius: $br-8; - background: var(--color-background-secondary); - padding: $s-2; + background: var(--tabs-background-color); cursor: pointer; font-size: $fs-12; height: 100%; @@ -27,7 +26,6 @@ flex-direction: row; height: 100%; width: 100%; - gap: $s-2; .tab-container-tab-title { @include flexCenter; @include tabTitleTipography; @@ -35,10 +33,11 @@ width: 100%; padding: 0 $s-8; margin: 0; - border-radius: $br-5; + border-radius: $br-8; background-color: transparent; color: var(--tab-foreground-color); white-space: nowrap; + border: $s-2 solid var(--tab-border-color); svg { @extend .button-icon; stroke: var(--tab-foreground-color); @@ -47,6 +46,7 @@ &.current, &.current:hover { background: var(--tab-background-color-selected); + border-color: var(--tab-border-color-selected); color: var(--tab-foreground-color-selected); svg { stroke: var(--tab-foreground-color-selected); diff --git a/frontend/src/app/main/ui/components/tabs_container.cljs b/frontend/src/app/main/ui/components/tabs_container.cljs deleted file mode 100644 index 2275d06feb..0000000000 --- a/frontend/src/app/main/ui/components/tabs_container.cljs +++ /dev/null @@ -1,53 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) KALEIDOS INC - -(ns app.main.ui.components.tabs-container - (:require - [app.common.data :as d] - [cuerdas.core :as str] - [rumext.v2 :as mf])) - -(mf/defc tabs-element - {::mf/wrap-props false} - [props] - (let [children (unchecked-get props "children")] - [:div.tab-element - [:div.tab-element-content children]])) - -(mf/defc tabs-container - {::mf/wrap-props false} - [props] - (let [children (->> - (unchecked-get props "children") - (filter some?)) - selected (unchecked-get props "selected") - on-change (unchecked-get props "on-change-tab") - - state (mf/use-state #(or selected (-> children first .-props .-id))) - selected (or selected @state) - - select-fn - (mf/use-fn - (mf/deps on-change) - (fn [event] - (let [id (d/read-string (.. event -target -dataset -id))] - (reset! state id) - (when (fn? on-change) (on-change id)))))] - - [:div.tab-container - [:div.tab-container-tabs - (for [tab children] - (let [props (.-props tab) - id (.-id props) - title (.-title props)] - [:div.tab-container-tab-title - {:key (str/concat "tab-" (d/name id)) - :data-id (pr-str id) - :on-click select-fn - :class (when (= selected id) "current")} - title]))] - [:div.tab-container-content - (d/seek #(= selected (-> % .-props .-id)) children)]])) diff --git a/frontend/src/app/main/ui/dashboard/comments.scss b/frontend/src/app/main/ui/dashboard/comments.scss index bdfcca1412..70b96cdaea 100644 --- a/frontend/src/app/main/ui/dashboard/comments.scss +++ b/frontend/src/app/main/ui/dashboard/comments.scss @@ -67,8 +67,8 @@ width: $s-32; svg { - min-width: $s-16; - min-height: $s-16; + width: $s-16; + height: $s-16; stroke: $df-secondary; fill: none; } diff --git a/frontend/src/app/main/ui/dashboard/import.scss b/frontend/src/app/main/ui/dashboard/import.scss index e4b3a7bb7b..ade895916e 100644 --- a/frontend/src/app/main/ui/dashboard/import.scss +++ b/frontend/src/app/main/ui/dashboard/import.scss @@ -163,12 +163,12 @@ &.loading { .file-name { - color: var(--element-foreground-pending); + color: var(--modal-text-foreground-color); .file-icon { :global(#loader-pencil) { - color: var(--element-foreground-pending); - stroke: var(--element-foreground-pending); - fill: var(--element-foreground-pending); + color: var(--modal-text-foreground-color); + stroke: var(--modal-text-foreground-color); + fill: var(--modal-text-foreground-color); } } } @@ -186,34 +186,34 @@ } &.success { .file-name { - color: var(--element-foreground-success); + color: var(--modal-text-foreground-color); .file-icon svg { - stroke: var(--element-foreground-success); + stroke: var(--modal-text-foreground-color); } .file-icon.icon-fill svg { - fill: var(--element-foreground-success); + fill: var(--modal-text-foreground-color); } } } &.error { .file-name { - color: var(--element-foreground-error); + color: var(--modal-text-foreground-color); .file-icon svg { - stroke: var(--element-foreground-error); + stroke: var(--modal-text-foreground-color); } .file-icon.icon-fill svg { - fill: var(--element-foreground-error); + fill: var(--modal-text-foreground-color); } } } &.editable { .file-name { - color: var(--icon-foreground); + color: var(--modal-text-foreground-color); .file-icon svg { - stroke: var(--icon-foreground); + stroke: var(--modal-text-foreground-color); } .file-icon.icon-fill svg { - fill: var(--icon-foreground); + fill: var(--modal-text-foreground-color); } } } diff --git a/frontend/src/app/main/ui/dashboard/pin_button.scss b/frontend/src/app/main/ui/dashboard/pin_button.scss index 50997fe243..9b00a1307c 100644 --- a/frontend/src/app/main/ui/dashboard/pin_button.scss +++ b/frontend/src/app/main/ui/dashboard/pin_button.scss @@ -7,13 +7,14 @@ @use "common/refactor/common-refactor.scss" as *; .button { - --pin-button-icon-color: #{$df-secondary}; + --pin-button-icon-color: var(--button-icon-foreground-color); --pin-button-bg-color: none; + --pin-button-border-color: none; width: $s-32; height: $s-32; background: var(--pin-button-bg-color); - border: none; + border: $s-2 solid var(--pin-button-border-color); border-radius: $br-8; display: grid; place-content: center; @@ -21,8 +22,9 @@ } .button-active { - --pin-button-icon-color: #{$da-primary}; - --pin-button-bg-color: #{$db-cuaternary}; + --pin-button-icon-color: var(--button-icon-foreground-color-selected); + --pin-button-bg-color: var(--button-icon-background-color-selected); + --pin-button-border-color: var(--button-icon-border-color-selected); } .icon { diff --git a/frontend/src/app/main/ui/export.scss b/frontend/src/app/main/ui/export.scss index cf72b301d5..16e280ba9c 100644 --- a/frontend/src/app/main/ui/export.scss +++ b/frontend/src/app/main/ui/export.scss @@ -276,27 +276,27 @@ } &.loading { .file-name { - color: var(--element-foreground-pending); + color: var(--modal-text-foreground-color); .file-icon svg:global(#loader-pencil) { - color: var(--element-foreground-pending); - stroke: var(--element-foreground-pending); - fill: var(--element-foreground-pending); + color: var(--modal-text-foreground-color); + stroke: var(--modal-text-foreground-color); + fill: var(--modal-text-foreground-color); } } } &.error { .file-name { - color: var(--element-foreground-error); + color: var(--modal-text-foreground-color); .file-icon svg { - stroke: var(--element-foreground-error); + stroke: var(--modal-text-foreground-color); } } } &.success { .file-name { - color: var(--element-foreground-success); + color: var(--modal-text-foreground-color); .file-icon svg { - stroke: var(--element-foreground-success); + stroke: var(--modal-text-foreground-color); } } } diff --git a/frontend/src/app/main/ui/workspace/comments.scss b/frontend/src/app/main/ui/workspace/comments.scss index 71c824b6af..b62b640f81 100644 --- a/frontend/src/app/main/ui/workspace/comments.scss +++ b/frontend/src/app/main/ui/workspace/comments.scss @@ -16,9 +16,7 @@ .comments-section-title { @include flexCenter; @include tabTitleTipography; - display: flex; - justify-content: space-between; - align-items: center; + position: relative; height: $s-32; min-height: $s-32; margin: $s-8 $s-8 0 $s-8; @@ -33,6 +31,9 @@ .close-button { @extend .button-tertiary; + position: absolute; + right: $s-2; + top: $s-2; height: $s-28; width: $s-28; border-radius: $br-6; diff --git a/frontend/src/app/main/ui/workspace/palette.scss b/frontend/src/app/main/ui/workspace/palette.scss index 56853ad43c..9d88d9e033 100644 --- a/frontend/src/app/main/ui/workspace/palette.scss +++ b/frontend/src/app/main/ui/workspace/palette.scss @@ -59,6 +59,7 @@ margin: $s-0; list-style: none; z-index: $z-index-2; + gap: $s-2; &.mid-palette, &.small-palette { display: flex; @@ -73,7 +74,6 @@ height: $s-32; width: $s-32; border-radius: $br-8; - border: $s-2 solid transparent; background-clip: padding-box; padding: 0; svg { @@ -83,12 +83,10 @@ &.selected { @extend .button-icon-selected; } - &:hover { - border: $s-2 solid transparent; - } } } } + .palette-actions { @extend .button-tertiary; grid-area: actions; diff --git a/frontend/src/app/main/ui/workspace/right_header.scss b/frontend/src/app/main/ui/workspace/right_header.scss index a56f736296..c7a45ab87b 100644 --- a/frontend/src/app/main/ui/workspace/right_header.scss +++ b/frontend/src/app/main/ui/workspace/right_header.scss @@ -128,6 +128,7 @@ margin: 0; height: $s-28; width: $s-28; + border: none; svg { @extend .button-icon; stroke: var(--icon-foreground); @@ -140,7 +141,6 @@ } &.selected { background-color: var(--button-tertiary-background-color-selected); - border: $s-2 solid var(--button-tertiary-border-color-selected); svg { stroke: var(--button-tertiary-foreground-color-active); } @@ -153,6 +153,7 @@ margin: 0; height: $s-28; width: $s-28; + border: none; svg { @extend .button-icon; stroke: var(--icon-foreground); @@ -165,7 +166,6 @@ } &.selected { background-color: var(--button-tertiary-background-color-selected); - border: $s-2 solid var(--button-tertiary-border-color-selected); svg { stroke: var(--button-tertiary-foreground-color-active); } @@ -218,6 +218,7 @@ margin: 0; width: $s-28; height: $s-28; + border: none; svg { @extend .button-icon; height: $s-16; @@ -226,5 +227,6 @@ } &:hover { background-color: transparent; + border: none; } } diff --git a/frontend/src/app/main/ui/workspace/sidebar.scss b/frontend/src/app/main/ui/workspace/sidebar.scss index af90a5ea57..57e839013b 100644 --- a/frontend/src/app/main/ui/workspace/sidebar.scss +++ b/frontend/src/app/main/ui/workspace/sidebar.scss @@ -30,6 +30,10 @@ $width-settings-bar-max: $s-500; } } +.layers-tab { + padding-top: $s-4; +} + .left-header { grid-area: header; } @@ -85,6 +89,5 @@ $width-settings-bar-max: $s-500; width: 100%; height: $s-12; border-top: $s-2 solid var(--resize-area-border-color); - background-color: var(--resize-area-background-color); cursor: ns-resize; } diff --git a/frontend/src/app/main/ui/workspace/sidebar/assets.cljs b/frontend/src/app/main/ui/workspace/sidebar/assets.cljs index d82ae14e11..62338f4ec4 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/assets.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/assets.cljs @@ -168,7 +168,8 @@ :placeholder (tr "workspace.assets.search")} [:button {:on-click on-open-menu - :class (stl/css :section-button)} + :class (stl/css-case :section-button true + :opened menu-open?)} i/filter-refactor]] [:& context-menu-a11y {:on-close on-menu-close diff --git a/frontend/src/app/main/ui/workspace/sidebar/assets.scss b/frontend/src/app/main/ui/workspace/sidebar/assets.scss index 27f8ca2180..723ac54fa3 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/assets.scss +++ b/frontend/src/app/main/ui/workspace/sidebar/assets.scss @@ -90,6 +90,10 @@ } } } + + &.opened { + @extend .button-icon-selected; + } } .sections-container { diff --git a/frontend/src/app/main/ui/workspace/sidebar/assets/components.scss b/frontend/src/app/main/ui/workspace/sidebar/assets/components.scss index baef288b8b..79fe504da0 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/assets/components.scss +++ b/frontend/src/app/main/ui/workspace/sidebar/assets/components.scss @@ -34,6 +34,7 @@ position: absolute; left: $s-4; bottom: $s-4; + height: $s-20; width: calc(100% - 2 * $s-4); padding: $s-2; column-gap: $s-4; @@ -50,10 +51,11 @@ span { display: flex; align-items: center; + height: 100%; } &.editing { border: $s-1 solid var(--input-border-color-focus); - border-radius: $br-2; + border-radius: $br-4; display: flex; align-items: center; background-color: var(--input-background-color); @@ -63,12 +65,12 @@ &:hover { background-color: var(--assets-item-background-color-hover); .cell-name { - display: flex; + display: block; } } &.selected { - border: $s-4 solid var(--assets-item-border-color); + border: $s-1 solid var(--assets-item-border-color); } } diff --git a/frontend/src/app/main/ui/workspace/sidebar/layers.cljs b/frontend/src/app/main/ui/workspace/sidebar/layers.cljs index 4ce3d31968..101ec3e93a 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/layers.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/layers.cljs @@ -239,6 +239,7 @@ [:button {:on-click toggle-filters :class (stl/css-case :filter-button true + :opened show-menu? :active active?)} i/filter-refactor]] diff --git a/frontend/src/app/main/ui/workspace/sidebar/layers.scss b/frontend/src/app/main/ui/workspace/sidebar/layers.scss index 55811f1102..4b39fbaa42 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/layers.scss +++ b/frontend/src/app/main/ui/workspace/sidebar/layers.scss @@ -49,6 +49,9 @@ stroke: var(--button-foreground-hover); } } + &.opened { + @extend .button-icon-selected; + } } .close-search { @extend .button-tertiary; diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/grid_cell.scss b/frontend/src/app/main/ui/workspace/sidebar/options/menus/grid_cell.scss index a2a3449f4e..602a132934 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/grid_cell.scss +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/grid_cell.scss @@ -52,5 +52,5 @@ .coord-input { @extend .input-element; border-radius: 0 $br-8 $br-8 0; - border-left: 1px solid var(--panel-background-color); + border-left: $s-1 solid var(--panel-background-color); } diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/interactions.scss b/frontend/src/app/main/ui/workspace/sidebar/options/menus/interactions.scss index e2f48a6e30..cc3679140f 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/interactions.scss +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/interactions.scss @@ -229,10 +229,7 @@ @extend .button-icon; } &.extended { - background-color: var(--button-radio-background-color-active); - svg { - stroke: var(--button-radio-foreground-color-active); - } + @extend .button-icon-selected; } } diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/layout_container.scss b/frontend/src/app/main/ui/workspace/sidebar/options/menus/layout_container.scss index d52e12d334..eb71a93928 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/layout_container.scss +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/layout_container.scss @@ -54,10 +54,7 @@ stroke: var(--icon-foreground); } &.selected { - background-color: var(--button-tertiary-background-color-hover); - svg { - stroke: var(--button-tertiary-foreground-color-active); - } + @extend .button-icon-selected; } } } @@ -138,10 +135,7 @@ stroke: var(--icon-foreground); } &.selected { - background-color: var(--button-tertiary-background-color-hover); - svg { - stroke: var(--button-tertiary-foreground-color-active); - } + @extend .button-icon-selected; } } } diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/layout_item.scss b/frontend/src/app/main/ui/workspace/sidebar/options/menus/layout_item.scss index a88b10213d..e876cbfafb 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/layout_item.scss +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/layout_item.scss @@ -83,11 +83,7 @@ @extend .button-icon; } &.selected { - background-color: var(--button-tertiary-background-color-active); - color: var(--button-tertiary-foreground-color-active); - svg { - stroke: var(--button-tertiary-foreground-color-active); - } + @extend .button-icon-selected; } } diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/measures.cljs b/frontend/src/app/main/ui/workspace/sidebar/options/menus/measures.cljs index 6956aa30fd..530ee7ed4d 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/measures.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/measures.cljs @@ -546,7 +546,7 @@ [:span {:class (stl/css :icon)} i/clip-content-refactor]]]) (when (options :show-in-viewer) - [:div {:class (stl/css :clip-content)} + [:div {:class (stl/css :show-in-viewer)} [:input {:type "checkbox" :id "show-in-viewer" :ref show-in-viewer-ref diff --git a/frontend/src/app/main/ui/workspace/sidebar/options/menus/measures.scss b/frontend/src/app/main/ui/workspace/sidebar/options/menus/measures.scss index f05c3e555e..c7e4e533bb 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/options/menus/measures.scss +++ b/frontend/src/app/main/ui/workspace/sidebar/options/menus/measures.scss @@ -203,10 +203,7 @@ stroke: var(--icon-foreground); } &.selected { - background-color: var(--button-tertiary-background-color-hover); - svg { - stroke: var(--button-tertiary-foreground-color-active); - } + @extend .button-icon-selected; } } @@ -235,10 +232,7 @@ } } &.selected { - background-color: var(--button-tertiary-background-color-hover); - svg { - stroke: var(--button-tertiary-foreground-color-active); - } + @extend .button-icon-selected; } } } diff --git a/frontend/src/app/main/ui/workspace/top_toolbar.scss b/frontend/src/app/main/ui/workspace/top_toolbar.scss index 03cfac6970..9777a3f7b0 100644 --- a/frontend/src/app/main/ui/workspace/top_toolbar.scss +++ b/frontend/src/app/main/ui/workspace/top_toolbar.scss @@ -39,7 +39,6 @@ height: $s-36; width: $s-36; flex-shrink: 0; - background-color: transparent; border-radius: $s-8; border: none; margin: 0 $s-2; @@ -48,11 +47,9 @@ @extend .button-icon; stroke: var(--color-foreground-secondary); } + &.selected { - background-color: var(--button-radio-background-color-active); - svg { - stroke: var(--button-radio-foreground-color-active); - } + @extend .button-icon-selected; } } } From 7f60946204d81788e6e6f3a19a0f5f256a55b4e1 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 22 Jan 2024 15:19:03 +0100 Subject: [PATCH 3/7] :recycle: Refactor exportation and duplicate mechanism Previously the file processing was implemented 3 times using similar approaches bug each own with its own bugs. This PR unifies the loging to a single implementation used by the 3 operations. --- backend/resources/app/templates/debug.tmpl | 11 - backend/src/app/binfile/common.clj | 491 ++++++++ backend/src/app/binfile/v1.clj | 762 +++++++++++++ backend/src/app/binfile/v2.clj | 429 ++----- backend/src/app/db/sql.clj | 5 +- backend/src/app/http/debug.clj | 64 +- backend/src/app/rpc/commands/binfile.clj | 1110 +------------------ backend/src/app/rpc/commands/management.clj | 517 +++------ backend/src/app/rpc/commands/teams.clj | 16 +- backend/src/app/srepl/main.clj | 4 +- backend/src/app/tasks/file_gc.clj | 32 +- frontend/src/app/worker/export.cljs | 4 +- 12 files changed, 1587 insertions(+), 1858 deletions(-) create mode 100644 backend/src/app/binfile/common.clj create mode 100644 backend/src/app/binfile/v1.clj diff --git a/backend/resources/app/templates/debug.tmpl b/backend/resources/app/templates/debug.tmpl index 0416a045df..caede1af89 100644 --- a/backend/resources/app/templates/debug.tmpl +++ b/backend/resources/app/templates/debug.tmpl @@ -145,17 +145,6 @@ Debug Main Page -
- - -
- - Do not break on index lookup errors (remap operation). - Useful when importing a broken file that has broken - relations or missing pieces. - -
-
diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj new file mode 100644 index 0000000000..aceb4ef7b6 --- /dev/null +++ b/backend/src/app/binfile/common.clj @@ -0,0 +1,491 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.binfile.common + "A binfile related file processing common code, used for different + binfile format implementations and management rpc methods." + (:require + [app.common.data :as d] + [app.common.data.macros :as dm] + [app.common.exceptions :as ex] + [app.common.features :as cfeat] + [app.common.files.defaults :as cfd] + [app.common.files.migrations :as fmg] + [app.common.files.validate :as fval] + [app.common.logging :as l] + [app.common.uuid :as uuid] + [app.config :as cf] + [app.db :as db] + [app.db.sql :as sql] + [app.features.components-v2 :as feat.compv2] + [app.features.fdata :as feat.fdata] + [app.loggers.audit :as-alias audit] + [app.loggers.webhooks :as-alias webhooks] + [app.util.blob :as blob] + [app.util.pointer-map :as pmap] + [app.util.time :as dt] + [app.worker :as-alias wrk] + [clojure.set :as set] + [clojure.walk :as walk] + [cuerdas.core :as str])) + +(set! *warn-on-reflection* true) + +(def ^:dynamic *state* nil) +(def ^:dynamic *options* nil) + +(def xf-map-id + (map :id)) + +(def xf-map-media-id + (comp + (mapcat (juxt :media-id + :thumbnail-id + :woff1-file-id + :woff2-file-id + :ttf-file-id + :otf-file-id)) + (filter uuid?))) + +(def into-vec + (fnil into [])) + +(def conj-vec + (fnil conj [])) + +(defn collect-storage-objects + [state items] + (update state :storage-objects into xf-map-media-id items)) + +(defn collect-summary + [state key items] + (update state key into xf-map-media-id items)) + +(defn lookup-index + [id] + (when id + (let [val (get-in @*state* [:index id])] + (l/trc :fn "lookup-index" :id (str id) :result (some-> val str) ::l/sync? true) + (or val id)))) + +(defn remap-id + [item key] + (cond-> item + (contains? item key) + (update key lookup-index))) + +(defn- index-object + [index obj & attrs] + (reduce (fn [index attr-fn] + (let [old-id (attr-fn obj) + new-id (if (::overwrite *options*) old-id (uuid/next))] + (assoc index old-id new-id))) + index + attrs)) + +(defn update-index + ([index coll] + (update-index index coll identity)) + ([index coll attr] + (reduce #(index-object %1 %2 attr) index coll))) + +(defn decode-row + "A generic decode row helper" + [{:keys [data features] :as row}] + (cond-> row + features (assoc :features (db/decode-pgarray features #{})) + data (assoc :data (blob/decode data)))) + +(defn get-file + [cfg file-id] + (db/run! cfg (fn [{:keys [::db/conn] :as cfg}] + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)] + (when-let [file (db/get* conn :file {:id file-id} + {::db/remove-deleted false})] + (-> file + (decode-row) + (update :data feat.fdata/process-pointers deref) + (update :data feat.fdata/process-objects (partial into {})))))))) + +(defn get-project + [cfg project-id] + (db/get cfg :project {:id project-id})) + +(defn get-team + [cfg team-id] + (-> (db/get cfg :team {:id team-id}) + (decode-row))) + +(defn get-fonts + [cfg team-id] + (db/query cfg :team-font-variant + {:team-id team-id + :deleted-at nil})) + +(defn get-files-rels + "Given a set of file-id's, return all matching relations with the libraries" + [cfg ids] + + (dm/assert! + "expected a set of uuids" + (and (set? ids) + (every? uuid? ids))) + + (db/run! cfg (fn [{:keys [::db/conn]}] + (let [ids (db/create-array conn "uuid" ids) + sql (str "SELECT flr.* FROM file_library_rel AS flr " + " JOIN file AS l ON (flr.library_file_id = l.id) " + " WHERE flr.file_id = ANY(?) AND l.deleted_at IS NULL")] + (db/exec! conn [sql ids]))))) + + +;; NOTE: Will be used in future, commented for satisfy linter +(def ^:private sql:get-libraries + "WITH RECURSIVE libs AS ( + SELECT fl.id + FROM file AS fl + JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id) + WHERE flr.file_id = ANY(?) + UNION + SELECT fl.id + FROM file AS fl + JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id) + JOIN libs AS l ON (flr.file_id = l.id) + ) + SELECT DISTINCT l.id + FROM libs AS l") + +(defn get-libraries + "Get all libraries ids related to provided file ids" + [cfg ids] + (db/run! cfg (fn [{:keys [::db/conn]}] + (let [ids' (db/create-array conn "uuid" ids)] + (->> (db/exec! conn [sql:get-libraries ids']) + (into #{} xf-map-id)))))) + +(defn get-file-object-thumbnails + "Return all file object thumbnails for a given file." + [cfg file-id] + (db/query cfg :file-tagged-object-thumbnail + {:file-id file-id + :deleted-at nil})) + +(defn get-file-thumbnail + "Return the thumbnail for the specified file-id" + [cfg {:keys [id revn]}] + (db/get* cfg :file-thumbnail + {:file-id id + :revn revn + :data nil} + {::sql/columns [:media-id :file-id :revn]})) + + +(def ^:private + xform:collect-media-id + (comp + (map :objects) + (mapcat vals) + (mapcat (fn [obj] + ;; NOTE: because of some bug, we ended with + ;; many shape types having the ability to + ;; have fill-image attribute (which initially + ;; designed for :path shapes). + (sequence + (keep :id) + (concat [(:fill-image obj) + (:metadata obj)] + (map :fill-image (:fills obj)) + (map :stroke-image (:strokes obj)) + (->> (:content obj) + (tree-seq map? :children) + (mapcat :fills) + (map :fill-image)))))))) + +(defn collect-used-media + "Given a fdata (file data), returns all media references." + [data] + (-> #{} + (into xform:collect-media-id (vals (:pages-index data))) + (into xform:collect-media-id (vals (:components data))) + (into (keys (:media data))))) + +(defn get-file-media + [cfg {:keys [data id] :as file}] + (db/run! cfg (fn [{:keys [::db/conn]}] + (let [ids (collect-used-media data) + ids (db/create-array conn "uuid" ids) + sql (str "SELECT * FROM file_media_object WHERE id = ANY(?)")] + + ;; We assoc the file-id again to the file-media-object row + ;; because there are cases that used objects refer to other + ;; files and we need to ensure in the exportation process that + ;; all ids matches + (->> (db/exec! conn [sql ids]) + (mapv #(assoc % :file-id id))))))) + +(def ^:private sql:get-team-files + "SELECT f.id FROM file AS f + JOIN project AS p ON (p.id = f.project_id) + WHERE p.team_id = ?") + +(defn get-team-files + "Get a set of file ids for the specified team-id" + [{:keys [::db/conn]} team-id] + (->> (db/exec! conn [sql:get-team-files team-id]) + (into #{} xf-map-id))) + +(def ^:private sql:get-team-projects + "SELECT p.id FROM project AS p + WHERE p.team_id = ? + AND p.deleted_at IS NULL") + +(defn get-team-projects + "Get a set of project ids for the team" + [{:keys [::db/conn]} team-id] + (->> (db/exec! conn [sql:get-team-projects team-id]) + (into #{} xf-map-id))) + +(def ^:private sql:get-project-files + "SELECT f.id FROM file AS f + WHERE f.project_id = ? + AND f.deleted_at IS NULL") + +(defn get-project-files + "Get a set of file ids for the project" + [{:keys [::db/conn]} project-id] + (->> (db/exec! conn [sql:get-project-files project-id]) + (into #{} xf-map-id))) + +(defn- relink-shapes + "A function responsible to analyze all file data and + replace the old :component-file reference with the new + ones, using the provided file-index." + [data] + (letfn [(process-map-form [form] + (cond-> form + ;; Relink image shapes + (and (map? (:metadata form)) + (= :image (:type form))) + (update-in [:metadata :id] lookup-index) + + ;; Relink paths with fill image + (map? (:fill-image form)) + (update-in [:fill-image :id] lookup-index) + + ;; This covers old shapes and the new :fills. + (uuid? (:fill-color-ref-file form)) + (update :fill-color-ref-file lookup-index) + + ;; This covers the old shapes and the new :strokes + (uuid? (:storage-color-ref-file form)) + (update :stroke-color-ref-file lookup-index) + + ;; This covers all text shapes that have typography referenced + (uuid? (:typography-ref-file form)) + (update :typography-ref-file lookup-index) + + ;; This covers the component instance links + (uuid? (:component-file form)) + (update :component-file lookup-index) + + ;; This covers the shadows and grids (they have directly + ;; the :file-id prop) + (uuid? (:file-id form)) + (update :file-id lookup-index))) + + (process-form [form] + (if (map? form) + (try + (process-map-form form) + (catch Throwable cause + (l/warn :hint "failed form" :form (pr-str form) ::l/sync? true) + (throw cause))) + form))] + + (walk/postwalk process-form data))) + +(defn- relink-media + "A function responsible of process the :media attr of file data and + remap the old ids with the new ones." + [media] + (reduce-kv (fn [res k v] + (let [id (lookup-index k)] + (if (uuid? id) + (-> res + (assoc id (assoc v :id id)) + (dissoc k)) + res))) + media + media)) + +(defn- relink-colors + "A function responsible of process the :colors attr of file data and + remap the old ids with the new ones." + [colors] + (reduce-kv (fn [res k v] + (if (:image v) + (update-in res [k :image :id] lookup-index) + res)) + colors + colors)) + +(defn embed-assets + [cfg data file-id] + (letfn [(walk-map-form [form state] + (cond + (uuid? (:fill-color-ref-file form)) + (do + (vswap! state conj [(:fill-color-ref-file form) :colors (:fill-color-ref-id form)]) + (assoc form :fill-color-ref-file file-id)) + + (uuid? (:stroke-color-ref-file form)) + (do + (vswap! state conj [(:stroke-color-ref-file form) :colors (:stroke-color-ref-id form)]) + (assoc form :stroke-color-ref-file file-id)) + + (uuid? (:typography-ref-file form)) + (do + (vswap! state conj [(:typography-ref-file form) :typographies (:typography-ref-id form)]) + (assoc form :typography-ref-file file-id)) + + (uuid? (:component-file form)) + (do + (vswap! state conj [(:component-file form) :components (:component-id form)]) + (assoc form :component-file file-id)) + + :else + form)) + + (process-group-of-assets [data [lib-id items]] + ;; NOTE: there is a possibility that shape refers to an + ;; non-existant file because the file was removed. In this + ;; case we just ignore the asset. + (if-let [lib (get-file cfg lib-id)] + (reduce (partial process-asset lib) data items) + data)) + + (process-asset [lib data [bucket asset-id]] + (let [asset (get-in lib [:data bucket asset-id]) + ;; Add a special case for colors that need to have + ;; correctly set the :file-id prop (pending of the + ;; refactor that will remove it). + asset (cond-> asset + (= bucket :colors) (assoc :file-id file-id))] + (update data bucket assoc asset-id asset)))] + + (let [assets (volatile! [])] + (walk/postwalk #(cond-> % (map? %) (walk-map-form assets)) data) + (->> (deref assets) + (filter #(as-> (first %) $ (and (uuid? $) (not= $ file-id)))) + (d/group-by first rest) + (reduce (partial process-group-of-assets) data))))) + + +(defn process-file + [{:keys [id] :as file}] + (-> file + (update :data (fn [fdata] + (-> fdata + (assoc :id id) + (dissoc :recent-colors) + (cond-> (> (:version fdata) cfd/version) + (assoc :version cfd/version)) + ;; FIXME: We're temporarily activating all + ;; migrations because a problem in the + ;; environments messed up with the version + ;; numbers When this problem is fixed delete + ;; the following line + (cond-> (> (:version fdata) 22) + (assoc :version 22))))) + (fmg/migrate-file) + (update :data (fn [fdata] + (-> fdata + (update :pages-index relink-shapes) + (update :components relink-shapes) + (update :media relink-media) + (update :colors relink-colors) + (d/without-nils)))))) + + +(defn- upsert-file! + [conn file] + (let [sql (str "INSERT INTO file (id, project_id, name, revn, is_shared, data, created_at, modified_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?) " + "ON CONFLICT (id) DO UPDATE SET data=?")] + (db/exec-one! conn [sql + (:id file) + (:project-id file) + (:name file) + (:revn file) + (:is-shared file) + (:data file) + (:created-at file) + (:modified-at file) + (:data file)]))) + +(defn persist-file! + "Applies all the final validations and perist the file." + [{:keys [::db/conn ::timestamp] :as cfg} {:keys [id] :as file}] + + (dm/assert! + "expected valid timestamp" + (dt/instant? timestamp)) + + (let [file (-> file + (assoc :created-at timestamp) + (assoc :modified-at timestamp) + (assoc :ignore-sync-until (dt/plus timestamp (dt/duration {:seconds 5}))) + (update :features + (fn [features] + (let [features (cfeat/check-supported-features! features)] + (-> (::features cfg #{}) + (set/difference cfeat/frontend-only-features) + (set/union features)))))) + + _ (when (contains? cf/flags :file-schema-validation) + (fval/validate-file-schema! file)) + + _ (when (contains? cf/flags :soft-file-schema-validation) + (let [result (ex/try! (fval/validate-file-schema! file))] + (when (ex/exception? result) + (l/error :hint "file schema validation error" :cause result)))) + + file (if (contains? (:features file) "fdata/objects-map") + (feat.fdata/enable-objects-map file) + file) + + file (if (contains? (:features file) "fdata/pointer-map") + (binding [pmap/*tracked* (pmap/create-tracked)] + (let [file (feat.fdata/enable-pointer-map file)] + (feat.fdata/persist-pointers! cfg id) + file)) + file) + + params (-> file + (update :features db/encode-pgarray conn "text") + (update :data blob/encode))] + + (if (::overwrite cfg) + (upsert-file! conn params) + (db/insert! conn :file params ::db/return-keys false)) + + file)) + +(defn apply-pending-migrations! + "Apply alredy registered pending migrations to files" + [cfg] + (doseq [[feature file-id] (-> *state* deref :pending-to-migrate)] + (case feature + "components/v2" + (feat.compv2/migrate-file! cfg file-id :validate? (::validate cfg true)) + + "fdata/shape-data-type" + nil + + (ex/raise :type :internal + :code :no-migration-defined + :hint (str/ffmt "no migation for feature '%' on file importation" feature) + :feature feature)))) diff --git a/backend/src/app/binfile/v1.clj b/backend/src/app/binfile/v1.clj new file mode 100644 index 0000000000..183c3ac697 --- /dev/null +++ b/backend/src/app/binfile/v1.clj @@ -0,0 +1,762 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.binfile.v1 + "A custom, perfromance and efficiency focused binfile format impl" + (:refer-clojure :exclude [assert]) + (:require + [app.binfile.common :as bfc] + [app.common.data :as d] + [app.common.data.macros :as dm] + [app.common.exceptions :as ex] + [app.common.features :as cfeat] + [app.common.fressian :as fres] + [app.common.logging :as l] + [app.common.spec :as us] + [app.common.types.file :as ctf] + [app.common.uuid :as uuid] + [app.config :as cf] + [app.db :as db] + [app.http.sse :as sse] + [app.loggers.audit :as-alias audit] + [app.loggers.webhooks :as-alias webhooks] + [app.media :as media] + [app.rpc :as-alias rpc] + [app.rpc.commands.teams :as teams] + [app.rpc.doc :as-alias doc] + [app.storage :as sto] + [app.storage.tmp :as tmp] + [app.tasks.file-gc] + [app.util.time :as dt] + [app.worker :as-alias wrk] + [clojure.java.io :as jio] + [clojure.set :as set] + [clojure.spec.alpha :as s] + [cuerdas.core :as str] + [datoteka.io :as io] + [promesa.util :as pu] + [yetti.adapter :as yt]) + (:import + com.github.luben.zstd.ZstdInputStream + com.github.luben.zstd.ZstdOutputStream + java.io.DataInputStream + java.io.DataOutputStream + java.io.InputStream + java.io.OutputStream)) + +(set! *warn-on-reflection* true) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; DEFAULTS +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Threshold in MiB when we pass from using +;; in-memory byte-array's to use temporal files. +(def temp-file-threshold + (* 1024 1024 2)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; LOW LEVEL STREAM IO API +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(def ^:const buffer-size (:xnio/buffer-size yt/defaults)) +(def ^:const penpot-magic-number 800099563638710213) +(def ^:const max-object-size (* 1024 1024 100)) ; Only allow 100MiB max file size. + +(def ^:dynamic *position* nil) + +(defn get-mark + [id] + (case id + :header 1 + :stream 2 + :uuid 3 + :label 4 + :obj 5 + (ex/raise :type :validation + :code :invalid-mark-id + :hint (format "invalid mark id %s" id)))) + +(defmacro assert + [expr hint] + `(when-not ~expr + (ex/raise :type :validation + :code :unexpected-condition + :hint ~hint))) + +(defmacro assert-mark + [v type] + `(let [expected# (get-mark ~type) + val# (long ~v)] + (when (not= val# expected#) + (ex/raise :type :validation + :code :unexpected-mark + :hint (format "received mark %s, expected %s" val# expected#))))) + +(defmacro assert-label + [expr label] + `(let [v# ~expr] + (when (not= v# ~label) + (ex/raise :type :assertion + :code :unexpected-label + :hint (format "received label %s, expected %s" v# ~label))))) + +;; --- PRIMITIVE IO + +(defn write-byte! + [^DataOutputStream output data] + (l/trace :fn "write-byte!" :data data :position @*position* ::l/sync? true) + (.writeByte output (byte data)) + (swap! *position* inc)) + +(defn read-byte! + [^DataInputStream input] + (let [v (.readByte input)] + (l/trace :fn "read-byte!" :val v :position @*position* ::l/sync? true) + (swap! *position* inc) + v)) + +(defn write-long! + [^DataOutputStream output data] + (l/trace :fn "write-long!" :data data :position @*position* ::l/sync? true) + (.writeLong output (long data)) + (swap! *position* + 8)) + + +(defn read-long! + [^DataInputStream input] + (let [v (.readLong input)] + (l/trace :fn "read-long!" :val v :position @*position* ::l/sync? true) + (swap! *position* + 8) + v)) + +(defn write-bytes! + [^DataOutputStream output ^bytes data] + (let [size (alength data)] + (l/trace :fn "write-bytes!" :size size :position @*position* ::l/sync? true) + (.write output data 0 size) + (swap! *position* + size))) + +(defn read-bytes! + [^InputStream input ^bytes buff] + (let [size (alength buff) + readed (.readNBytes input buff 0 size)] + (l/trace :fn "read-bytes!" :expected (alength buff) :readed readed :position @*position* ::l/sync? true) + (swap! *position* + readed) + readed)) + +;; --- COMPOSITE IO + +(defn write-uuid! + [^DataOutputStream output id] + (l/trace :fn "write-uuid!" :position @*position* :WRITTEN? (.size output) ::l/sync? true) + + (doto output + (write-byte! (get-mark :uuid)) + (write-long! (uuid/get-word-high id)) + (write-long! (uuid/get-word-low id)))) + +(defn read-uuid! + [^DataInputStream input] + (l/trace :fn "read-uuid!" :position @*position* ::l/sync? true) + (let [m (read-byte! input)] + (assert-mark m :uuid) + (let [a (read-long! input) + b (read-long! input)] + (uuid/custom a b)))) + +(defn write-obj! + [^DataOutputStream output data] + (l/trace :fn "write-obj!" :position @*position* ::l/sync? true) + (let [^bytes data (fres/encode data)] + (doto output + (write-byte! (get-mark :obj)) + (write-long! (alength data)) + (write-bytes! data)))) + +(defn read-obj! + [^DataInputStream input] + (l/trace :fn "read-obj!" :position @*position* ::l/sync? true) + (let [m (read-byte! input)] + (assert-mark m :obj) + (let [size (read-long! input)] + (assert (pos? size) "incorrect header size found on reading header") + (let [buff (byte-array size)] + (read-bytes! input buff) + (fres/decode buff))))) + +(defn write-label! + [^DataOutputStream output label] + (l/trace :fn "write-label!" :label label :position @*position* ::l/sync? true) + (doto output + (write-byte! (get-mark :label)) + (write-obj! label))) + +(defn read-label! + [^DataInputStream input] + (l/trace :fn "read-label!" :position @*position* ::l/sync? true) + (let [m (read-byte! input)] + (assert-mark m :label) + (read-obj! input))) + +(defn write-header! + [^OutputStream output version] + (l/trace :fn "write-header!" + :version version + :position @*position* + ::l/sync? true) + (let [vers (-> version name (subs 1) parse-long) + output (io/data-output-stream output)] + (doto output + (write-byte! (get-mark :header)) + (write-long! penpot-magic-number) + (write-long! vers)))) + +(defn read-header! + [^InputStream input] + (l/trace :fn "read-header!" :position @*position* ::l/sync? true) + (let [input (io/data-input-stream input) + mark (read-byte! input) + mnum (read-long! input) + vers (read-long! input)] + + (when (or (not= mark (get-mark :header)) + (not= mnum penpot-magic-number)) + (ex/raise :type :validation + :code :invalid-penpot-file + :hint "invalid penpot file")) + + (keyword (str "v" vers)))) + +(defn copy-stream! + [^OutputStream output ^InputStream input ^long size] + (let [written (io/copy! input output :size size)] + (l/trace :fn "copy-stream!" :position @*position* :size size :written written ::l/sync? true) + (swap! *position* + written) + written)) + +(defn write-stream! + [^DataOutputStream output stream size] + (l/trace :fn "write-stream!" :position @*position* ::l/sync? true :size size) + (doto output + (write-byte! (get-mark :stream)) + (write-long! size)) + + (copy-stream! output stream size)) + +(defn read-stream! + [^DataInputStream input] + (l/trace :fn "read-stream!" :position @*position* ::l/sync? true) + (let [m (read-byte! input) + s (read-long! input) + p (tmp/tempfile :prefix "penpot.binfile.")] + (assert-mark m :stream) + + (when (> s max-object-size) + (ex/raise :type :validation + :code :max-file-size-reached + :hint (str/ffmt "unable to import storage object with size % bytes" s))) + + (if (> s temp-file-threshold) + (with-open [^OutputStream output (io/output-stream p)] + (let [readed (io/copy! input output :offset 0 :size s)] + (l/trace :fn "read-stream*!" :expected s :readed readed :position @*position* ::l/sync? true) + (swap! *position* + readed) + [s p])) + [s (io/read-as-bytes input :size s)]))) + +(defmacro assert-read-label! + [input expected-label] + `(let [readed# (read-label! ~input) + expected# ~expected-label] + (when (not= readed# expected#) + (ex/raise :type :validation + :code :unexpected-label + :hint (format "unexpected label found: %s, expected: %s" readed# expected#))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; API +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; --- HELPERS + +(defn zstd-input-stream + ^InputStream + [input] + (ZstdInputStream. ^InputStream input)) + +(defn zstd-output-stream + ^OutputStream + [output & {:keys [level] :or {level 0}}] + (ZstdOutputStream. ^OutputStream output (int level))) + +(defn- get-files + [cfg ids] + (db/run! cfg (fn [{:keys [::db/conn]}] + (let [sql (str "SELECT id FROM file " + " WHERE id = ANY(?) ") + ids (db/create-array conn "uuid" ids)] + (->> (db/exec! conn [sql ids]) + (into [] (map :id)) + (not-empty)))))) + +;; --- EXPORT WRITER + +(defmulti write-export ::version) +(defmulti write-section ::section) + +(defn write-export! + [{:keys [::include-libraries ::embed-assets] :as cfg}] + (when (and include-libraries embed-assets) + (throw (IllegalArgumentException. + "the `include-libraries` and `embed-assets` are mutally excluding options"))) + + (write-export cfg)) + +(defmethod write-export :default + [{:keys [::output] :as options}] + (write-header! output :v1) + (pu/with-open [output (zstd-output-stream output :level 12) + output (io/data-output-stream output)] + (binding [bfc/*state* (volatile! {})] + (run! (fn [section] + (l/dbg :hint "write section" :section section ::l/sync? true) + (write-label! output section) + (let [options (-> options + (assoc ::output output) + (assoc ::section section))] + (binding [bfc/*options* options] + (write-section options)))) + + [:v1/metadata :v1/files :v1/rels :v1/sobjects])))) + +(defmethod write-section :v1/metadata + [{:keys [::output ::ids ::include-libraries] :as cfg}] + (if-let [fids (get-files cfg ids)] + (let [lids (when include-libraries + (bfc/get-libraries cfg ids)) + ids (into fids lids)] + (write-obj! output {:version cf/version :files ids}) + (vswap! bfc/*state* assoc :files ids)) + (ex/raise :type :not-found + :code :files-not-found + :hint "unable to retrieve files for export"))) + +(defmethod write-section :v1/files + [{:keys [::output ::embed-assets ::include-libraries] :as cfg}] + + ;; Initialize SIDS with empty vector + (vswap! bfc/*state* assoc :sids []) + + (doseq [file-id (-> bfc/*state* deref :files)] + (let [detach? (and (not embed-assets) (not include-libraries)) + thumbnails (->> (bfc/get-file-object-thumbnails cfg file-id) + (mapv #(dissoc % :file-id))) + + file (cond-> (bfc/get-file cfg file-id) + detach? + (-> (ctf/detach-external-references file-id) + (dissoc :libraries)) + + embed-assets + (update :data #(bfc/embed-assets cfg % file-id)) + + :always + (assoc :thumbnails thumbnails)) + + media (bfc/get-file-media cfg file)] + + (l/dbg :hint "write penpot file" + :id (str file-id) + :name (:name file) + :thumbnails (count thumbnails) + :features (:features file) + :media (count media) + ::l/sync? true) + + (doseq [item media] + (l/dbg :hint "write penpot file media object" :id (:id item) ::l/sync? true)) + + (doseq [item thumbnails] + (l/dbg :hint "write penpot file object thumbnail" :media-id (str (:media-id item)) ::l/sync? true)) + + (doto output + (write-obj! file) + (write-obj! media)) + + (vswap! bfc/*state* update :sids into bfc/xf-map-media-id media) + (vswap! bfc/*state* update :sids into bfc/xf-map-media-id thumbnails)))) + +(defmethod write-section :v1/rels + [{:keys [::output ::include-libraries] :as cfg}] + (let [ids (-> bfc/*state* deref :files set) + rels (when include-libraries + (bfc/get-files-rels cfg ids))] + (l/dbg :hint "found rels" :total (count rels) ::l/sync? true) + (write-obj! output rels))) + +(defmethod write-section :v1/sobjects + [{:keys [::sto/storage ::output]}] + (let [sids (-> bfc/*state* deref :sids) + storage (media/configure-assets-storage storage)] + + (l/dbg :hint "found sobjects" + :items (count sids) + ::l/sync? true) + + ;; Write all collected storage objects + (write-obj! output sids) + + (doseq [id sids] + (let [{:keys [size] :as obj} (sto/get-object storage id)] + (l/dbg :hint "write sobject" :id (str id) ::l/sync? true) + + (doto output + (write-uuid! id) + (write-obj! (meta obj))) + + (pu/with-open [stream (sto/get-object-data storage obj)] + (let [written (write-stream! output stream size)] + (when (not= written size) + (ex/raise :type :validation + :code :mismatch-readed-size + :hint (str/ffmt "found unexpected object size; size=% written=%" size written))))))))) + +;; --- EXPORT READER + +(defmulti read-import ::version) +(defmulti read-section ::section) + +(s/def ::profile-id ::us/uuid) +(s/def ::project-id ::us/uuid) +(s/def ::input io/input-stream?) +(s/def ::overwrite? (s/nilable ::us/boolean)) +(s/def ::ignore-index-errors? (s/nilable ::us/boolean)) + +;; FIXME: replace with schema +(s/def ::read-import-options + (s/keys :req [::db/pool ::sto/storage ::project-id ::profile-id ::input] + :opt [::overwrite? ::ignore-index-errors?])) + +(defn read-import! + "Do the importation of the specified resource in penpot custom binary + format. There are some options for customize the importation + behavior: + + `::bfc/overwrite`: if true, instead of creating new files and remapping id references, + it reuses all ids and updates existing objects; defaults to `false`." + [{:keys [::input ::bfc/timestamp] :or {timestamp (dt/now)} :as options}] + + (dm/assert! + "expected input stream" + (io/input-stream? input)) + + (dm/assert! + "expected valid instant" + (dt/instant? timestamp)) + + (let [version (read-header! input)] + (read-import (assoc options ::version version ::bfc/timestamp timestamp)))) + +(defn- read-import-v1 + [{:keys [::db/conn ::project-id ::profile-id ::input] :as cfg}] + (db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"]) + (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]) + + (pu/with-open [input (zstd-input-stream input) + input (io/data-input-stream input)] + (binding [bfc/*state* (volatile! {:media [] :index {}})] + (let [team (teams/get-team conn + :profile-id profile-id + :project-id project-id) + + features (cfeat/get-team-enabled-features cf/flags team)] + + (sse/tap {:type :import-progress + :section :read-import}) + + ;; Process all sections + (run! (fn [section] + (l/dbg :hint "reading section" :section section ::l/sync? true) + (assert-read-label! input section) + (let [options (-> cfg + (assoc ::bfc/features features) + (assoc ::section section) + (assoc ::input input))] + (binding [bfc/*options* options] + (sse/tap {:type :import-progress + :section section}) + (read-section options)))) + [:v1/metadata :v1/files :v1/rels :v1/sobjects]) + + (bfc/apply-pending-migrations! cfg) + + ;; Knowing that the ids of the created files are in index, + ;; just lookup them and return it as a set + (let [files (-> bfc/*state* deref :files)] + (into #{} (keep #(get-in @bfc/*state* [:index %])) files)))))) + +(defmethod read-import :v1 + [options] + (db/tx-run! options read-import-v1)) + +(defmethod read-section :v1/metadata + [{:keys [::input]}] + (let [{:keys [version files]} (read-obj! input)] + (l/dbg :hint "metadata readed" + :version (:full version) + :files (mapv str files) + ::l/sync? true) + (vswap! bfc/*state* update :index bfc/update-index files) + (vswap! bfc/*state* assoc :version version :files files))) + +(defn- remap-thumbnails + [thumbnails file-id] + (mapv (fn [thumbnail] + (-> thumbnail + (assoc :file-id file-id) + (update :object-id #(str/replace-first % #"^(.*?)/" (str file-id "/"))))) + thumbnails)) + +(defmethod read-section :v1/files + [{:keys [::db/conn ::input ::project-id ::bfc/overwrite ::name] :as system}] + + (doseq [[idx expected-file-id] (d/enumerate (-> bfc/*state* deref :files))] + (let [file (read-obj! input) + media (read-obj! input) + + file-id (:id file) + file-id' (bfc/lookup-index file-id) + + thumbnails (:thumbnails file)] + + (when (not= file-id expected-file-id) + (ex/raise :type :validation + :code :inconsistent-penpot-file + :found-id file-id + :expected-id expected-file-id + :hint "the penpot file seems corrupt, found unexpected uuid (file-id)")) + + (l/dbg :hint "processing file" + :id (str file-id) + :features (:features file) + :version (-> file :data :version) + :media (count media) + :thumbnails (count thumbnails) + ::l/sync? true) + + (when (seq thumbnails) + (let [thumbnails (remap-thumbnails thumbnails file-id')] + (l/dbg :hint "updated index with thumbnails" :total (count thumbnails) ::l/sync? true) + (vswap! bfc/*state* update :thumbnails bfc/into-vec thumbnails))) + + (when (seq media) + ;; Update index with media + (l/dbg :hint "update index with media" :total (count media) ::l/sync? true) + (vswap! bfc/*state* update :index bfc/update-index (map :id media)) + + ;; Store file media for later insertion + (l/dbg :hint "update media references" ::l/sync? true) + (vswap! bfc/*state* update :media into (map #(update % :id bfc/lookup-index)) media)) + + (let [file (-> file + (assoc :id file-id') + (cond-> (and (= idx 0) (some? name)) + (assoc :name name)) + (assoc :project-id project-id) + (dissoc :thumbnails) + (bfc/process-file))] + + ;; All features that are enabled and requires explicit migration are + ;; added to the state for a posterior migration step. + (doseq [feature (-> (::bfc/features system) + (set/difference cfeat/no-migration-features) + (set/difference (:features file)))] + (vswap! bfc/*state* update :pending-to-migrate (fnil conj []) [feature file-id'])) + + (l/dbg :hint "create file" :id (str file-id') ::l/sync? true) + (bfc/persist-file! system file) + + (when overwrite + (db/delete! conn :file-thumbnail {:file-id file-id'})) + + file-id')))) + +(defmethod read-section :v1/rels + [{:keys [::db/conn ::input ::bfc/timestamp]}] + (let [rels (read-obj! input) + ids (into #{} (-> bfc/*state* deref :files))] + ;; Insert all file relations + (doseq [{:keys [library-file-id] :as rel} rels] + (let [rel (-> rel + (assoc :synced-at timestamp) + (update :file-id bfc/lookup-index) + (update :library-file-id bfc/lookup-index))] + + (if (contains? ids library-file-id) + (do + (l/dbg :hint "create file library link" + :file-id (:file-id rel) + :lib-id (:library-file-id rel) + ::l/sync? true) + (db/insert! conn :file-library-rel rel)) + + (l/warn :hint "ignoring file library link" + :file-id (:file-id rel) + :lib-id (:library-file-id rel) + ::l/sync? true)))))) + +(defmethod read-section :v1/sobjects + [{:keys [::sto/storage ::db/conn ::input ::bfc/overwrite ::bfc/timestamp]}] + (let [storage (media/configure-assets-storage storage) + ids (read-obj! input) + thumb? (into #{} (map :media-id) (:thumbnails @bfc/*state*))] + + (doseq [expected-storage-id ids] + (let [id (read-uuid! input) + mdata (read-obj! input)] + + (when (not= id expected-storage-id) + (ex/raise :type :validation + :code :inconsistent-penpot-file + :hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)")) + + (l/dbg :hint "readed storage object" :id (str id) ::l/sync? true) + + (let [[size resource] (read-stream! input) + hash (sto/calculate-hash resource) + content (-> (sto/content resource size) + (sto/wrap-with-hash hash)) + + params (-> mdata + (assoc ::sto/content content) + (assoc ::sto/deduplicate? true) + (assoc ::sto/touched-at timestamp)) + + params (if (thumb? id) + (assoc params :bucket "file-object-thumbnail") + (assoc params :bucket "file-media-object")) + + sobject (sto/put-object! storage params)] + + (l/dbg :hint "persisted storage object" + :old-id (str id) + :new-id (str (:id sobject)) + :is-thumbnail (boolean (thumb? id)) + ::l/sync? true) + + (vswap! bfc/*state* update :index assoc id (:id sobject))))) + + (doseq [item (:media @bfc/*state*)] + (l/dbg :hint "inserting file media object" + :id (str (:id item)) + :file-id (str (:file-id item)) + ::l/sync? true) + + (let [file-id (bfc/lookup-index (:file-id item))] + (if (= file-id (:file-id item)) + (l/warn :hint "ignoring file media object" :file-id (str file-id) ::l/sync? true) + (db/insert! conn :file-media-object + (-> item + (assoc :file-id file-id) + (d/update-when :media-id bfc/lookup-index) + (d/update-when :thumbnail-id bfc/lookup-index)) + {::db/on-conflict-do-nothing? overwrite})))) + + (doseq [item (:thumbnails @bfc/*state*)] + (let [item (update item :media-id bfc/lookup-index)] + (l/dbg :hint "inserting file object thumbnail" + :file-id (str (:file-id item)) + :media-id (str (:media-id item)) + :object-id (:object-id item) + ::l/sync? true) + (db/insert! conn :file-tagged-object-thumbnail item + {::db/on-conflict-do-nothing? overwrite}))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; HIGH LEVEL API +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn export-files! + "Do the exportation of a specified file in custom penpot binary + format. There are some options available for customize the output: + + `::include-libraries`: additionally to the specified file, all the + linked libraries also will be included (including transitive + dependencies). + + `::embed-assets`: instead of including the libraries, embed in the + same file library all assets used from external libraries." + + [{:keys [::ids] :as cfg} output] + + (dm/assert! + "expected a set of uuid's for `::ids` parameter" + (and (set? ids) + (every? uuid? ids))) + + (dm/assert! + "expected instance of jio/IOFactory for `input`" + (satisfies? jio/IOFactory output)) + + (let [id (uuid/next) + tp (dt/tpoint) + ab (volatile! false) + cs (volatile! nil)] + (try + (l/info :hint "start exportation" :export-id (str id)) + (pu/with-open [output (io/output-stream output)] + (binding [*position* (atom 0)] + (write-export! (assoc cfg ::output output)))) + + (catch java.io.IOException _cause + ;; Do nothing, EOF means client closes connection abruptly + (vreset! ab true) + nil) + + (catch Throwable cause + (vreset! cs cause) + (vreset! ab true) + (throw cause)) + + (finally + (l/info :hint "exportation finished" :export-id (str id) + :elapsed (str (inst-ms (tp)) "ms") + :aborted @ab + :cause @cs))))) + +(defn import-files! + [cfg input] + + (dm/assert! + "expected valid profile-id and project-id on `cfg`" + (and (uuid? (::profile-id cfg)) + (uuid? (::project-id cfg)))) + + (dm/assert! + "expected instance of jio/IOFactory for `input`" + (satisfies? jio/IOFactory input)) + + (let [id (uuid/next) + tp (dt/tpoint) + cs (volatile! nil)] + + (l/info :hint "import: started" :id (str id)) + (try + (binding [*position* (atom 0)] + (pu/with-open [input (io/input-stream input)] + (read-import! (assoc cfg ::input input)))) + + (catch Throwable cause + (vreset! cs cause) + (throw cause)) + + (finally + (l/info :hint "import: terminated" + :id (str id) + :elapsed (dt/format-duration (tp)) + :error? (some? @cs)))))) + diff --git a/backend/src/app/binfile/v2.clj b/backend/src/app/binfile/v2.clj index 2b3639b397..33a92a03b9 100644 --- a/backend/src/app/binfile/v2.clj +++ b/backend/src/app/binfile/v2.clj @@ -9,31 +9,24 @@ of entire team (or multiple teams) at once." (:refer-clojure :exclude [read]) (:require + [app.binfile.common :as bfc] [app.common.data :as d] - [app.common.exceptions :as ex] [app.common.features :as cfeat] - [app.common.files.defaults :as cfd] - [app.common.files.migrations :as fmg] - [app.common.files.validate :as fval] [app.common.logging :as l] [app.common.transit :as t] [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] [app.db.sql :as sql] - [app.features.fdata :as feat.fdata] [app.http.sse :as sse] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] [app.media :as media] [app.storage :as sto] [app.storage.tmp :as tmp] - [app.util.blob :as blob] - [app.util.pointer-map :as pmap] [app.util.time :as dt] [app.worker :as-alias wrk] [clojure.set :as set] - [clojure.walk :as walk] [cuerdas.core :as str] [datoteka.io :as io] [promesa.util :as pu]) @@ -42,34 +35,10 @@ (set! *warn-on-reflection* true) -(def ^:dynamic *state* nil) -(def ^:dynamic *options* nil) - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; LOW LEVEL API ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(defn- lookup-index - [id] - (when-let [val (get-in @*state* [:index id])] - (l/trc :fn "lookup-index" :id (some-> id str) :result (some-> val str) ::l/sync? true) - (or val id))) - -(defn- index-object - [index obj & attrs] - (reduce (fn [index attr-fn] - (let [old-id (attr-fn obj) - new-id (uuid/next)] - (assoc index old-id new-id))) - index - attrs)) - -(defn- update-index - ([index coll] - (update-index index coll identity)) - ([index coll attr] - (reduce #(index-object %1 %2 attr) index coll))) - (defn- create-database ([cfg] (let [path (tmp/tempfile :prefix "penpot.binfile." :suffix ".sqlite")] @@ -92,12 +61,6 @@ "CREATE INDEX kvdata__tag_key__idx ON kvdata (tag, key)") -(defn- decode-row - [{:keys [data features] :as row}] - (cond-> row - features (assoc :features (db/decode-pgarray features #{})) - data (assoc :data (blob/decode data)))) - (defn- setup-schema! [{:keys [::db]}] (db/exec-one! db [sql:create-kvdata-table]) @@ -147,156 +110,62 @@ ;; IMPORT/EXPORT IMPL ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(def ^:private xf-map-id - (map :id)) - -(def ^:private xf-map-media-id - (comp - (mapcat (juxt :media-id - :thumbnail-id - :woff1-file-id - :woff2-file-id - :ttf-file-id - :otf-file-id)) - (filter uuid?))) - -;; NOTE: Will be used in future, commented for satisfy linter -;; (def ^:private sql:get-libraries -;; "WITH RECURSIVE libs AS ( -;; SELECT fl.id -;; FROM file AS fl -;; JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id) -;; WHERE flr.file_id = ANY(?) -;; UNION -;; SELECT fl.id -;; FROM file AS fl -;; JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id) -;; JOIN libs AS l ON (flr.file_id = l.id) -;; ) -;; SELECT DISTINCT l.id -;; FROM libs AS l") -;; -;; (defn- get-libraries -;; "Get all libraries ids related to provided file ids" -;; [{:keys [::db/conn]} ids] -;; (let [ids' (db/create-array conn "uuid" ids)] -;; (->> (db/exec! conn [sql:get-libraries ids]) -;; (into #{} xf-map-id)))) -;; -;; (def ^:private sql:get-project-files -;; "SELECT f.id FROM file AS f -;; WHERE f.project_id = ?") - -;; (defn- get-project-files -;; "Get a set of file ids for the project" -;; [{:keys [::db/conn]} project-id] -;; (->> (db/exec! conn [sql:get-project-files project-id]) -;; (into #{} xf-map-id))) - -(def ^:private sql:get-team-files - "SELECT f.id FROM file AS f - JOIN project AS p ON (p.id = f.project_id) - WHERE p.team_id = ?") - -(defn- get-team-files - "Get a set of file ids for the specified team-id" - [{:keys [::db/conn]} team-id] - (->> (db/exec! conn [sql:get-team-files team-id]) - (into #{} xf-map-id))) - -(def ^:private sql:get-team-projects - "SELECT p.id FROM project AS p - WHERE p.team_id = ?") - -(defn- get-team-projects - "Get a set of project ids for the team" - [{:keys [::db/conn]} team-id] - (->> (db/exec! conn [sql:get-team-projects team-id]) - (into #{} xf-map-id))) - (declare ^:private write-project!) (declare ^:private write-file!) (defn- write-team! - [{:keys [::db/conn] :as cfg} team-id] + [cfg team-id] (sse/tap {:type :export-progress :section :write-team :id team-id}) - (let [team (db/get conn :team {:id team-id} - ::db/remove-deleted false - ::db/check-deleted false) - team (decode-row team) - fonts (db/query conn :team-font-variant - {:team-id team-id - :deleted-at nil} - {::sql/for-share true})] + (let [team (bfc/get-team cfg team-id) + fonts (bfc/get-fonts cfg team-id)] (l/trc :hint "write" :obj "team" :id (str team-id) :fonts (count fonts)) - (vswap! *state* update :teams conj team-id) - (vswap! *state* update :storage-objects into xf-map-media-id fonts) + (vswap! bfc/*state* update :teams conj team-id) + (vswap! bfc/*state* bfc/collect-storage-objects fonts) (write! cfg :team team-id team) (doseq [{:keys [id] :as font} fonts] - (vswap! *state* update :team-font-variants conj id) + (vswap! bfc/*state* update :team-font-variants conj id) (write! cfg :team-font-variant id font)))) (defn- write-project! - [{:keys [::db/conn] :as cfg} project-id] + [cfg project-id] (sse/tap {:type :export-progress :section :write-project :id project-id}) - (let [project (db/get conn :project {:id project-id} - ::db/remove-deleted false - ::db/check-deleted false)] - + (let [project (bfc/get-project cfg project-id)] (l/trc :hint "write" :obj "project" :id (str project-id)) (write! cfg :project (str project-id) project) - - (vswap! *state* update :projects conj project-id))) + (vswap! bfc/*state* update :projects conj project-id))) (defn- write-file! - [{:keys [::db/conn] :as cfg} file-id] + [cfg file-id] (sse/tap {:type :export-progress :section :write-file :id file-id}) - (let [file (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)] - (-> (db/get conn :file {:id file-id} - ::sql/for-share true - ::db/remove-deleted false - ::db/check-deleted false) - (decode-row) - (update :data feat.fdata/process-pointers deref) - (update :data feat.fdata/process-objects (partial into {})))) + (let [file (bfc/get-file cfg file-id) + thumbs (bfc/get-file-object-thumbnails cfg file-id) + media (bfc/get-file-media cfg file) + rels (bfc/get-files-rels cfg #{file-id})] - thumbs (db/query conn :file-tagged-object-thumbnail - {:file-id file-id - :deleted-at nil} - {::sql/for-share true}) - - media (db/query conn :file-media-object - {:file-id file-id - :deleted-at nil} - {::sql/for-share true}) - - rels (db/query conn :file-library-rel - {:file-id file-id})] - - (vswap! *state* (fn [state] - (-> state - (update :files conj file-id) - (update :file-media-objects into (map :id) media) - (update :storage-objects into xf-map-media-id thumbs) - (update :storage-objects into xf-map-media-id media)))) + (vswap! bfc/*state* (fn [state] + (-> state + (update :files conj file-id) + (update :file-media-objects into bfc/xf-map-id media) + (bfc/collect-storage-objects thumbs) + (bfc/collect-storage-objects media)))) (write! cfg :file file-id file) (write! cfg :file-rels file-id rels) @@ -304,13 +173,8 @@ (run! (partial write! cfg :file-media-object file-id) media) (run! (partial write! cfg :file-object-thumbnail file-id) thumbs) - (when-let [thumb (db/get* conn :file-thumbnail - {:file-id file-id - :revn (:revn file) - :data nil} - {::sql/for-share true - ::sql/columns [:media-id :file-id :revn]})] - (vswap! *state* update :storage-objects into xf-map-media-id [thumb]) + (when-let [thumb (bfc/get-file-thumbnail cfg file)] + (vswap! bfc/*state* bfc/collect-storage-objects [thumb]) (write! cfg :file-thumbnail file-id thumb)) (l/trc :hint "write" :obj "file" @@ -328,7 +192,7 @@ (write! cfg :storage-object id (meta sobj) data))) (defn- read-storage-object! - [{:keys [::sto/storage ::timestamp] :as cfg} id] + [{:keys [::sto/storage ::bfc/timestamp] :as cfg} id] (let [mdata (read-obj cfg :storage-object id) data (read-blob cfg :storage-object id) hash (sto/calculate-hash data) @@ -343,7 +207,7 @@ sobject (sto/put-object! storage params)] - (vswap! *state* update :index assoc id (:id sobject)) + (vswap! bfc/*state* update :index assoc id (:id sobject)) (l/trc :hint "read" :obj "storage-object" :id (str id) @@ -351,7 +215,7 @@ :size (:size sobject)))) (defn read-team! - [{:keys [::db/conn ::timestamp] :as cfg} team-id] + [{:keys [::db/conn ::bfc/timestamp] :as cfg} team-id] (l/trc :hint "read" :obj "team" :id (str team-id)) (sse/tap {:type :import-progress @@ -360,8 +224,8 @@ (let [team (read-obj cfg :team team-id) team (-> team - (update :id lookup-index) - (update :photo-id lookup-index) + (update :id bfc/lookup-index) + (update :photo-id bfc/lookup-index) (assoc :created-at timestamp) (assoc :modified-at timestamp))] @@ -372,12 +236,12 @@ (doseq [font (->> (read-seq cfg :team-font-variant) (filter #(= team-id (:team-id %))))] (let [font (-> font - (update :id lookup-index) - (update :team-id lookup-index) - (update :woff1-file-id lookup-index) - (update :woff2-file-id lookup-index) - (update :ttf-file-id lookup-index) - (update :otf-file-id lookup-index) + (update :id bfc/lookup-index) + (update :team-id bfc/lookup-index) + (update :woff1-file-id bfc/lookup-index) + (update :woff2-file-id bfc/lookup-index) + (update :ttf-file-id bfc/lookup-index) + (update :otf-file-id bfc/lookup-index) (assoc :created-at timestamp) (assoc :modified-at timestamp))] (db/insert! conn :team-font-variant font @@ -386,7 +250,7 @@ team)) (defn read-project! - [{:keys [::db/conn ::timestamp] :as cfg} project-id] + [{:keys [::db/conn ::bfc/timestamp] :as cfg} project-id] (l/trc :hint "read" :obj "project" :id (str project-id)) (sse/tap {:type :import-progress @@ -395,175 +259,40 @@ (let [project (read-obj cfg :project project-id) project (-> project - (update :id lookup-index) - (update :team-id lookup-index) + (update :id bfc/lookup-index) + (update :team-id bfc/lookup-index) (assoc :created-at timestamp) (assoc :modified-at timestamp))] (db/insert! conn :project project ::db/return-keys false))) -(defn- relink-shapes - "A function responsible to analyze all file data and - replace the old :component-file reference with the new - ones, using the provided file-index." - [data] - (letfn [(process-map-form [form] - (cond-> form - ;; Relink image shapes - (and (map? (:metadata form)) - (= :image (:type form))) - (update-in [:metadata :id] lookup-index) - - ;; Relink paths with fill image - (map? (:fill-image form)) - (update-in [:fill-image :id] lookup-index) - - ;; This covers old shapes and the new :fills. - (uuid? (:fill-color-ref-file form)) - (update :fill-color-ref-file lookup-index) - - ;; This covers the old shapes and the new :strokes - (uuid? (:storage-color-ref-file form)) - (update :stroke-color-ref-file lookup-index) - - ;; This covers all text shapes that have typography referenced - (uuid? (:typography-ref-file form)) - (update :typography-ref-file lookup-index) - - ;; This covers the component instance links - (uuid? (:component-file form)) - (update :component-file lookup-index) - - ;; This covers the shadows and grids (they have directly - ;; the :file-id prop) - (uuid? (:file-id form)) - (update :file-id lookup-index)))] - - (walk/postwalk (fn [form] - (if (map? form) - (try - (process-map-form form) - (catch Throwable cause - (l/warn :hint "failed form" :form (pr-str form) ::l/sync? true) - (throw cause))) - form)) - data))) - -(defn- relink-media - "A function responsible of process the :media attr of file data and - remap the old ids with the new ones." - [media] - (reduce-kv (fn [res k v] - (let [id (lookup-index k)] - (if (uuid? id) - (-> res - (assoc id (assoc v :id id)) - (dissoc k)) - res))) - media - media)) - -(defn- relink-colors - "A function responsible of process the :colors attr of file data and - remap the old ids with the new ones." - [colors] - (reduce-kv (fn [res k v] - (if (:image v) - (update-in res [k :image :id] lookup-index) - res)) - colors - colors)) - -(defn- process-file - [{:keys [id] :as file}] - (-> file - (update :data (fn [fdata] - (-> fdata - (assoc :id id) - (dissoc :recent-colors) - (cond-> (> (:version fdata) cfd/version) - (assoc :version cfd/version)) - ;; FIXME: We're temporarily activating all - ;; migrations because a problem in the - ;; environments messed up with the version - ;; numbers When this problem is fixed delete - ;; the following line - (cond-> (> (:version fdata) 22) - (assoc :version 22))))) - (fmg/migrate-file) - (update :data (fn [fdata] - (-> fdata - (update :pages-index relink-shapes) - (update :components relink-shapes) - (update :media relink-media) - (update :colors relink-colors) - (d/without-nils)))))) - (defn read-file! - [{:keys [::db/conn ::timestamp] :as cfg} file-id] + [{:keys [::db/conn ::bfc/timestamp] :as cfg} file-id] (l/trc :hint "read" :obj "file" :id (str file-id)) (sse/tap {:type :import-progress :section :read-file :id file-id}) - (let [file (read-obj cfg :file file-id) + (let [file (-> (read-obj cfg :file file-id) + (update :id bfc/lookup-index) + (update :project-id bfc/lookup-index) + (bfc/process-file))] - file (-> file - (update :id lookup-index) - (process-file)) + ;; All features that are enabled and requires explicit migration are + ;; added to the state for a posterior migration step. + (doseq [feature (-> (::bfc/features cfg) + (set/difference cfeat/no-migration-features) + (set/difference (:features file)))] + (vswap! bfc/*state* update :pending-to-migrate (fnil conj []) [feature (:id file)])) - ;; All features that are enabled and requires explicit migration are - ;; added to the state for a posterior migration step. - _ (doseq [feature (-> (::features cfg) - (set/difference cfeat/no-migration-features) - (set/difference (:features file)))] - (vswap! *state* update :pending-to-migrate (fnil conj []) [feature (:id file)])) - - - file (-> file - (update :project-id lookup-index)) - - file (-> file - (assoc :created-at timestamp) - (assoc :modified-at timestamp) - (update :features - (fn [features] - (let [features (cfeat/check-supported-features! features)] - (-> (::features cfg) - (set/difference cfeat/frontend-only-features) - (set/union features)))))) - - _ (when (contains? cf/flags :file-schema-validation) - (fval/validate-file-schema! file)) - - _ (when (contains? cf/flags :soft-file-schema-validation) - (let [result (ex/try! (fval/validate-file-schema! file))] - (when (ex/exception? result) - (l/error :hint "file schema validation error" :cause result)))) - - file (if (contains? (:features file) "fdata/objects-map") - (feat.fdata/enable-objects-map file) - file) - - file (if (contains? (:features file) "fdata/pointer-map") - (binding [pmap/*tracked* (pmap/create-tracked)] - (let [file (feat.fdata/enable-pointer-map file)] - (feat.fdata/persist-pointers! cfg (:id file)) - file)) - file)] - - (db/insert! conn :file - (-> file - (update :features db/encode-pgarray conn "text") - (update :data blob/encode)) - {::db/return-keys false})) + (bfc/persist-file! cfg file)) (doseq [thumbnail (read-seq cfg :file-object-thumbnail file-id)] (let [thumbnail (-> thumbnail - (update :file-id lookup-index) - (update :media-id lookup-index)) + (update :file-id bfc/lookup-index) + (update :media-id bfc/lookup-index)) file-id (:file-id thumbnail) thumbnail (update thumbnail :object-id @@ -574,20 +303,21 @@ (doseq [rel (read-obj cfg :file-rels file-id)] (let [rel (-> rel - (update :file-id lookup-index) - (update :library-file-id lookup-index) + (update :file-id bfc/lookup-index) + (update :library-file-id bfc/lookup-index) (assoc :synced-at timestamp))] (db/insert! conn :file-library-rel rel ::db/return-keys false))) (doseq [media (read-seq cfg :file-media-object file-id)] (let [media (-> media - (update :id lookup-index) - (update :file-id lookup-index) - (update :media-id lookup-index) - (update :thumbnail-id lookup-index))] + (update :id bfc/lookup-index) + (update :file-id bfc/lookup-index) + (update :media-id bfc/lookup-index) + (update :thumbnail-id bfc/lookup-index))] (db/insert! conn :file-media-object media - ::db/return-keys false)))) + ::db/return-keys false + ::sql/on-conflict-do-nothing true)))) (def ^:private empty-summary {:teams #{} @@ -617,20 +347,20 @@ (try (db/tx-run! cfg (fn [cfg] (setup-schema! cfg) - (binding [*state* (volatile! empty-summary)] + (binding [bfc/*state* (volatile! empty-summary)] (write-team! cfg team-id) (run! (partial write-project! cfg) - (get-team-projects cfg team-id)) + (bfc/get-team-projects cfg team-id)) (run! (partial write-file! cfg) - (get-team-files cfg team-id)) + (bfc/get-team-files cfg team-id)) (run! (partial write-storage-object! cfg) - (-> *state* deref :storage-objects)) + (-> bfc/*state* deref :storage-objects)) (write! cfg :manifest "team-id" team-id) - (write! cfg :manifest "objects" (deref *state*)) + (write! cfg :manifest "objects" (deref bfc/*state*)) (::path cfg)))) (finally @@ -642,19 +372,6 @@ :id (str id) :elapsed (dt/format-duration elapsed))))))) -;; NOTE: will be used in future, commented for satisfy linter -;; (defn- run-pending-migrations! -;; [cfg] -;; ;; Run all pending migrations -;; (doseq [[feature file-id] (-> *state* deref :pending-to-migrate)] -;; (case feature -;; "components/v2" -;; (feat.compv2/migrate-file! cfg file-id :validate? (::validate cfg true)) -;; (ex/raise :type :internal -;; :code :no-migration-defined -;; :hint (str/ffmt "no migation for feature '%' on file importation" feature) -;; :feature feature)))) - (defn import-team! [cfg path] (let [id (uuid/next) @@ -662,7 +379,7 @@ cfg (-> (create-database cfg path) (update ::sto/storage media/configure-assets-storage) - (assoc ::timestamp (dt/now)))] + (assoc ::bfc/timestamp (dt/now)))] (l/inf :hint "start" :operation "import" @@ -674,7 +391,7 @@ (db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"]) (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]) - (binding [*state* (volatile! {:index {}})] + (binding [bfc/*state* (volatile! {:index {}})] (let [objects (read-obj cfg :manifest "objects")] ;; We first process all storage objects, they have @@ -683,19 +400,19 @@ (run! (partial read-storage-object! cfg) (:storage-objects objects)) ;; Populate index with all the incoming objects - (vswap! *state* update :index + (vswap! bfc/*state* update :index (fn [index] (-> index - (update-index (:teams objects)) - (update-index (:projects objects)) - (update-index (:files objects)) - (update-index (:file-media-objects objects)) - (update-index (:team-font-variants objects))))) + (bfc/update-index (:teams objects)) + (bfc/update-index (:projects objects)) + (bfc/update-index (:files objects)) + (bfc/update-index (:file-media-objects objects)) + (bfc/update-index (:team-font-variants objects))))) (let [team-id (read-obj cfg :manifest "team-id") team (read-team! cfg team-id) features (cfeat/get-team-enabled-features cf/flags team) - cfg (assoc cfg ::features features)] + cfg (assoc cfg ::bfc/features features)] (run! (partial read-project! cfg) (:projects objects)) (run! (partial read-file! cfg) (:files objects)) diff --git a/backend/src/app/db/sql.clj b/backend/src/app/db/sql.clj index 37814733de..81ce636c5a 100644 --- a/backend/src/app/db/sql.clj +++ b/backend/src/app/db/sql.clj @@ -30,6 +30,9 @@ (let [opts (merge default-opts opts) opts (cond-> opts (::db/on-conflict-do-nothing? opts) + (assoc :suffix "ON CONFLICT DO NOTHING") + + (::on-conflict-do-nothing opts) (assoc :suffix "ON CONFLICT DO NOTHING"))] (sql/for-insert table key-map opts)))) @@ -46,7 +49,7 @@ opts (cond-> opts (::columns opts) (assoc :columns (::columns opts)) (::for-update opts) (assoc :suffix "FOR UPDATE") - (::for-share opts) (assoc :suffix "FOR KEY SHARE"))] + (::for-share opts) (assoc :suffix "FOR SHARE"))] (sql/for-query table where-params opts)))) (defn update diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index 1630289046..3f5d6a7b79 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -7,6 +7,7 @@ (ns app.http.debug (:refer-clojure :exclude [error-handler]) (:require + [app.binfile.v1 :as bf.v1] [app.common.data :as d] [app.common.exceptions :as ex] [app.common.logging :as l] @@ -17,11 +18,11 @@ [app.http.session :as session] [app.main :as-alias main] [app.rpc.commands.auth :as auth] - [app.rpc.commands.binfile :as binf] [app.rpc.commands.files-create :refer [create-file]] [app.rpc.commands.profile :as profile] [app.srepl.helpers :as srepl] [app.storage :as-alias sto] + [app.storage.tmp :as tmp] [app.util.blob :as blob] [app.util.template :as tmpl] [app.util.time :as dt] @@ -268,9 +269,10 @@ (defn export-handler [{:keys [::db/pool] :as cfg} {:keys [params ::session/profile-id] :as request}] - (let [file-ids (->> (:file-ids params) - (remove empty?) - (mapv parse-uuid)) + (let [file-ids (into #{} + (comp (remove empty?) + (map parse-uuid)) + (:file-ids params)) libs? (contains? params :includelibs) clone? (contains? params :clone) embed? (contains? params :embedassets)] @@ -279,22 +281,22 @@ (ex/raise :type :validation :code :missing-arguments)) - (let [path (-> cfg - (assoc ::binf/file-ids file-ids) - (assoc ::binf/embed-assets? embed?) - (assoc ::binf/include-libraries? libs?) - (binf/export-to-tmpfile!))] + (let [path (tmp/tempfile :prefix "penpot.export.")] + (with-open [output (io/output-stream path)] + (-> cfg + (assoc ::bf.v1/ids file-ids) + (assoc ::bf.v1/embed-assets embed?) + (assoc ::bf.v1/include-libraries libs?) + (bf.v1/export-files! output))) + (if clone? (let [profile (profile/get-profile pool profile-id) - project-id (:default-project-id profile)] - (binf/import! - (assoc cfg - ::binf/input path - ::binf/overwrite? false - ::binf/ignore-index-errors? true - ::binf/profile-id profile-id - ::binf/project-id project-id)) - + project-id (:default-project-id profile) + cfg (assoc cfg + ::bf.v1/overwrite false + ::bf.v1/profile-id profile-id + ::bf.v1/project-id project-id)] + (bf.v1/import-files! cfg path) {::rres/status 200 ::rres/headers {"content-type" "text/plain"} ::rres/body "OK CLONED"}) @@ -305,7 +307,6 @@ "content-disposition" (str "attachmen; filename=" (first file-ids) ".penpot")}})))) - (defn import-handler [{:keys [::db/pool] :as cfg} {:keys [params ::session/profile-id] :as request}] (when-not (contains? params :file) @@ -316,26 +317,23 @@ (let [profile (profile/get-profile pool profile-id) project-id (:default-project-id profile) overwrite? (contains? params :overwrite) - migrate? (contains? params :migrate) - ignore-index-errors? (contains? params :ignore-index-errors)] + migrate? (contains? params :migrate)] (when-not project-id (ex/raise :type :validation :code :missing-project :hint "project not found")) - (binf/import! - (assoc cfg - ::binf/input (-> params :file :path) - ::binf/overwrite? overwrite? - ::binf/migrate? migrate? - ::binf/ignore-index-errors? ignore-index-errors? - ::binf/profile-id profile-id - ::binf/project-id project-id)) - - {::rres/status 200 - ::rres/headers {"content-type" "text/plain"} - ::rres/body "OK"})) + (let [path (-> params :file :path) + cfg (assoc cfg + ::bf.v1/overwrite overwrite? + ::bf.v1/migrate migrate? + ::bf.v1/profile-id profile-id + ::bf.v1/project-id project-id)] + (bf.v1/import-files! cfg path) + {::rres/status 200 + ::rres/headers {"content-type" "text/plain"} + ::rres/body "OK"}))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; ACTIONS diff --git a/backend/src/app/rpc/commands/binfile.clj b/backend/src/app/rpc/commands/binfile.clj index 206b8931a3..2621cce6a2 100644 --- a/backend/src/app/rpc/commands/binfile.clj +++ b/backend/src/app/rpc/commands/binfile.clj @@ -7,22 +7,9 @@ (ns app.rpc.commands.binfile (:refer-clojure :exclude [assert]) (:require - [app.common.data :as d] - [app.common.exceptions :as ex] - [app.common.features :as cfeat] - [app.common.files.defaults :as cfd] - [app.common.files.migrations :as fmg] - [app.common.files.validate :as fval] - [app.common.fressian :as fres] - [app.common.logging :as l] + [app.binfile.v1 :as bf.v1] [app.common.schema :as sm] - [app.common.spec :as us] - [app.common.types.file :as ctf] - [app.common.uuid :as uuid] - [app.config :as cf] [app.db :as db] - [app.features.components-v2 :as feat.compv2] - [app.features.fdata :as feat.fdata] [app.http.sse :as sse] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] @@ -30,1065 +17,33 @@ [app.rpc :as-alias rpc] [app.rpc.commands.files :as files] [app.rpc.commands.projects :as projects] - [app.rpc.commands.teams :as teams] [app.rpc.doc :as-alias doc] - [app.storage :as sto] - [app.storage.tmp :as tmp] [app.tasks.file-gc] - [app.util.blob :as blob] - [app.util.pointer-map :as pmap] [app.util.services :as sv] [app.util.time :as dt] [app.worker :as-alias wrk] - [clojure.set :as set] - [clojure.spec.alpha :as s] - [clojure.walk :as walk] - [cuerdas.core :as str] - [datoteka.io :as io] - [promesa.core :as p] - [promesa.util :as pu] - [ring.response :as rres] - [yetti.adapter :as yt]) - (:import - com.github.luben.zstd.ZstdInputStream - com.github.luben.zstd.ZstdOutputStream - java.io.DataInputStream - java.io.DataOutputStream - java.io.InputStream - java.io.OutputStream)) + [promesa.exec :as px] + [ring.response :as rres])) (set! *warn-on-reflection* true) -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; DEFAULTS -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -;; Threshold in MiB when we pass from using -;; in-memory byte-array's to use temporal files. -(def temp-file-threshold - (* 1024 1024 2)) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; LOW LEVEL STREAM IO API -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -(def ^:const buffer-size (:xnio/buffer-size yt/defaults)) -(def ^:const penpot-magic-number 800099563638710213) -(def ^:const max-object-size (* 1024 1024 100)) ; Only allow 100MiB max file size. - -(def ^:dynamic *position* nil) - -(defn get-mark - [id] - (case id - :header 1 - :stream 2 - :uuid 3 - :label 4 - :obj 5 - (ex/raise :type :validation - :code :invalid-mark-id - :hint (format "invalid mark id %s" id)))) - -(defmacro assert - [expr hint] - `(when-not ~expr - (ex/raise :type :validation - :code :unexpected-condition - :hint ~hint))) - -(defmacro assert-mark - [v type] - `(let [expected# (get-mark ~type) - val# (long ~v)] - (when (not= val# expected#) - (ex/raise :type :validation - :code :unexpected-mark - :hint (format "received mark %s, expected %s" val# expected#))))) - -(defmacro assert-label - [expr label] - `(let [v# ~expr] - (when (not= v# ~label) - (ex/raise :type :assertion - :code :unexpected-label - :hint (format "received label %s, expected %s" v# ~label))))) - -;; --- PRIMITIVE IO - -(defn write-byte! - [^DataOutputStream output data] - (l/trace :fn "write-byte!" :data data :position @*position* ::l/sync? true) - (.writeByte output (byte data)) - (swap! *position* inc)) - -(defn read-byte! - [^DataInputStream input] - (let [v (.readByte input)] - (l/trace :fn "read-byte!" :val v :position @*position* ::l/sync? true) - (swap! *position* inc) - v)) - -(defn write-long! - [^DataOutputStream output data] - (l/trace :fn "write-long!" :data data :position @*position* ::l/sync? true) - (.writeLong output (long data)) - (swap! *position* + 8)) - - -(defn read-long! - [^DataInputStream input] - (let [v (.readLong input)] - (l/trace :fn "read-long!" :val v :position @*position* ::l/sync? true) - (swap! *position* + 8) - v)) - -(defn write-bytes! - [^DataOutputStream output ^bytes data] - (let [size (alength data)] - (l/trace :fn "write-bytes!" :size size :position @*position* ::l/sync? true) - (.write output data 0 size) - (swap! *position* + size))) - -(defn read-bytes! - [^InputStream input ^bytes buff] - (let [size (alength buff) - readed (.readNBytes input buff 0 size)] - (l/trace :fn "read-bytes!" :expected (alength buff) :readed readed :position @*position* ::l/sync? true) - (swap! *position* + readed) - readed)) - -;; --- COMPOSITE IO - -(defn write-uuid! - [^DataOutputStream output id] - (l/trace :fn "write-uuid!" :position @*position* :WRITTEN? (.size output) ::l/sync? true) - - (doto output - (write-byte! (get-mark :uuid)) - (write-long! (uuid/get-word-high id)) - (write-long! (uuid/get-word-low id)))) - -(defn read-uuid! - [^DataInputStream input] - (l/trace :fn "read-uuid!" :position @*position* ::l/sync? true) - (let [m (read-byte! input)] - (assert-mark m :uuid) - (let [a (read-long! input) - b (read-long! input)] - (uuid/custom a b)))) - -(defn write-obj! - [^DataOutputStream output data] - (l/trace :fn "write-obj!" :position @*position* ::l/sync? true) - (let [^bytes data (fres/encode data)] - (doto output - (write-byte! (get-mark :obj)) - (write-long! (alength data)) - (write-bytes! data)))) - -(defn read-obj! - [^DataInputStream input] - (l/trace :fn "read-obj!" :position @*position* ::l/sync? true) - (let [m (read-byte! input)] - (assert-mark m :obj) - (let [size (read-long! input)] - (assert (pos? size) "incorrect header size found on reading header") - (let [buff (byte-array size)] - (read-bytes! input buff) - (fres/decode buff))))) - -(defn write-label! - [^DataOutputStream output label] - (l/trace :fn "write-label!" :label label :position @*position* ::l/sync? true) - (doto output - (write-byte! (get-mark :label)) - (write-obj! label))) - -(defn read-label! - [^DataInputStream input] - (l/trace :fn "read-label!" :position @*position* ::l/sync? true) - (let [m (read-byte! input)] - (assert-mark m :label) - (read-obj! input))) - -(defn write-header! - [^OutputStream output version] - (l/trace :fn "write-header!" - :version version - :position @*position* - ::l/sync? true) - (let [vers (-> version name (subs 1) parse-long) - output (io/data-output-stream output)] - (doto output - (write-byte! (get-mark :header)) - (write-long! penpot-magic-number) - (write-long! vers)))) - -(defn read-header! - [^InputStream input] - (l/trace :fn "read-header!" :position @*position* ::l/sync? true) - (let [input (io/data-input-stream input) - mark (read-byte! input) - mnum (read-long! input) - vers (read-long! input)] - - (when (or (not= mark (get-mark :header)) - (not= mnum penpot-magic-number)) - (ex/raise :type :validation - :code :invalid-penpot-file - :hint "invalid penpot file")) - - (keyword (str "v" vers)))) - -(defn copy-stream! - [^OutputStream output ^InputStream input ^long size] - (let [written (io/copy! input output :size size)] - (l/trace :fn "copy-stream!" :position @*position* :size size :written written ::l/sync? true) - (swap! *position* + written) - written)) - -(defn write-stream! - [^DataOutputStream output stream size] - (l/trace :fn "write-stream!" :position @*position* ::l/sync? true :size size) - (doto output - (write-byte! (get-mark :stream)) - (write-long! size)) - - (copy-stream! output stream size)) - -(defn read-stream! - [^DataInputStream input] - (l/trace :fn "read-stream!" :position @*position* ::l/sync? true) - (let [m (read-byte! input) - s (read-long! input) - p (tmp/tempfile :prefix "penpot.binfile.")] - (assert-mark m :stream) - - (when (> s max-object-size) - (ex/raise :type :validation - :code :max-file-size-reached - :hint (str/ffmt "unable to import storage object with size % bytes" s))) - - (if (> s temp-file-threshold) - (with-open [^OutputStream output (io/output-stream p)] - (let [readed (io/copy! input output :offset 0 :size s)] - (l/trace :fn "read-stream*!" :expected s :readed readed :position @*position* ::l/sync? true) - (swap! *position* + readed) - [s p])) - [s (io/read-as-bytes input :size s)]))) - -(defmacro assert-read-label! - [input expected-label] - `(let [readed# (read-label! ~input) - expected# ~expected-label] - (when (not= readed# expected#) - (ex/raise :type :validation - :code :unexpected-label - :hint (format "unexpected label found: %s, expected: %s" readed# expected#))))) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; API -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -;; --- HELPERS - -(defn zstd-input-stream - ^InputStream - [input] - (ZstdInputStream. ^InputStream input)) - -(defn zstd-output-stream - ^OutputStream - [output & {:keys [level] :or {level 0}}] - (ZstdOutputStream. ^OutputStream output (int level))) - -(defn- get-files - [cfg ids] - (db/run! cfg (fn [{:keys [::db/conn]}] - (let [sql (str "SELECT id FROM file " - " WHERE id = ANY(?) ") - ids (db/create-array conn "uuid" ids)] - (->> (db/exec! conn [sql ids]) - (into [] (map :id)) - (not-empty)))))) - -(defn- get-file - [cfg file-id] - (db/run! cfg (fn [{:keys [::db/conn] :as cfg}] - (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg file-id)] - (some-> (db/get* conn :file {:id file-id} {::db/remove-deleted false}) - (files/decode-row) - (update :data feat.fdata/process-pointers deref)))))) - -(defn- get-file-media - [{:keys [::db/pool]} {:keys [data id] :as file}] - (pu/with-open [conn (db/open pool)] - (let [ids (app.tasks.file-gc/collect-used-media data) - ids (db/create-array conn "uuid" ids) - sql (str "SELECT * FROM file_media_object WHERE id = ANY(?)")] - - ;; We assoc the file-id again to the file-media-object row - ;; because there are cases that used objects refer to other - ;; files and we need to ensure in the exportation process that - ;; all ids matches - (->> (db/exec! conn [sql ids]) - (mapv #(assoc % :file-id id)))))) - -(defn- get-file-thumbnails - "Return all file thumbnails for a given file." - [{:keys [::db/pool]} id] - (pu/with-open [conn (db/open pool)] - (let [sql "SELECT * FROM file_tagged_object_thumbnail WHERE file_id = ?"] - (->> (db/exec! conn [sql id]) - (mapv #(dissoc % :file-id)))))) - -(def ^:private storage-object-id-xf - (comp - (mapcat (juxt :media-id :thumbnail-id)) - (filter uuid?))) - -(def ^:private sql:file-libraries - "WITH RECURSIVE libs AS ( - SELECT fl.id - FROM file AS fl - JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id) - WHERE flr.file_id = ANY(?) - UNION - SELECT fl.id - FROM file AS fl - JOIN file_library_rel AS flr ON (flr.library_file_id = fl.id) - JOIN libs AS l ON (flr.file_id = l.id) - ) - SELECT DISTINCT l.id - FROM libs AS l") - -(defn- get-libraries - [{:keys [::db/pool]} ids] - (pu/with-open [conn (db/open pool)] - (let [ids (db/create-array conn "uuid" ids)] - (map :id (db/exec! pool [sql:file-libraries ids]))))) - -(defn- get-library-relations - [cfg ids] - (db/run! cfg (fn [{:keys [::db/conn]}] - (let [ids (db/create-array conn "uuid" ids) - sql (str "SELECT flr.* FROM file_library_rel AS flr " - " WHERE flr.file_id = ANY(?)")] - (db/exec! conn [sql ids]))))) - -(defn- create-or-update-file! - [conn params] - (let [sql (str "INSERT INTO file (id, project_id, name, revn, is_shared, data, created_at, modified_at) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?) " - "ON CONFLICT (id) DO UPDATE SET data=?")] - (db/exec-one! conn [sql - (:id params) - (:project-id params) - (:name params) - (:revn params) - (:is-shared params) - (:data params) - (:created-at params) - (:modified-at params) - (:data params)]))) - -;; --- GENERAL PURPOSE DYNAMIC VARS - -(def ^:dynamic *state* nil) -(def ^:dynamic *options* nil) - -;; --- EXPORT WRITER - -(defn- embed-file-assets - [data cfg file-id] - (letfn [(walk-map-form [form state] - (cond - (uuid? (:fill-color-ref-file form)) - (do - (vswap! state conj [(:fill-color-ref-file form) :colors (:fill-color-ref-id form)]) - (assoc form :fill-color-ref-file file-id)) - - (uuid? (:stroke-color-ref-file form)) - (do - (vswap! state conj [(:stroke-color-ref-file form) :colors (:stroke-color-ref-id form)]) - (assoc form :stroke-color-ref-file file-id)) - - (uuid? (:typography-ref-file form)) - (do - (vswap! state conj [(:typography-ref-file form) :typographies (:typography-ref-id form)]) - (assoc form :typography-ref-file file-id)) - - (uuid? (:component-file form)) - (do - (vswap! state conj [(:component-file form) :components (:component-id form)]) - (assoc form :component-file file-id)) - - :else - form)) - - (process-group-of-assets [data [lib-id items]] - ;; NOTE: there is a possibility that shape refers to an - ;; non-existant file because the file was removed. In this - ;; case we just ignore the asset. - (if-let [lib (get-file cfg lib-id)] - (reduce (partial process-asset lib) data items) - data)) - - (process-asset [lib data [bucket asset-id]] - (let [asset (get-in lib [:data bucket asset-id]) - ;; Add a special case for colors that need to have - ;; correctly set the :file-id prop (pending of the - ;; refactor that will remove it). - asset (cond-> asset - (= bucket :colors) (assoc :file-id file-id))] - (update data bucket assoc asset-id asset)))] - - (let [assets (volatile! [])] - (walk/postwalk #(cond-> % (map? %) (walk-map-form assets)) data) - (->> (deref assets) - (filter #(as-> (first %) $ (and (uuid? $) (not= $ file-id)))) - (d/group-by first rest) - (reduce (partial process-group-of-assets) data))))) - -(defmulti write-export ::version) -(defmulti write-section ::section) - -(s/def ::output io/output-stream?) -(s/def ::file-ids (s/every ::us/uuid :kind vector? :min-count 1)) -(s/def ::include-libraries? (s/nilable ::us/boolean)) -(s/def ::embed-assets? (s/nilable ::us/boolean)) - -(s/def ::write-export-options - (s/keys :req [::db/pool ::sto/storage ::output ::file-ids] - :opt [::include-libraries? ::embed-assets?])) - -(defn write-export! - "Do the exportation of a specified file in custom penpot binary - format. There are some options available for customize the output: - - `::include-libraries?`: additionally to the specified file, all the - linked libraries also will be included (including transitive - dependencies). - - `::embed-assets?`: instead of including the libraries, embed in the - same file library all assets used from external libraries." - [{:keys [::include-libraries? ::embed-assets?] :as options}] - - (us/assert! ::write-export-options options) - (us/verify! - :expr (not (and include-libraries? embed-assets?)) - :hint "the `include-libraries?` and `embed-assets?` are mutally excluding options") - (write-export options)) - -(defmethod write-export :default - [{:keys [::output] :as options}] - (write-header! output :v1) - (pu/with-open [output (zstd-output-stream output :level 12) - output (io/data-output-stream output)] - (binding [*state* (volatile! {})] - (run! (fn [section] - (l/dbg :hint "write section" :section section ::l/sync? true) - (write-label! output section) - (let [options (-> options - (assoc ::output output) - (assoc ::section section))] - (binding [*options* options] - (write-section options)))) - - [:v1/metadata :v1/files :v1/rels :v1/sobjects])))) - -(defmethod write-section :v1/metadata - [{:keys [::output ::file-ids ::include-libraries?] :as cfg}] - (if-let [fids (get-files cfg file-ids)] - (let [lids (when include-libraries? - (get-libraries cfg file-ids)) - ids (into fids lids)] - (write-obj! output {:version cf/version :files ids}) - (vswap! *state* assoc :files ids)) - (ex/raise :type :not-found - :code :files-not-found - :hint "unable to retrieve files for export"))) - -(defmethod write-section :v1/files - [{:keys [::output ::embed-assets? ::include-libraries?] :as cfg}] - - ;; Initialize SIDS with empty vector - (vswap! *state* assoc :sids []) - - (doseq [file-id (-> *state* deref :files)] - (let [detach? (and (not embed-assets?) (not include-libraries?)) - thumbnails (get-file-thumbnails cfg file-id) - file (cond-> (get-file cfg file-id) - detach? - (-> (ctf/detach-external-references file-id) - (dissoc :libraries)) - - embed-assets? - (update :data embed-file-assets cfg file-id) - - :always - (assoc :thumbnails thumbnails)) - - media (get-file-media cfg file)] - - (l/dbg :hint "write penpot file" - :id (str file-id) - :name (:name file) - :thumbnails (count thumbnails) - :features (:features file) - :media (count media) - ::l/sync? true) - - (doseq [item media] - (l/dbg :hint "write penpot file media object" :id (:id item) ::l/sync? true)) - - (doseq [item thumbnails] - (l/dbg :hint "write penpot file object thumbnail" :media-id (str (:media-id item)) ::l/sync? true)) - - (doto output - (write-obj! file) - (write-obj! media)) - - (vswap! *state* update :sids into storage-object-id-xf media) - (vswap! *state* update :sids into storage-object-id-xf thumbnails)))) - -(defmethod write-section :v1/rels - [{:keys [::output ::include-libraries?] :as cfg}] - (let [ids (-> *state* deref :files) - rels (when include-libraries? - (get-library-relations cfg ids))] - (l/dbg :hint "found rels" :total (count rels) ::l/sync? true) - (write-obj! output rels))) - -(defmethod write-section :v1/sobjects - [{:keys [::sto/storage ::output]}] - (let [sids (-> *state* deref :sids) - storage (media/configure-assets-storage storage)] - - (l/dbg :hint "found sobjects" - :items (count sids) - ::l/sync? true) - - ;; Write all collected storage objects - (write-obj! output sids) - - (doseq [id sids] - (let [{:keys [size] :as obj} (sto/get-object storage id)] - (l/dbg :hint "write sobject" :id (str id) ::l/sync? true) - - (doto output - (write-uuid! id) - (write-obj! (meta obj))) - - (pu/with-open [stream (sto/get-object-data storage obj)] - (let [written (write-stream! output stream size)] - (when (not= written size) - (ex/raise :type :validation - :code :mismatch-readed-size - :hint (str/ffmt "found unexpected object size; size=% written=%" size written))))))))) - -;; --- EXPORT READER - -(declare lookup-index) -(declare update-index) -(declare relink-media) -(declare relink-colors) -(declare relink-shapes) - -(defmulti read-import ::version) -(defmulti read-section ::section) - -(s/def ::profile-id ::us/uuid) -(s/def ::project-id ::us/uuid) -(s/def ::input io/input-stream?) -(s/def ::overwrite? (s/nilable ::us/boolean)) -(s/def ::ignore-index-errors? (s/nilable ::us/boolean)) - -;; FIXME: replace with schema -(s/def ::read-import-options - (s/keys :req [::db/pool ::sto/storage ::project-id ::profile-id ::input] - :opt [::overwrite? ::ignore-index-errors?])) - -(defn read-import! - "Do the importation of the specified resource in penpot custom binary - format. There are some options for customize the importation - behavior: - - `::overwrite?`: if true, instead of creating new files and remapping id references, - it reuses all ids and updates existing objects; defaults to `false`. - - `::ignore-index-errors?`: if true, do not fail on index lookup errors, can - happen with broken files; defaults to: `false`. - " - - [{:keys [::input ::timestamp] :or {timestamp (dt/now)} :as options}] - (us/verify! ::read-import-options options) - (let [version (read-header! input)] - (read-import (assoc options ::version version ::timestamp timestamp)))) - -(defn- read-import-v1 - [{:keys [::db/conn ::project-id ::profile-id ::input] :as options}] - (db/exec-one! conn ["SET idle_in_transaction_session_timeout = 0"]) - (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]) - - (pu/with-open [input (zstd-input-stream input) - input (io/data-input-stream input)] - (binding [*state* (volatile! {:media [] :index {}})] - (let [team (teams/get-team conn - :profile-id profile-id - :project-id project-id) - - validate? (contains? cf/flags :file-validation) - features (cfeat/get-team-enabled-features cf/flags team)] - - (sse/tap {:type :import-progress - :section :read-import}) - - ;; Process all sections - (run! (fn [section] - (l/dbg :hint "reading section" :section section ::l/sync? true) - (assert-read-label! input section) - (let [options (-> options - (assoc ::enabled-features features) - (assoc ::section section) - (assoc ::input input))] - (binding [*options* options] - (sse/tap {:type :import-progress - :section section}) - (read-section options)))) - [:v1/metadata :v1/files :v1/rels :v1/sobjects]) - - ;; Run all pending migrations - (doseq [[feature file-id] (-> *state* deref :pending-to-migrate)] - (case feature - "components/v2" - (feat.compv2/migrate-file! options file-id - :validate? validate?) - - "fdata/shape-data-type" - nil - - (ex/raise :type :internal - :code :no-migration-defined - :hint (str/ffmt "no migation for feature '%' on file importation" feature) - :feature feature))) - - ;; Knowing that the ids of the created files are in index, - ;; just lookup them and return it as a set - (let [files (-> *state* deref :files)] - (into #{} (keep #(get-in @*state* [:index %])) files)))))) - -(defmethod read-import :v1 - [options] - (db/tx-run! options read-import-v1)) - -(defmethod read-section :v1/metadata - [{:keys [::input]}] - (let [{:keys [version files]} (read-obj! input)] - (l/dbg :hint "metadata readed" - :version (:full version) - :files (mapv str files) - ::l/sync? true) - (vswap! *state* update :index update-index files) - (vswap! *state* assoc :version version :files files))) - -(defn- get-remaped-thumbnails - [thumbnails file-id] - (mapv (fn [thumbnail] - (-> thumbnail - (assoc :file-id file-id) - (update :object-id #(str/replace-first % #"^(.*?)/" (str file-id "/"))))) - thumbnails)) - -(defn- process-file - [{:keys [id] :as file}] - (-> file - (update :data (fn [fdata] - (-> fdata - (assoc :id id) - (dissoc :recent-colors) - (cond-> (> (:version fdata) cfd/version) - (assoc :version cfd/version)) - ;; FIXME: We're temporarily activating all - ;; migrations because a problem in the - ;; environments messed up with the version - ;; numbers When this problem is fixed delete - ;; the following line - (cond-> (> (:version fdata) 22) - (assoc :version 22))))) - (fmg/migrate-file) - (update :data (fn [fdata] - (-> fdata - (update :pages-index relink-shapes) - (update :components relink-shapes) - (update :media relink-media) - (update :colors relink-colors) - (d/without-nils)))))) - - -(defmethod read-section :v1/files - [{:keys [::db/conn ::input ::project-id ::enabled-features ::timestamp ::overwrite? ::name] :as system}] - - (doseq [[idx expected-file-id] (d/enumerate (-> *state* deref :files))] - (let [file (read-obj! input) - - media (read-obj! input) - - file-id (:id file) - file-id' (lookup-index file-id) - - thumbnails (:thumbnails file)] - - (when (not= file-id expected-file-id) - (ex/raise :type :validation - :code :inconsistent-penpot-file - :found-id file-id - :expected-id expected-file-id - :hint "the penpot file seems corrupt, found unexpected uuid (file-id)")) - - (l/dbg :hint "processing file" - :id (str file-id) - :features (:features file) - :version (-> file :data :version) - :media (count media) - :thumbnails (count thumbnails) - ::l/sync? true) - - (when (seq thumbnails) - (let [thumbnails (get-remaped-thumbnails thumbnails file-id')] - (l/dbg :hint "updated index with thumbnails" :total (count thumbnails) ::l/sync? true) - (vswap! *state* update :thumbnails (fnil into []) thumbnails))) - - (when (seq media) - ;; Update index with media - (l/dbg :hint "update index with media" :total (count media) ::l/sync? true) - (vswap! *state* update :index update-index (map :id media)) - - ;; Store file media for later insertion - (l/dbg :hint "update media references" ::l/sync? true) - (vswap! *state* update :media into (map #(update % :id lookup-index)) media)) - - (let [file (-> file - (assoc :id file-id') - (cond-> (and (= idx 0) (some? name)) - (assoc :name name)) - (process-file)) - - ;; All features that are enabled and requires explicit migration are - ;; added to the state for a posterior migration step. - _ (doseq [feature (-> enabled-features - (set/difference cfeat/no-migration-features) - (set/difference (:features file)))] - (vswap! *state* update :pending-to-migrate (fnil conj []) [feature file-id'])) - - file (-> file - (assoc :project-id project-id) - (assoc :created-at timestamp) - (assoc :modified-at timestamp) - (dissoc :thumbnails) - (update :features - (fn [features] - (let [features (cfeat/check-supported-features! features)] - (-> enabled-features - (set/difference cfeat/frontend-only-features) - (set/union features)))))) - - _ (when (contains? cf/flags :file-schema-validation) - (fval/validate-file-schema! file)) - - _ (when (contains? cf/flags :soft-file-schema-validation) - (let [result (ex/try! (fval/validate-file-schema! file))] - (when (ex/exception? result) - (l/error :hint "file schema validation error" :cause result)))) - - file (if (contains? (:features file) "fdata/objects-map") - (feat.fdata/enable-objects-map file) - file) - - file (if (contains? (:features file) "fdata/pointer-map") - (binding [pmap/*tracked* (pmap/create-tracked)] - (let [file (feat.fdata/enable-pointer-map file)] - (feat.fdata/persist-pointers! system file-id') - file)) - file) - - file (-> file - (update :features #(db/create-array conn "text" %)) - (update :data blob/encode))] - - (l/dbg :hint "create file" :id (str file-id') ::l/sync? true) - - (if overwrite? - (create-or-update-file! conn file) - (db/insert! conn :file file)) - - (when overwrite? - (db/delete! conn :file-thumbnail {:file-id file-id'})) - - file-id')))) - -(defmethod read-section :v1/rels - [{:keys [::db/conn ::input ::timestamp]}] - (let [rels (read-obj! input) - ids (into #{} (-> *state* deref :files))] - ;; Insert all file relations - (doseq [{:keys [library-file-id] :as rel} rels] - (let [rel (-> rel - (assoc :synced-at timestamp) - (update :file-id lookup-index) - (update :library-file-id lookup-index))] - - (if (contains? ids library-file-id) - (do - (l/dbg :hint "create file library link" - :file-id (:file-id rel) - :lib-id (:library-file-id rel) - ::l/sync? true) - (db/insert! conn :file-library-rel rel)) - - (l/warn :hint "ignoring file library link" - :file-id (:file-id rel) - :lib-id (:library-file-id rel) - ::l/sync? true)))))) - -(defmethod read-section :v1/sobjects - [{:keys [::sto/storage ::db/conn ::input ::overwrite? ::timestamp]}] - (let [storage (media/configure-assets-storage storage) - ids (read-obj! input) - thumb? (into #{} (map :media-id) (:thumbnails @*state*))] - - (doseq [expected-storage-id ids] - (let [id (read-uuid! input) - mdata (read-obj! input)] - - (when (not= id expected-storage-id) - (ex/raise :type :validation - :code :inconsistent-penpot-file - :hint "the penpot file seems corrupt, found unexpected uuid (storage-object-id)")) - - (l/dbg :hint "readed storage object" :id (str id) ::l/sync? true) - - (let [[size resource] (read-stream! input) - hash (sto/calculate-hash resource) - content (-> (sto/content resource size) - (sto/wrap-with-hash hash)) - - params (-> mdata - (assoc ::sto/content content) - (assoc ::sto/deduplicate? true) - (assoc ::sto/touched-at timestamp)) - - params (if (thumb? id) - (assoc params :bucket "file-object-thumbnail") - (assoc params :bucket "file-media-object")) - - sobject (sto/put-object! storage params)] - - (l/dbg :hint "persisted storage object" - :old-id (str id) - :new-id (str (:id sobject)) - :is-thumbnail (boolean (thumb? id)) - ::l/sync? true) - - (vswap! *state* update :index assoc id (:id sobject))))) - - (doseq [item (:media @*state*)] - (l/dbg :hint "inserting file media object" - :id (str (:id item)) - :file-id (str (:file-id item)) - ::l/sync? true) - - (let [file-id (lookup-index (:file-id item))] - (if (= file-id (:file-id item)) - (l/warn :hint "ignoring file media object" :file-id (str file-id) ::l/sync? true) - (db/insert! conn :file-media-object - (-> item - (assoc :file-id file-id) - (d/update-when :media-id lookup-index) - (d/update-when :thumbnail-id lookup-index)) - {::db/on-conflict-do-nothing? overwrite?})))) - - (doseq [item (:thumbnails @*state*)] - (let [item (update item :media-id lookup-index)] - (l/dbg :hint "inserting file object thumbnail" - :file-id (str (:file-id item)) - :media-id (str (:media-id item)) - :object-id (:object-id item) - ::l/sync? true) - (db/insert! conn :file-tagged-object-thumbnail item - {::db/on-conflict-do-nothing? overwrite?}))))) - -(defn- lookup-index - [id] - (let [val (get-in @*state* [:index id])] - (l/trc :fn "lookup-index" :id id :val val ::l/sync? true) - (when (and (not (::ignore-index-errors? *options*)) (not val)) - (ex/raise :type :validation - :code :incomplete-index - :hint "looks like index has missing data")) - (or val id))) - -(defn- update-index - [index coll] - (loop [items (seq coll) - index index] - (if-let [id (first items)] - (let [new-id (if (::overwrite? *options*) id (uuid/next))] - (l/trc :fn "update-index" :id id :new-id new-id ::l/sync? true) - (recur (rest items) - (assoc index id new-id))) - index))) - -(defn- relink-shapes - "A function responsible to analyze all file data and - replace the old :component-file reference with the new - ones, using the provided file-index." - [data] - (letfn [(process-map-form [form] - (cond-> form - ;; Relink image shapes - (and (map? (:metadata form)) - (= :image (:type form))) - (update-in [:metadata :id] lookup-index) - - ;; Relink paths with fill image - (map? (:fill-image form)) - (update-in [:fill-image :id] lookup-index) - - ;; This covers old shapes and the new :fills. - (uuid? (:fill-color-ref-file form)) - (update :fill-color-ref-file lookup-index) - - ;; This covers the old shapes and the new :strokes - (uuid? (:storage-color-ref-file form)) - (update :stroke-color-ref-file lookup-index) - - ;; This covers all text shapes that have typography referenced - (uuid? (:typography-ref-file form)) - (update :typography-ref-file lookup-index) - - ;; This covers the component instance links - (uuid? (:component-file form)) - (update :component-file lookup-index) - - ;; This covers the shadows and grids (they have directly - ;; the :file-id prop) - (uuid? (:file-id form)) - (update :file-id lookup-index)))] - - (walk/postwalk (fn [form] - (if (map? form) - (try - (process-map-form form) - (catch Throwable cause - (l/warn :hint "failed form" :form (pr-str form) ::l/sync? true) - (throw cause))) - form)) - data))) - -(defn- relink-media - "A function responsible of process the :media attr of file data and - remap the old ids with the new ones." - [media] - (reduce-kv (fn [res k v] - (let [id (lookup-index k)] - (if (uuid? id) - (-> res - (assoc id (assoc v :id id)) - (dissoc k)) - res))) - media - media)) - -(defn- relink-colors - "A function responsible of process the :colors attr of file data and - remap the old ids with the new ones." - [colors] - (reduce-kv (fn [res k v] - (if (:image v) - (update-in res [k :image :id] lookup-index) - res)) - colors - colors)) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; HIGH LEVEL API -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -(defn export! - [cfg output] - (let [id (uuid/next) - tp (dt/tpoint) - ab (volatile! false) - cs (volatile! nil)] - (try - (l/info :hint "start exportation" :export-id (str id)) - (pu/with-open [output (io/output-stream output)] - (binding [*position* (atom 0)] - (write-export! (assoc cfg ::output output)))) - - (catch java.io.IOException _cause - ;; Do nothing, EOF means client closes connection abruptly - (vreset! ab true) - nil) - - (catch Throwable cause - (vreset! cs cause) - (vreset! ab true) - (throw cause)) - - (finally - (l/info :hint "exportation finished" :export-id (str id) - :elapsed (str (inst-ms (tp)) "ms") - :aborted @ab - :cause @cs))))) - -(defn export-to-tmpfile! - [cfg] - (let [path (tmp/tempfile :prefix "penpot.export.")] - (pu/with-open [output (io/output-stream path)] - (export! cfg output) - path))) - -(defn import! - [{:keys [::input] :as cfg}] - (let [id (uuid/next) - tp (dt/tpoint) - cs (volatile! nil)] - (l/info :hint "import: started" :id (str id)) - (try - (binding [*position* (atom 0)] - (pu/with-open [input (io/input-stream input)] - (read-import! (assoc cfg ::input input)))) - - (catch Throwable cause - (vreset! cs cause) - (throw cause)) - - (finally - (l/info :hint "import: terminated" - :id (str id) - :elapsed (dt/format-duration (tp)) - :error? (some? @cs)))))) - - ;; --- Command: export-binfile (def ^:private schema:export-binfile (sm/define [:map {:title "export-binfile"} + [:name :string] [:file-id ::sm/uuid] - [:include-libraries? :boolean] - [:embed-assets? :boolean]])) + [:include-libraries :boolean] + [:embed-assets :boolean]])) (sv/defmethod ::export-binfile "Export a penpot file in a binary format." {::doc/added "1.15" ::webhooks/event? true ::sm/result schema:export-binfile} - [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id include-libraries? embed-assets?] :as params}] + [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id include-libraries embed-assets] :as params}] (files/check-read-permissions! pool profile-id file-id) (fn [_] {::rres/status 200 @@ -1096,14 +51,27 @@ ::rres/body (reify rres/StreamableResponseBody (-write-body-to-stream [_ _ output-stream] (-> cfg - (assoc ::file-ids [file-id]) - (assoc ::embed-assets? embed-assets?) - (assoc ::include-libraries? include-libraries?) - (export! output-stream))))})) - + (assoc ::bf.v1/ids #{file-id}) + (assoc ::bf.v1/embed-assets embed-assets) + (assoc ::bf.v1/include-libraries include-libraries) + (bf.v1/export-files! output-stream))))})) ;; --- Command: import-binfile +(defn- import-binfile + [{:keys [::wrk/executor ::bf.v1/project-id] :as cfg} input] + (db/tx-run! cfg + (fn [{:keys [::db/conn] :as cfg}] + ;; NOTE: the importation process performs some operations that + ;; are not very friendly with virtual threads, and for avoid + ;; unexpected blocking of other concurrent operations we + ;; dispatch that operation to a dedicated executor. + (let [result (px/submit! executor (partial bf.v1/import-files! cfg input))] + (db/update! conn :project + {:modified-at (dt/now)} + {:id project-id}) + (deref result))))) + (def ^:private schema:import-binfile (sm/define @@ -1112,8 +80,6 @@ [:project-id ::sm/uuid] [:file ::media/upload]])) -(declare ^:private import-binfile) - (sv/defmethod ::import-binfile "Import a penpot file in a binary format." {::doc/added "1.15" @@ -1122,26 +88,10 @@ ::sm/params schema:import-binfile} [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id name project-id file] :as params}] (projects/check-read-permissions! pool profile-id project-id) - (let [params (-> cfg - (assoc ::input (:path file)) - (assoc ::project-id project-id) - (assoc ::profile-id profile-id) - (assoc ::name name) - (assoc ::ignore-index-errors? true))] + (let [cfg (-> cfg + (assoc ::bf.v1/project-id project-id) + (assoc ::bf.v1/profile-id profile-id) + (assoc ::bf.v1/name name))] (with-meta - (sse/response #(import-binfile params)) + (sse/response #(import-binfile cfg (:path file))) {::audit/props {:file nil}}))) - -(defn- import-binfile - [{:keys [::wrk/executor ::project-id] :as params}] - (db/tx-run! params - (fn [{:keys [::db/conn] :as params}] - ;; NOTE: the importation process performs some operations that - ;; are not very friendly with virtual threads, and for avoid - ;; unexpected blocking of other concurrent operations we - ;; dispatch that operation to a dedicated executor. - (let [result (p/thread-call executor (partial import! params))] - (db/update! conn :project - {:modified-at (dt/now)} - {:id project-id}) - (deref result))))) diff --git a/backend/src/app/rpc/commands/management.clj b/backend/src/app/rpc/commands/management.clj index 416056a914..5d01d9ec60 100644 --- a/backend/src/app/rpc/commands/management.clj +++ b/backend/src/app/rpc/commands/management.clj @@ -7,51 +7,83 @@ (ns app.rpc.commands.management "A collection of RPC methods for manage the files, projects and team organization." (:require - [app.common.data :as d] + [app.binfile.common :as bfc] + [app.binfile.v1 :as bf.v1] [app.common.exceptions :as ex] [app.common.features :as cfeat] - [app.common.files.migrations :as pmg] [app.common.schema :as sm] [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] - [app.features.fdata :as feat.fdata] [app.http.sse :as sse] [app.loggers.webhooks :as-alias webhooks] [app.rpc :as-alias rpc] - [app.rpc.commands.binfile :as binfile] [app.rpc.commands.files :as files] [app.rpc.commands.projects :as proj] [app.rpc.commands.teams :as teams] [app.rpc.doc :as-alias doc] [app.setup :as-alias setup] [app.setup.templates :as tmpl] - [app.util.blob :as blob] - [app.util.pointer-map :as pmap] [app.util.services :as sv] [app.util.time :as dt] [app.worker :as-alias wrk] - [clojure.walk :as walk] - [promesa.core :as p] [promesa.exec :as px])) -(defn- index-row - [index obj] - (assoc index (:id obj) (uuid/next))) - -(defn- lookup-index - [id index] - (get index id id)) - -(defn- remap-id - [item index key] - (cond-> item - (contains? item key) - (update key lookup-index index))) - ;; --- COMMAND: Duplicate File -(declare duplicate-file) +(defn duplicate-file + [{:keys [::db/conn ::bfc/timestamp] :as cfg} {:keys [profile-id file-id name reset-shared-flag] :as params}] + (let [;; We don't touch the original file on duplication + file (bfc/get-file cfg file-id) + project-id (:project-id file) + file (-> file + (update :id bfc/lookup-index) + (update :project-id bfc/lookup-index) + (cond-> (string? name) + (assoc :name name)) + (cond-> (true? reset-shared-flag) + (assoc :is-shared false))) + + flibs (bfc/get-files-rels cfg #{file-id}) + fmeds (bfc/get-file-media cfg file)] + + (when (uuid? profile-id) + (proj/check-edition-permissions! conn profile-id project-id)) + + (vswap! bfc/*state* update :index bfc/update-index fmeds :id) + + ;; Process and persist file + (let [file (->> (bfc/process-file file) + (bfc/persist-file! cfg))] + + ;; The file profile creation is optional, so when no profile is + ;; present (when this function is called from profile less + ;; environment: SREPL) we just omit the creation of the relation + (when (uuid? profile-id) + (db/insert! conn :file-profile-rel + {:file-id (:id file) + :profile-id profile-id + :is-owner true + :is-admin true + :can-edit true} + {::db/return-keys? false})) + + (doseq [params (sequence (comp + (map #(bfc/remap-id % :file-id)) + (map #(bfc/remap-id % :library-file-id)) + (map #(assoc % :synced-at timestamp)) + (map #(assoc % :created-at timestamp))) + flibs)] + (db/insert! conn :file-library-rel params ::db/return-keys false)) + + (doseq [params (sequence (comp + (map #(bfc/remap-id % :id)) + (map #(assoc % :created-at timestamp)) + (map #(bfc/remap-id % :file-id))) + fmeds)] + (db/insert! conn :file-media-object params ::db/return-keys false)) + + file))) (def ^:private schema:duplicate-file @@ -69,178 +101,51 @@ (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}] (db/exec-one! conn ["SET CONSTRAINTS ALL DEFERRED"]) - (let [params (-> params - (assoc :index {file-id (uuid/next)}) - (assoc :profile-id profile-id) - (assoc ::reset-shared-flag? true))] - (duplicate-file cfg params))))) - -(defn- process-file - [cfg index {:keys [id] :as file}] - (letfn [(process-form [form] - (cond-> form - ;; Relink library items - (and (map? form) - (uuid? (:component-file form))) - (update :component-file #(get index % %)) - - (and (map? form) - (uuid? (:fill-color-ref-file form))) - (update :fill-color-ref-file #(get index % %)) - - (and (map? form) - (uuid? (:stroke-color-ref-file form))) - (update :stroke-color-ref-file #(get index % %)) - - (and (map? form) - (uuid? (:typography-ref-file form))) - (update :typography-ref-file #(get index % %)) - - ;; Relink Image Shapes - (and (map? form) - (map? (:metadata form)) - (= :image (:type form))) - (update-in [:metadata :id] #(get index % %)))) - - ;; A function responsible to analyze all file data and - ;; replace the old :component-file reference with the new - ;; ones, using the provided file-index - (relink-shapes [data] - (walk/postwalk process-form data)) - - ;; A function responsible of process the :media attr of file - ;; data and remap the old ids with the new ones. - (relink-media [media] - (reduce-kv (fn [res k v] - (let [id (get index k)] - (if (uuid? id) - (-> res - (assoc id (assoc v :id id)) - (dissoc k)) - res))) - media - media)) - - (process-file [{:keys [id] :as file}] - (-> file - (update :data assoc :id id) - (pmg/migrate-file) - (update :data (fn [data] - (-> data - (update :pages-index relink-shapes) - (update :components relink-shapes) - (update :media relink-media) - (d/without-nils))))))] - - (let [file (binding [pmap/*load-fn* (partial feat.fdata/load-pointer cfg id)] - (-> file - (update :id lookup-index index) - (update :project-id lookup-index index) - (update :data feat.fdata/process-pointers deref) - (process-file))) - - file (if (contains? (:features file) "fdata/objects-map") - (feat.fdata/enable-objects-map file) - file) - - file (if (contains? (:features file) "fdata/pointer-map") - (binding [pmap/*tracked* (pmap/create-tracked)] - (let [file (feat.fdata/enable-pointer-map file)] - (feat.fdata/persist-pointers! cfg (:id file)) - file)) - file)] - file))) - -(defn duplicate-file - [{:keys [::db/conn] :as cfg} {:keys [profile-id index file-id name ::reset-shared-flag?]}] - (let [;; We don't touch the original file on duplication - file (files/get-file cfg file-id :migrate? false) - - ;; We only check permissions if profile-id is present; it can - ;; be omited when this function is called from SREPL helpers - _ (when (uuid? profile-id) - (proj/check-edition-permissions! conn profile-id (:project-id file))) - - flibs (let [sql (str "SELECT flr.* " - " FROM file_library_rel AS flr " - " JOIN file AS l ON (flr.library_file_id = l.id) " - " WHERE flr.file_id = ? AND l.deleted_at is null")] - (db/exec! conn [sql file-id])) - - fmeds (let [sql (str "SELECT fmo.* " - " FROM file_media_object AS fmo " - " JOIN storage_object AS so ON (fmo.media_id = so.id) " - " WHERE fmo.file_id = ? AND so.deleted_at is null")] - (db/exec! conn [sql file-id])) - - ;; memo uniform creation/modification date - now (dt/now) - ignore (dt/plus now (dt/duration {:seconds 5})) - - ;; add to the index all file media objects. - index (reduce index-row index fmeds) - - flibs-xf (comp - (map #(remap-id % index :file-id)) - (map #(remap-id % index :library-file-id)) - (map #(assoc % :synced-at now)) - (map #(assoc % :created-at now))) - - ;; remap all file-library-rel row - flibs (sequence flibs-xf flibs) - - fmeds-xf (comp - (map #(assoc % :id (get index (:id %)))) - (map #(assoc % :created-at now)) - (map #(remap-id % index :file-id))) - - ;; remap all file-media-object rows - fmeds (sequence fmeds-xf fmeds) - - file (cond-> file - (string? name) - (assoc :name name) - - (true? reset-shared-flag?) - (assoc :is-shared false)) - - file (-> file - (assoc :created-at now) - (assoc :modified-at now) - (assoc :ignore-sync-until ignore)) - - file (process-file cfg index file)] - - (db/insert! conn :file - (-> file - (update :features #(db/create-array conn "text" %)) - (update :data blob/encode)) - {::db/return-keys false}) - - ;; The file profile creation is optional, so when no profile is - ;; present (when this function is called from profile less - ;; environment: SREPL) we just omit the creation of the relation - - (when (uuid? profile-id) - (db/insert! conn :file-profile-rel - {:file-id (:id file) - :profile-id profile-id - :is-owner true - :is-admin true - :can-edit true} - {::db/return-keys? false})) - - (doseq [params flibs] - (db/insert! conn :file-library-rel params ::db/return-keys false)) - - (doseq [params fmeds] - (db/insert! conn :file-media-object params ::db/return-keys false)) - - file)) + (binding [bfc/*state* (volatile! {:index {file-id (uuid/next)}})] + (duplicate-file (assoc cfg ::bfc/timestamp (dt/now)) + (-> params + (assoc :profile-id profile-id) + (assoc :reset-shared-flag true))))))) ;; --- COMMAND: Duplicate Project -(declare duplicate-project) +(defn duplicate-project + [{:keys [::db/conn ::bfc/timestamp] :as cfg} {:keys [profile-id project-id name] :as params}] + (binding [bfc/*state* (volatile! {:index {project-id (uuid/next)}})] + (let [project (-> (db/get-by-id conn :project project-id) + (assoc :created-at timestamp) + (assoc :modified-at timestamp) + (assoc :is-pinned false) + (update :id bfc/lookup-index) + (cond-> (string? name) + (assoc :name name))) + + files (bfc/get-project-files cfg project-id)] + + ;; Update index with the project files and the project-id + (vswap! bfc/*state* update :index bfc/update-index files) + + + ;; Check if the source team-id allow creating new project for current user + (teams/check-edition-permissions! conn profile-id (:team-id project)) + + ;; create the duplicated project and assign the current profile as + ;; a project owner + (let [project (teams/create-project conn project)] + ;; The project profile creation is optional, so when no profile is + ;; present (when this function is called from profile less + ;; environment: SREPL) we just omit the creation of the relation + (when (uuid? profile-id) + (teams/create-project-role conn profile-id (:id project) :owner)) + + (doseq [file-id files] + (let [params (-> params + (dissoc :name) + (assoc :file-id file-id) + (assoc :reset-shared-flag false))] + (duplicate-file cfg params))) + + project)))) (def ^:private schema:duplicate-project @@ -256,54 +161,13 @@ ::sm/params schema:duplicate-project} [cfg {:keys [::rpc/profile-id] :as params}] (db/tx-run! cfg (fn [cfg] - ;; Defer all constraints + ;; Defer all constraints (db/exec-one! cfg ["SET CONSTRAINTS ALL DEFERRED"]) - (duplicate-project cfg (assoc params :profile-id profile-id))))) - -(defn duplicate-project - [{:keys [::db/conn] :as cfg} {:keys [profile-id project-id name] :as params}] - (let [project (-> (db/get-by-id conn :project project-id) - (assoc :is-pinned false)) - - files (db/query conn :file - {:project-id project-id - :deleted-at nil} - {:columns [:id]}) - - index (reduce index-row {project-id (uuid/next)} files) - - project (cond-> project - (string? name) - (assoc :name name) - - :always - (update :id lookup-index index))] - - ;; Check if the source team-id allow creating new project for current user - (teams/check-edition-permissions! conn profile-id (:team-id project)) - - ;; create the duplicated project and assign the current profile as - ;; a project owner - (teams/create-project conn project) - - ;; The project profile creation is optional, so when no profile is - ;; present (when this function is called from profile less - ;; environment: SREPL) we just omit the creation of the relation - (when (uuid? profile-id) - (teams/create-project-role conn profile-id (:id project) :owner)) - - (doseq [{:keys [id] :as file} files] - (let [params (-> params - (dissoc :name) - (assoc :file-id id) - (assoc :index index) - (assoc ::reset-shared-flag? false))] - (duplicate-file cfg params))) - - project)) + (-> (assoc cfg ::bfc/timestamp (dt/now)) + (duplicate-project (assoc params :profile-id profile-id)))))) (defn duplicate-team - [{:keys [::db/conn] :as cfg} & {:keys [profile-id team-id name] :as params}] + [{:keys [::db/conn ::bfc/timestamp] :as cfg} & {:keys [profile-id team-id name] :as params}] ;; Check if the source team-id allowed to be read by the user if ;; profile-id is present; it can be ommited if this function is @@ -311,92 +175,79 @@ (when (uuid? profile-id) (teams/check-read-permissions! conn profile-id team-id)) - (let [projs (db/query conn :project - {:team-id team-id}) + (binding [bfc/*state* (volatile! {:index {team-id (uuid/next)}})] + (let [projs (bfc/get-team-projects cfg team-id) + files (bfc/get-team-files cfg team-id) + frels (bfc/get-files-rels cfg files) - files (let [sql (str "SELECT f.id " - " FROM file AS f " - " JOIN project AS p ON (p.id = f.project_id) " - " WHERE p.team_id = ? " - " AND p.deleted_at IS NULL " - " AND f.deleted_at IS NULL")] - (db/exec! conn [sql team-id])) + team (-> (db/get-by-id conn :team team-id) + (assoc :created-at timestamp) + (assoc :modified-at timestamp) + (update :id bfc/lookup-index) + (cond-> (string? name) + (assoc :name name))) - trels (db/query conn :team-profile-rel - {:team-id team-id}) + fonts (db/query conn :team-font-variant + {:team-id team-id})] - prels (let [sql (str "SELECT r.* FROM project_profile_rel AS r " - " JOIN project AS p ON (r.project_id = p.id) " - " WHERE p.team_id = ?")] - (db/exec! conn [sql team-id])) + (vswap! bfc/*state* update :index + (fn [index] + (-> index + (bfc/update-index projs) + (bfc/update-index files) + (bfc/update-index fonts :id)))) + ;; FIXME: disallow clone default team + ;; Create the new team in the database + (db/insert! conn :team team) - fonts (db/query conn :team-font-variant - {:team-id team-id}) + ;; Duplicate team <-> profile relations + (doseq [params frels] + (let [params (-> params + (assoc :id (uuid/next)) + (update :team-id bfc/lookup-index) + (assoc :created-at timestamp) + (assoc :modified-at timestamp))] + (db/insert! conn :team-profile-rel params + {::db/return-keys false}))) - index (as-> {team-id (uuid/next)} index - (reduce index-row index projs) - (reduce index-row index files) - (reduce index-row index fonts)) + ;; Duplicate team fonts + (doseq [font fonts] + (let [params (-> font + (update :id bfc/lookup-index) + (update :team-id bfc/lookup-index) + (assoc :created-at timestamp) + (assoc :modified-at timestamp))] + (db/insert! conn :team-font-variant params + {::db/return-keys false}))) - team (db/get-by-id conn :team team-id) - team (cond-> team - (string? name) - (assoc :name name) + ;; Duplicate projects; We don't reuse the `duplicate-project` + ;; here because we handle files duplication by whole team + ;; instead of by project and we want to preserve some project + ;; props which are reset on the `duplicate-project` impl + (doseq [project-id projs] + (let [project (db/get conn :project {:id project-id}) + project (-> project + (assoc :created-at timestamp) + (assoc :modified-at timestamp) + (update :id bfc/lookup-index) + (update :team-id bfc/lookup-index))] + (teams/create-project conn project) - :always - (update :id lookup-index index))] + ;; The project profile creation is optional, so when no profile is + ;; present (when this function is called from profile less + ;; environment: SREPL) we just omit the creation of the relation + (when (uuid? profile-id) + (teams/create-project-role conn profile-id (:id project) :owner)))) - ;; FIXME: disallow clone default team + (doseq [file-id files] + (let [params (-> params + (dissoc :name) + (assoc :file-id file-id) + (assoc :reset-shared-flag false))] + (duplicate-file cfg params))) - ;; Create the new team in the database - (db/insert! conn :team team) - - ;; Duplicate team <-> profile relations - (doseq [params trels] - (let [params (-> params - (assoc :id (uuid/next)) - (update :team-id lookup-index index))] - (db/insert! conn :team-profile-rel params - {::db/return-keys? false}))) - - ;; Duplucate team fonts - (doseq [font fonts] - (let [params (-> font - (update :id lookup-index index) - (update :team-id lookup-index index))] - (db/insert! conn :team-font-variant params - {::db/return-keys? false}))) - - ;; Create all the projects in the database - (doseq [project projs] - (let [project (-> project - (update :id lookup-index index) - (update :team-id lookup-index index))] - (teams/create-project conn project) - - ;; The project profile creation is optional, so when no profile is - ;; present (when this function is called from profile less - ;; environment: SREPL) we just omit the creation of the relation - (when (uuid? profile-id) - (teams/create-project-role conn profile-id (:id project) :owner)))) - - ;; Duplicate project <-> profile relations - (doseq [params prels] - (let [params (-> params - (assoc :id (uuid/next)) - (update :project-id lookup-index index))] - (db/insert! conn :project-profile-rel params))) - - (doseq [file-id (map :id files)] - (let [params (-> params - (dissoc :name) - (assoc :index index) - (assoc :file-id file-id) - (assoc ::reset-shared-flag? false))] - (duplicate-file cfg params))) - - team)) + team))) ;; --- COMMAND: Move file @@ -545,6 +396,19 @@ ;; --- COMMAND: Clone Template +(defn- clone-template + [{:keys [::wrk/executor ::bf.v1/project-id] :as cfg} template] + (db/tx-run! cfg (fn [{:keys [::db/conn] :as cfg}] + ;; NOTE: the importation process performs some operations that + ;; are not very friendly with virtual threads, and for avoid + ;; unexpected blocking of other concurrent operations we + ;; dispatch that operation to a dedicated executor. + (let [result (px/submit! executor (partial bf.v1/import-files! cfg template))] + (db/update! conn :project + {:modified-at (dt/now)} + {:id project-id}) + (deref result))))) + (def ^:private schema:clone-template (sm/define @@ -552,8 +416,6 @@ [:project-id ::sm/uuid] [:template-id ::sm/word-string]])) -(declare ^:private clone-template) - (sv/defmethod ::clone-template "Clone into the specified project the template by its id." {::doc/added "1.16" @@ -565,33 +427,14 @@ _ (teams/check-edition-permissions! pool profile-id (:team-id project)) template (tmpl/get-template-stream cfg template-id) params (-> cfg - (assoc ::binfile/input template) - (assoc ::binfile/project-id (:id project)) - (assoc ::binfile/profile-id profile-id) - (assoc ::binfile/ignore-index-errors? true) - (assoc ::binfile/migrate? true))] - + (assoc ::bf.v1/project-id (:id project)) + (assoc ::bf.v1/profile-id profile-id))] (when-not template (ex/raise :type :not-found :code :template-not-found :hint "template not found")) - (sse/response #(clone-template params)))) - -(defn- clone-template - [{:keys [::wrk/executor ::binfile/project-id] :as params}] - (db/tx-run! params - (fn [{:keys [::db/conn] :as params}] - ;; NOTE: the importation process performs some operations that - ;; are not very friendly with virtual threads, and for avoid - ;; unexpected blocking of other concurrent operations we - ;; dispatch that operation to a dedicated executor. - (let [result (p/thread-call executor (partial binfile/import! params))] - (db/update! conn :project - {:modified-at (dt/now)} - {:id project-id}) - - (deref result))))) + (sse/response #(clone-template params template)))) ;; --- COMMAND: Get list of builtin templates diff --git a/backend/src/app/rpc/commands/teams.clj b/backend/src/app/rpc/commands/teams.clj index 264fca2a1e..381611f818 100644 --- a/backend/src/app/rpc/commands/teams.clj +++ b/backend/src/app/rpc/commands/teams.clj @@ -416,14 +416,16 @@ ;; namespace too. (defn create-project - [conn {:keys [id team-id name is-default] :as params}] + [conn {:keys [id team-id name is-default created-at modified-at]}] (let [id (or id (uuid/next)) - is-default (if (boolean? is-default) is-default false)] - (db/insert! conn :project - {:id id - :name name - :team-id team-id - :is-default is-default}))) + is-default (if (boolean? is-default) is-default false) + params {:id id + :name name + :team-id team-id + :is-default is-default + :created-at created-at + :modified-at modified-at}] + (db/insert! conn :project (d/without-nils params)))) (defn create-project-role [conn profile-id project-id role] diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index d186bfe11f..599afa646d 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -9,6 +9,7 @@ #_:clj-kondo/ignore (:require [app.auth :refer [derive-password]] + [app.binfile.common :as bfc] [app.common.data :as d] [app.common.data.macros :as dm] [app.common.features :as cfeat] @@ -336,4 +337,5 @@ (db/tx-run! main/system (fn [cfg] (db/exec-one! cfg ["SET CONSTRAINTS ALL DEFERRED"]) - (mgmt/duplicate-team cfg :team-id team-id :name name))))) + (-> (assoc cfg ::bfc/timestamp (dt/now)) + (mgmt/duplicate-team :team-id team-id :name name)))))) diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index dcb92a4570..b5b9e4dd91 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -10,6 +10,7 @@ file is eligible to be garbage collected after some period of inactivity (the default threshold is 72h)." (:require + [app.binfile.common :as bfc] [app.common.files.migrations :as pmg] [app.common.logging :as l] [app.common.thumbnails :as thc] @@ -99,35 +100,6 @@ (->> (db/cursor conn [sql:get-candidates min-age] {:chunk-size 1}) (map #(update % :features db/decode-pgarray #{})))))) -(defn collect-used-media - "Given a fdata (file data), returns all media references." - [data] - (let [xform (comp - (map :objects) - (mapcat vals) - (mapcat (fn [obj] - ;; NOTE: because of some bug, we ended with - ;; many shape types having the ability to - ;; have fill-image attribute (which initially - ;; designed for :path shapes). - (sequence - (keep :id) - (concat [(:fill-image obj) - (:metadata obj)] - (map :fill-image (:fills obj)) - (map :stroke-image (:strokes obj)) - (->> (:content obj) - (tree-seq map? :children) - (mapcat :fills) - (map :fill-image))))))) - pages (concat - (vals (:pages-index data)) - (vals (:components data)))] - (-> #{} - (into xform pages) - (into (keys (:media data)))))) - - (def ^:private sql:mark-file-media-object-deleted "UPDATE file_media_object SET deleted_at = now() @@ -137,7 +109,7 @@ (defn- clean-file-media! "Performs the garbage collection of file media objects." [conn file-id data] - (let [used (collect-used-media data) + (let [used (bfc/collect-used-media data) ids (db/create-array conn "uuid" used) unused (->> (db/exec! conn [sql:mark-file-media-object-deleted file-id ids]) (into #{} (map :id)))] diff --git a/frontend/src/app/worker/export.cljs b/frontend/src/app/worker/export.cljs index 24de022fdc..604845c73c 100644 --- a/frontend/src/app/worker/export.cljs +++ b/frontend/src/app/worker/export.cljs @@ -414,8 +414,8 @@ (rx/mapcat (fn [file] (->> (rp/cmd! :export-binfile {:file-id (:id file) - :include-libraries? (= export-type :all) - :embed-assets? (= export-type :merge)}) + :include-libraries (= export-type :all) + :embed-assets (= export-type :merge)}) (rx/map #(hash-map :type :finish :file-id (:id file) :filename (:name file) From cdf312fdd97c4e1940e8a528432207e71282e9fb Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 23 Jan 2024 19:52:23 +0100 Subject: [PATCH 4/7] :sparkles: Add better progress reporting For components migration and for binfile import process --- backend/src/app/binfile/v1.clj | 8 +-- backend/src/app/binfile/v2.clj | 63 +++++++++------- backend/src/app/features/components_v2.clj | 68 ++++++++++-------- backend/src/app/http/sse.clj | 41 +++-------- backend/src/app/srepl/cli.clj | 46 +++++++----- backend/src/app/srepl/components_v2.clj | 71 +++++++++---------- backend/src/app/util/events.clj | 64 +++++++++++++++++ .../backend_tests/rpc_management_test.clj | 2 +- frontend/src/app/main/data/dashboard.cljs | 2 +- frontend/src/app/worker/import.cljs | 2 +- 10 files changed, 215 insertions(+), 152 deletions(-) create mode 100644 backend/src/app/util/events.clj diff --git a/backend/src/app/binfile/v1.clj b/backend/src/app/binfile/v1.clj index 183c3ac697..1e3a7b9176 100644 --- a/backend/src/app/binfile/v1.clj +++ b/backend/src/app/binfile/v1.clj @@ -20,7 +20,6 @@ [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] - [app.http.sse :as sse] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] [app.media :as media] @@ -30,6 +29,7 @@ [app.storage :as sto] [app.storage.tmp :as tmp] [app.tasks.file-gc] + [app.util.events :as events] [app.util.time :as dt] [app.worker :as-alias wrk] [clojure.java.io :as jio] @@ -475,9 +475,6 @@ features (cfeat/get-team-enabled-features cf/flags team)] - (sse/tap {:type :import-progress - :section :read-import}) - ;; Process all sections (run! (fn [section] (l/dbg :hint "reading section" :section section ::l/sync? true) @@ -487,8 +484,7 @@ (assoc ::section section) (assoc ::input input))] (binding [bfc/*options* options] - (sse/tap {:type :import-progress - :section section}) + (events/tap :progress {:op :import :section section}) (read-section options)))) [:v1/metadata :v1/files :v1/rels :v1/sobjects]) diff --git a/backend/src/app/binfile/v2.clj b/backend/src/app/binfile/v2.clj index 33a92a03b9..8ec2e19210 100644 --- a/backend/src/app/binfile/v2.clj +++ b/backend/src/app/binfile/v2.clj @@ -18,12 +18,12 @@ [app.config :as cf] [app.db :as db] [app.db.sql :as sql] - [app.http.sse :as sse] [app.loggers.audit :as-alias audit] [app.loggers.webhooks :as-alias webhooks] [app.media :as media] [app.storage :as sto] [app.storage.tmp :as tmp] + [app.util.events :as events] [app.util.time :as dt] [app.worker :as-alias wrk] [clojure.set :as set] @@ -116,13 +116,15 @@ (defn- write-team! [cfg team-id] - (sse/tap {:type :export-progress - :section :write-team - :id team-id}) - (let [team (bfc/get-team cfg team-id) fonts (bfc/get-fonts cfg team-id)] + (events/tap :progress + {:op :export + :section :write-team + :id team-id + :name (:name team)}) + (l/trc :hint "write" :obj "team" :id (str team-id) :fonts (count fonts)) @@ -138,28 +140,29 @@ (defn- write-project! [cfg project-id] - - (sse/tap {:type :export-progress - :section :write-project - :id project-id}) - (let [project (bfc/get-project cfg project-id)] + (events/tap :progress + {:op :export + :section :write-project + :id project-id + :name (:name project)}) (l/trc :hint "write" :obj "project" :id (str project-id)) (write! cfg :project (str project-id) project) (vswap! bfc/*state* update :projects conj project-id))) (defn- write-file! [cfg file-id] - - (sse/tap {:type :export-progress - :section :write-file - :id file-id}) - (let [file (bfc/get-file cfg file-id) thumbs (bfc/get-file-object-thumbnails cfg file-id) media (bfc/get-file-media cfg file) rels (bfc/get-files-rels cfg #{file-id})] + (events/tap :progress + {:op :export + :section :write-file + :id file-id + :name (:name file)}) + (vswap! bfc/*state* (fn [state] (-> state (update :files conj file-id) @@ -218,10 +221,6 @@ [{:keys [::db/conn ::bfc/timestamp] :as cfg} team-id] (l/trc :hint "read" :obj "team" :id (str team-id)) - (sse/tap {:type :import-progress - :section :read-team - :id team-id}) - (let [team (read-obj cfg :team team-id) team (-> team (update :id bfc/lookup-index) @@ -229,6 +228,12 @@ (assoc :created-at timestamp) (assoc :modified-at timestamp))] + (events/tap :progress + {:op :import + :section :read-team + :id team-id + :name (:name team)}) + (db/insert! conn :team (update team :features db/encode-pgarray conn "text") ::db/return-keys false) @@ -253,10 +258,6 @@ [{:keys [::db/conn ::bfc/timestamp] :as cfg} project-id] (l/trc :hint "read" :obj "project" :id (str project-id)) - (sse/tap {:type :import-progress - :section :read-project - :id project-id}) - (let [project (read-obj cfg :project project-id) project (-> project (update :id bfc/lookup-index) @@ -264,6 +265,12 @@ (assoc :created-at timestamp) (assoc :modified-at timestamp))] + (events/tap :progress + {:op :import + :section :read-project + :id project-id + :name (:name project)}) + (db/insert! conn :project project ::db/return-keys false))) @@ -271,15 +278,17 @@ [{:keys [::db/conn ::bfc/timestamp] :as cfg} file-id] (l/trc :hint "read" :obj "file" :id (str file-id)) - (sse/tap {:type :import-progress - :section :read-file - :id file-id}) - (let [file (-> (read-obj cfg :file file-id) (update :id bfc/lookup-index) (update :project-id bfc/lookup-index) (bfc/process-file))] + (events/tap :progress + {:op :import + :section :read-file + :id file-id + :name (:name file)}) + ;; All features that are enabled and requires explicit migration are ;; added to the state for a posterior migration step. (doseq [feature (-> (::bfc/features cfg) diff --git a/backend/src/app/features/components_v2.clj b/backend/src/app/features/components_v2.clj index 03240a9044..548255a5a4 100644 --- a/backend/src/app/features/components_v2.clj +++ b/backend/src/app/features/components_v2.clj @@ -41,7 +41,6 @@ [app.db :as db] [app.db.sql :as sql] [app.features.fdata :as fdata] - [app.http.sse :as sse] [app.media :as media] [app.rpc.commands.files :as files] [app.rpc.commands.files-snapshot :as fsnap] @@ -51,6 +50,7 @@ [app.svgo :as svgo] [app.util.blob :as blob] [app.util.cache :as cache] + [app.util.events :as events] [app.util.pointer-map :as pmap] [app.util.time :as dt] [buddy.core.codecs :as bc] @@ -767,8 +767,6 @@ backup', generate main instances for all components there and remove shapes from library components. Mark the file with the :components-v2 option." [file-data libraries] - (sse/tap {:type :migration-progress - :section :components}) (let [file-data (prepare-file-data file-data libraries) components (ctkl/components-seq file-data)] (if (empty? components) @@ -843,9 +841,9 @@ add-instance-grid (fn [fdata frame-id grid assets] (reduce (fn [result [component position]] - (sse/tap {:type :migration-progress - :section :components - :name (:name component)}) + (events/tap :progress {:op :migrate-component + :id (:id component) + :name (:name component)}) (add-main-instance result component frame-id (gpt/add position (gpt/point grid-gap grid-gap)))) fdata @@ -881,9 +879,9 @@ (gpt/add position (gpt/point 0 (+ height (* 2 grid-gap) frame-gap)))))))))] (let [total (count components)] - (some-> *stats* (swap! update :processed/components (fnil + 0) total)) - (some-> *team-stats* (swap! update :processed/components (fnil + 0) total)) - (some-> *file-stats* (swap! assoc :processed/components total))) + (some-> *stats* (swap! update :processed-components (fnil + 0) total)) + (some-> *team-stats* (swap! update :processed-components (fnil + 0) total)) + (some-> *file-stats* (swap! assoc :processed-components total))) (add-instance-grids file-data))))) @@ -1143,16 +1141,14 @@ (->> (d/zip media-group grid) (reduce (fn [fdata [mobj position]] - (sse/tap {:type :migration-progress - :section :graphics - :name (:name mobj)}) + (events/tap :progress {:op :migrate-graphic + :id (:id mobj) + :name (:name mobj)}) (or (process fdata mobj position) fdata)) (assoc-in fdata [:options :components-v2] true))))) (defn- migrate-graphics [fdata] - (sse/tap {:type :migration-progress - :section :graphics}) (if (empty? (:media fdata)) fdata (let [[fdata page-id start-pos] @@ -1167,9 +1163,9 @@ groups (get-asset-groups media "Graphics")] (let [total (count media)] - (some-> *stats* (swap! update :processed/graphics (fnil + 0) total)) - (some-> *team-stats* (swap! update :processed/graphics (fnil + 0) total)) - (some-> *file-stats* (swap! assoc :processed/graphics total))) + (some-> *stats* (swap! update :processed-graphics (fnil + 0) total)) + (some-> *team-stats* (swap! update :processed-graphics (fnil + 0) total)) + (some-> *file-stats* (swap! assoc :processed-graphics total))) (loop [groups (seq groups) fdata fdata @@ -1236,10 +1232,8 @@ (cfv/validate-file-schema! file)) (defn- process-file - [{:keys [::db/conn] :as system} id & {:keys [validate?]}] - (let [file (get-file system id) - - libs (->> (files/get-file-libraries conn id) + [{:keys [::db/conn] :as system} {:keys [id] :as file} & {:keys [validate?]}] + (let [libs (->> (files/get-file-libraries conn id) (into [file] (comp (map :id) (map (partial get-file system)))) (d/index-by :id)) @@ -1314,7 +1308,13 @@ (when (string? label) (fsnap/take-file-snapshot! system {:file-id file-id :label (str "migration/" label)})) - (process-file system file-id :validate? validate?)) + (let [file (get-file system file-id)] + (events/tap :progress + {:op :migrate-file + :name (:name file) + :id (:id file)}) + + (process-file system file :validate? validate?))) (catch Throwable cause (let [team-id *team-id*] @@ -1325,8 +1325,8 @@ (finally (let [elapsed (tpoint) - components (get @*file-stats* :processed/components 0) - graphics (get @*file-stats* :processed/graphics 0)] + components (get @*file-stats* :processed-components 0) + graphics (get @*file-stats* :processed-graphics 0)] (l/dbg :hint "migrate:file:end" :file-id (str file-id) @@ -1335,8 +1335,8 @@ :validate validate? :elapsed (dt/format-duration elapsed)) - (some-> *stats* (swap! update :processed/files (fnil inc 0))) - (some-> *team-stats* (swap! update :processed/files (fnil inc 0))))))))) + (some-> *stats* (swap! update :processed-files (fnil inc 0))) + (some-> *team-stats* (swap! update :processed-files (fnil inc 0))))))))) (defn migrate-team! [system team-id & {:keys [validate? skip-on-graphic-error? label]}] @@ -1355,7 +1355,7 @@ :skip-on-graphic-error? skip-on-graphic-error?)) migrate-team (fn [{:keys [::db/conn] :as system} team-id] - (let [{:keys [id features]} (get-team system team-id)] + (let [{:keys [id features name]} (get-team system team-id)] (if (contains? features "components/v2") (l/inf :hint "team already migrated") (let [features (-> features @@ -1364,6 +1364,11 @@ (conj "layout/grid") (conj "styles/v2"))] + (events/tap :progress + {:op :migrate-team + :name name + :id id}) + (run! (partial migrate-file system) (get-and-lock-files conn id)) @@ -1380,11 +1385,12 @@ (finally (let [elapsed (tpoint) - components (get @*team-stats* :processed/components 0) - graphics (get @*team-stats* :processed/graphics 0) - files (get @*team-stats* :processed/files 0)] + components (get @*team-stats* :processed-components 0) + graphics (get @*team-stats* :processed-graphics 0) + files (get @*team-stats* :processed-files 0)] - (some-> *stats* (swap! update :processed/teams (fnil inc 0))) + (when-not @err + (some-> *stats* (swap! update :processed-teams (fnil inc 0)))) (if (cache/cache? *cache*) (let [cache-stats (cache/stats *cache*)] diff --git a/backend/src/app/http/sse.clj b/backend/src/app/http/sse.clj index 0ece3b3295..ec80df72d9 100644 --- a/backend/src/app/http/sse.clj +++ b/backend/src/app/http/sse.clj @@ -9,11 +9,10 @@ (:refer-clojure :exclude [tap]) (:require [app.common.data :as d] - [app.common.exceptions :as ex] [app.common.logging :as l] [app.common.transit :as t] [app.http.errors :as errors] - [promesa.core :as p] + [app.util.events :as events] [promesa.exec :as px] [promesa.exec.csp :as sp] [promesa.util :as pu] @@ -21,26 +20,12 @@ (:import java.io.OutputStream)) -(def ^:dynamic *channel* nil) - (defn- write! - [^OutputStream output ^bytes data] + [^OutputStream output ^bytes data] (l/trc :hint "writting data" :data data :length (alength data)) (.write output data) (.flush output)) -(defn- create-writer-loop - [^OutputStream output] - (try - (loop [] - (when-let [event (sp/take! *channel*)] - (let [result (ex/try! (write! output event))] - (if (ex/exception? result) - (l/wrn :hint "unexpected exception on sse writer" :cause result) - (recur))))) - (finally - (pu/close! output)))) - (defn- encode [[name data]] (try @@ -61,13 +46,6 @@ "Cache-Control" "no-cache, no-store, max-age=0, must-revalidate" "Pragma" "no-cache"}) -(defn tap - ([data] (tap "event" data)) - ([name data] - (when-let [channel *channel*] - (sp/put! channel [name data]) - nil))) - (defn response [handler & {:keys [buf] :or {buf 32} :as opts}] (fn [request] @@ -75,15 +53,18 @@ ::rres/status 200 ::rres/body (reify rres/StreamableResponseBody (-write-body-to-stream [_ _ output] - (binding [*channel* (sp/chan :buf buf :xf (keep encode))] - (let [writer (px/run! :virtual (partial create-writer-loop output))] + (binding [events/*channel* (sp/chan :buf buf :xf (keep encode))] + (let [listener (events/start-listener + (partial write! output) + (partial pu/close! output))] (try - (tap "end" (handler)) + (let [result (handler)] + (events/tap :end result)) (catch Throwable cause (binding [l/*context* (errors/request->context request)] (l/err :hint "unexpected error process streaming response" :cause cause)) - (tap "error" (errors/handle' cause request))) + (events/tap :error (errors/handle' cause request))) (finally - (sp/close! *channel*) - (p/await! writer)))))))})) + (sp/close! events/*channel*) + (px/await! listener)))))))})) diff --git a/backend/src/app/srepl/cli.clj b/backend/src/app/srepl/cli.clj index 9b4943bdcf..6bcca5c0c8 100644 --- a/backend/src/app/srepl/cli.clj +++ b/backend/src/app/srepl/cli.clj @@ -13,7 +13,8 @@ [app.db :as db] [app.main :as main] [app.rpc.commands.auth :as cmd.auth] - [app.srepl.components-v2] + [app.srepl.components-v2 :refer [migrate-teams!]] + [app.util.events :as events] [app.util.json :as json] [app.util.time :as dt] [cuerdas.core :as str])) @@ -106,25 +107,36 @@ (defmethod exec-command :migrate-v2 [_] - (letfn [(on-start [{:keys [total rollback]}] - (println - (str/ffmt "The components/v2 migration started (rollback:%, teams:%)" - (if rollback "on" "off") - total))) + (letfn [(on-progress-report [{:keys [elapsed completed errors]}] + (println (str/ffmt "-> Progress: completed: %, errors: %, elapsed: %" + completed errors elapsed))) + + (on-progress [{:keys [op name]}] + (case op + :migrate-team + (println (str/ffmt "-> Migrating team: \"%\"" name)) + :migrate-file + (println (str/ffmt "=> Migrating file: \"%\"" name)) + nil)) + + (on-event [[type payload]] + (case type + :progress-report (on-progress-report payload) + :progress (on-progress payload) + :error (on-error payload) + nil)) - (on-progress [{:keys [total elapsed progress completed]}] - (println (str/ffmt "Progress % (total: %, completed: %, elapsed: %)" - progress total completed elapsed))) (on-error [cause] - (println "ERR:" (ex-message cause))) + (println "EE:" (ex-message cause)))] - (on-end [_] - (println "Migration finished"))] - (app.srepl.components-v2/migrate-teams! main/system - :on-start on-start - :on-error on-error - :on-progress on-progress - :on-end on-end))) + (println "The components/v2 migration started...") + + (try + (let [result (-> (partial migrate-teams! main/system {:rollback? true}) + (events/run-with! on-event))] + (println (str/ffmt "Migration process finished (elapsed: %)" (:elapsed result)))) + (catch Throwable cause + (on-error cause))))) (defmethod exec-command :default [{:keys [::cmd]}] diff --git a/backend/src/app/srepl/components_v2.clj b/backend/src/app/srepl/components_v2.clj index 5e6a697bb7..3db7746001 100644 --- a/backend/src/app/srepl/components_v2.clj +++ b/backend/src/app/srepl/components_v2.clj @@ -8,13 +8,13 @@ (:require [app.common.data :as d] [app.common.logging :as l] - [app.common.pprint :as pp] [app.common.uuid :as uuid] [app.db :as db] [app.features.components-v2 :as feat] [app.main :as main] [app.svgo :as svgo] [app.util.cache :as cache] + [app.util.events :as events] [app.util.time :as dt] [app.worker :as-alias wrk] [cuerdas.core :as str] @@ -29,32 +29,30 @@ ;; PRIVATE HELPERS ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(defn- print-stats! - [stats] - (->> stats - (into (sorted-map)) - (pp/pprint))) - (defn- report-progress-files [tpoint] (fn [_ _ oldv newv] - (when (not= (:processed/files oldv) - (:processed/files newv)) + (when (not= (:processed-files oldv) + (:processed-files newv)) (let [elapsed (tpoint)] (l/dbg :hint "progress" - :completed (:processed/files newv) + :completed (:processed-files newv) :elapsed (dt/format-duration elapsed)))))) (defn- report-progress-teams - [tpoint on-progress] + [tpoint] (fn [_ _ oldv newv] - (when (not= (:processed/teams oldv) - (:processed/teams newv)) - (let [completed (:processed/teams newv) + (when (or (not= (:processed-teams oldv) + (:processed-teams newv)) + (not= (:errors oldv) + (:errors newv))) + (let [completed (:processed-teams newv 0) + errors (:errors newv 0) elapsed (dt/format-duration (tpoint))] - (when (fn? on-progress) - (on-progress {:elapsed elapsed - :completed completed})) + (events/tap :progress-report + {:elapsed elapsed + :completed completed + :errors errors}) (l/dbg :hint "progress" :completed completed :elapsed elapsed))))) @@ -235,10 +233,10 @@ (feat/migrate-team! team-id :label label :validate? validate? - :skip-on-graphic-error? skip-on-graphic-error?)) - (print-stats! - (-> (deref feat/*stats*) - (assoc :elapsed (dt/format-duration (tpoint))))) + :skip-on-graphics-error? skip-on-graphic-error?)) + + (-> (deref feat/*stats*) + (assoc :elapsed (dt/format-duration (tpoint)))) (catch Throwable cause (l/dbg :hint "migrate:error" :cause cause)) @@ -261,8 +259,8 @@ a correct `:label`. That label is also used for persist a file snaphot before continue with the migration." [& {:keys [max-jobs max-items max-time rollback? validate? query - pred max-procs cache on-start on-progress on-error on-end - skip-on-graphic-error? label partitions current-partition] + pred max-procs cache skip-on-graphic-error? + label partitions current-partition] :or {validate? false rollback? true max-jobs 1 @@ -310,6 +308,14 @@ (l/wrn :hint "unexpected error on processing team (skiping)" :team-id (str team-id) :cause cause) + + (events/tap :error + (ex-info "unexpected error on processing team (skiping)" + {:team-id team-id} + cause)) + + (swap! stats update :errors (fnil inc 0)) + (when (string? label) (report! main/system team-id label (tpoint) (ex-message cause)))) @@ -336,15 +342,12 @@ :max-jobs max-jobs :max-items max-items) - (add-watch stats :progress-report (report-progress-teams tpoint on-progress)) + (add-watch stats :progress-report (report-progress-teams tpoint)) (binding [feat/*stats* stats feat/*cache* cache svgo/*semaphore* sprocs] (try - (when (fn? on-start) - (on-start {:rollback rollback?})) - (when (string? label) (create-report-table! main/system) (clean-reports! main/system label)) @@ -367,20 +370,12 @@ ;; Close and await tasks (pu/close! executor))) - (if (fn? on-end) - (-> (deref stats) - (assoc :elapsed/total (tpoint)) - (on-end)) - (-> (deref stats) - (assoc :elapsed/total (tpoint)) - (update :elapsed/total dt/format-duration) - (dissoc :total/teams) - (print-stats!))) + (-> (deref stats) + (assoc :elapsed (dt/format-duration (tpoint)))) (catch Throwable cause (l/dbg :hint "migrate:error" :cause cause) - (when (fn? on-error) - (on-error cause))) + (events/tap :error cause)) (finally (let [elapsed (dt/format-duration (tpoint))] diff --git a/backend/src/app/util/events.clj b/backend/src/app/util/events.clj new file mode 100644 index 0000000000..a41843c6b1 --- /dev/null +++ b/backend/src/app/util/events.clj @@ -0,0 +1,64 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC + +(ns app.util.events + "A generic asynchronous events notifications subsystem; used mainly + for mark event points in functions and be able to attach listeners + to them. Mainly used in http.sse for progress reporting." + (:refer-clojure :exclude [tap run!]) + (:require + [app.common.data.macros :as dm] + [app.common.exceptions :as ex] + [app.common.logging :as l] + [promesa.exec :as px] + [promesa.exec.csp :as sp])) + +(def ^:dynamic *channel* nil) + +(defn channel + [] + (sp/chan :buf 32)) + +(defn tap + [type data] + (when-let [channel *channel*] + (sp/put! channel [type data]) + nil)) + +(defn start-listener + [on-event on-close] + + (dm/assert! + "expected active events channel" + (sp/chan? *channel*)) + + (px/thread + {:virtual true} + (try + (loop [] + (when-let [event (sp/take! *channel*)] + (let [result (ex/try! (on-event event))] + (if (ex/exception? result) + (do + (l/wrn :hint "unexpected exception" :cause result) + (sp/close! *channel*)) + (recur))))) + (finally + (on-close))))) + +(defn run-with! + "A high-level facility for to run a function in context of event + emiter." + [f on-event] + + (binding [*channel* (sp/chan :buf 32)] + (let [listener (start-listener on-event (constantly nil))] + (try + (f) + (finally + (sp/close! *channel*) + (px/await! listener)))))) + diff --git a/backend/test/backend_tests/rpc_management_test.clj b/backend/test/backend_tests/rpc_management_test.clj index f9b62d1b02..3258314803 100644 --- a/backend/test/backend_tests/rpc_management_test.clj +++ b/backend/test/backend_tests/rpc_management_test.clj @@ -612,7 +612,7 @@ (t/is (fn? result)) (let [events (th/consume-sse result)] - (t/is (= 8 (count events))) + (t/is (= 6 (count events))) (t/is (= :end (first (last events)))))))) (t/deftest get-list-of-buitin-templates diff --git a/frontend/src/app/main/data/dashboard.cljs b/frontend/src/app/main/data/dashboard.cljs index c782b49fae..0511e335d6 100644 --- a/frontend/src/app/main/data/dashboard.cljs +++ b/frontend/src/app/main/data/dashboard.cljs @@ -1013,7 +1013,7 @@ (rx/tap (fn [event] (let [payload (sse/get-payload event) type (sse/get-type event)] - (if (= type "event") + (if (= type "progress") (log/dbg :hint "clone-template: progress" :section (:section payload) :name (:name payload)) (log/dbg :hint "clone-template: end"))))) diff --git a/frontend/src/app/worker/import.cljs b/frontend/src/app/worker/import.cljs index daea0c2419..f43df1cf24 100644 --- a/frontend/src/app/worker/import.cljs +++ b/frontend/src/app/worker/import.cljs @@ -735,7 +735,7 @@ (rx/tap (fn [event] (let [payload (sse/get-payload event) type (sse/get-type event)] - (if (= type "event") + (if (= type "progress") (log/dbg :hint "import-binfile: progress" :section (:section payload) :name (:name payload)) (log/dbg :hint "import-binfile: end"))))) (rx/filter sse/end-of-stream?) From 208b06d9cb8e560dfcb103a4127c96f308022b1d Mon Sep 17 00:00:00 2001 From: Alejandro Alonso Date: Tue, 30 Jan 2024 16:36:21 +0100 Subject: [PATCH 5/7] :sparkle: Allow select text on debug shape info panel --- frontend/src/app/main/ui/workspace/sidebar/debug_shape_info.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/app/main/ui/workspace/sidebar/debug_shape_info.scss b/frontend/src/app/main/ui/workspace/sidebar/debug_shape_info.scss index 59c2e3fc60..827c10938f 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/debug_shape_info.scss +++ b/frontend/src/app/main/ui/workspace/sidebar/debug_shape_info.scss @@ -12,6 +12,7 @@ background-color: var(--panel-background-color); color: white; font-size: $fs-12; + user-select: text; } .shape-info-title { From 8f004c0c75b223100b86c5858058a07510bfb40f Mon Sep 17 00:00:00 2001 From: Eva Date: Tue, 30 Jan 2024 16:47:34 +0100 Subject: [PATCH 6/7] :recycle: Change shortcut for change theme --- frontend/src/app/main/data/dashboard/shortcuts.cljs | 4 ++-- frontend/src/app/main/data/workspace/shortcuts.cljs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/main/data/dashboard/shortcuts.cljs b/frontend/src/app/main/data/dashboard/shortcuts.cljs index 3d27669b6d..0a2e8d3e96 100644 --- a/frontend/src/app/main/data/dashboard/shortcuts.cljs +++ b/frontend/src/app/main/data/dashboard/shortcuts.cljs @@ -32,8 +32,8 @@ :subsections [:general-dashboard] :fn #(st/emit! (dd/create-element))} - :toggle-theme {:tooltip (ds/meta (ds/alt "M")) - :command (ds/c-mod "alt+m") + :toggle-theme {:tooltip (ds/alt "M") + :command (ds/a-mod "m") :subsections [:general-dashboard] :fn #(st/emit! (du/toggle-theme))}}) diff --git a/frontend/src/app/main/data/workspace/shortcuts.cljs b/frontend/src/app/main/data/workspace/shortcuts.cljs index b5d91faa3a..28e2be350d 100644 --- a/frontend/src/app/main/data/workspace/shortcuts.cljs +++ b/frontend/src/app/main/data/workspace/shortcuts.cljs @@ -551,8 +551,8 @@ ;; THEME - :toggle-theme {:tooltip (ds/meta (ds/alt "M")) - :command (ds/c-mod "alt+m") + :toggle-theme {:tooltip (ds/alt "M") + :command (ds/a-mod "m") :subsections [:basics] :fn #(st/emit! (du/toggle-theme))}}) From 891dab7f06697a150975e0f5a413542b3b25a0c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Moya?= Date: Tue, 30 Jan 2024 16:15:54 +0100 Subject: [PATCH 7/7] :wrench: Improve debug tool --- .../workspace/sidebar/debug_shape_info.cljs | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/main/ui/workspace/sidebar/debug_shape_info.cljs b/frontend/src/app/main/ui/workspace/sidebar/debug_shape_info.cljs index 1e1c8c55b1..1213477f38 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/debug_shape_info.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/debug_shape_info.cljs @@ -17,12 +17,42 @@ [debug :as dbg] [rumext.v2 :as mf])) +(def display-attrs + [:type + :id + :parent-id + :frame-id + :shapes + :component-id + :component-file + :component-root + :main-instance + :shape-ref + :x + :y + :width + :height + :selrect + :points + :transform + :transform-inverse]) + (def remove-attrs - #{:id :name}) + #{:name, :remote-synced}) (def vertical-layout-attrs #{}) +(defn get-attrs + [shape] + (let [shape-attrs (->> (keys shape) + (remove (set display-attrs)) + (remove remove-attrs) + (sort-by name))] + (as-> display-attrs $ + (d/removev #(nil? (get shape %)) $) + (into $ shape-attrs)))) + (def custom-renderer {:parent-id :shape-link :frame-id :shape-link @@ -87,10 +117,7 @@ [:button {:on-click #(debug/dump-subtree (dm/str (:id current)) true)} "tree"]] [:div {:class (stl/css :shape-attrs)} - (let [attrs (->> (keys current) - (remove remove-attrs)) - attrs (concat [:frame-id :parent-id :shapes] - (->> attrs (remove #{:frame-id :parent-id :shapes})))] + (let [attrs (get-attrs current)] (for [attr attrs] (when-let [value (get current attr)] [:div {:class (stl/css-case :attrs-container-attr true