diff --git a/backend/deps.edn b/backend/deps.edn index 87aac35b6f..32bab17706 100644 --- a/backend/deps.edn +++ b/backend/deps.edn @@ -69,7 +69,10 @@ :paths ["src" "resources" "target/classes"] :aliases {:dev - {:extra-deps + { + :jvm-opts ["--sun-misc-unsafe-memory-access=allow" + "--enable-native-access=ALL-UNNAMED"] + :extra-deps {com.bhauman/rebel-readline {:mvn/version "0.1.7"} clojure-humanize/clojure-humanize {:mvn/version "0.2.2"} org.clojure/data.csv {:mvn/version "1.1.1"} @@ -84,9 +87,7 @@ :test {:main-opts ["-m" "kaocha.runner"] - :jvm-opts ["-Dlog4j2.configurationFile=log4j2-devenv-repl.xml" - "--sun-misc-unsafe-memory-access=allow" - "--enable-native-access=ALL-UNNAMED"] + :jvm-opts ["-Dlog4j2.configurationFile=log4j2-devenv-repl.xml"] :extra-deps {lambdaisland/kaocha {:mvn/version "1.91.1392"}}} :outdated diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index 402dc30750..76fafece3b 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -922,4 +922,4 @@ "Return all shared files in `team-id` whose slugified name equals `slug`." [cfg team-id slug] (->> (get-shared-files-for-team cfg team-id) - (filterv #(= slug (slugify-name (:name %)))))) + (filter #(= slug (slugify-name (:name %)))))) diff --git a/backend/src/app/binfile/v3.clj b/backend/src/app/binfile/v3.clj index c36ed30263..ac96860227 100644 --- a/backend/src/app/binfile/v3.clj +++ b/backend/src/app/binfile/v3.clj @@ -380,6 +380,7 @@ (defn- export-files [{:keys [::bfc/ids ::bfc/include-libraries ::output] :as cfg}] + (let [original-ids ids ids (into ids (when include-libraries (bfc/get-libraries cfg ids))) rels (if include-libraries @@ -912,51 +913,58 @@ (vswap! bfc/*state* update :index assoc id (:id sobject))))))) -(defn- resolve-external-libraries +(defn- resolve-external-libraries-references "For each external library in the manifest, look for matching shared files in the team by slugified name. Returns a map of old-lib-id -> [{:id uuid :name string}] for libraries with matches." [{:keys [::manifest ::bfc/team-id] :as cfg}] - (let [external-libs (:external-libraries manifest)] - (when (and team-id (seq external-libs)) - (reduce (fn [result {:keys [id slug]}] - (let [candidates (bfc/find-shared-files-by-slug cfg team-id slug)] - (if (seq candidates) - (assoc result id candidates) - result))) - {} - external-libs)))) + (assert (uuid? team-id) "team-id should be provided") + (reduce (fn [result {:keys [id slug] :as lib}] + (if-let [candidates (-> (bfc/find-shared-files-by-slug cfg team-id slug) + (vec) + (not-empty))] + (assoc result id (assoc lib :candidates candidates)) + result)) + {} + (:external-libraries manifest))) (defn- auto-link-libraries "Auto-link imported files to libraries that have exactly one candidate - match. Returns a vector of {:id old-lib-id :name string :new-id uuid} + match. Returns a vector of {:id old-lib-id :name string :linked-to uuid} for each auto-linked library." [{:keys [::db/conn ::manifest ::bfc/timestamp ::bfc/profile-id] :as cfg} resolution file-ids] - (let [external-libs (:external-libraries manifest)] - (reduce (fn [linked {:keys [id name used-by] :as ext-lib}] - (let [candidates (get resolution id)] - (if (= 1 (count candidates)) - (let [new-lib-id (:id (first candidates)) - perms (bfc/get-file-permissions conn profile-id new-lib-id)] - ;; Only auto-link when the importer has edit permission - ;; on the matched library, matching the manual link RPC. - (if (:can-edit perms) - (let [;; Link only files that actually used this library - relevant-file-ids (if (seq used-by) - (let [used-set (set (map bfc/lookup-index used-by))] - (filterv used-set file-ids)) - file-ids)] - (doseq [fid relevant-file-ids] - (let [rel-params {:file-id fid - :library-file-id new-lib-id}] - (db/insert! conn :file-library-rel rel-params - ::db/on-conflict-do-nothing? true) - (bfc/upsert-file-library-sync! conn (assoc rel-params :synced-at timestamp)))) - (conj linked {:id id :name name :new-id new-lib-id})) - linked)) - linked))) - [] - external-libs))) + (reduce-kv (fn [result id {:keys [name used-by candidates]}] + (if (= 1 (count candidates)) + (let [candidate (first candidates) + lib-id (get candidate :id) + perms (bfc/get-file-permissions conn profile-id lib-id)] + + ;; Only auto-link when the importer has edit permission + ;; on the matched library, matching the manual link RPC. + (if (:can-edit perms) + ;; Link only files that actually used this library + (let [used-by (into #{} (map bfc/lookup-index) used-by) + file-ids (-> (filter #(contains? used-by %) file-ids) + (not-empty))] + + (doseq [file-id file-ids] + (when (contains? used-by file-id) + (let [rel-params {:file-id file-id + :library-file-id lib-id}] + (db/insert! conn :file-library-rel rel-params + {::db/on-conflict-do-nothing? true}) + (bfc/upsert-file-library-sync! conn (assoc rel-params :synced-at timestamp))))) + + (if file-ids + (update result id (fn [lib] + (-> lib + (assoc :linked-to lib-id) + (dissoc :candidates)))) + (dissoc result id))) + result)) + result)) + resolution + resolution)) (defn- import-files* [{:keys [::manifest] :as cfg}] @@ -966,40 +974,28 @@ (import-storage-objects cfg) - (let [files (get manifest :files) - result (reduce (fn [result file] - (let [name' (get file :name) - file (assoc file :name name')] - (conj result (import-file cfg file)))) - [] - files)] + (let [files (get manifest :files) + file-ids (reduce (fn [result file] + (let [name' (get file :name) + file (assoc file :name name')] + (conj result (import-file cfg file)))) + [] + files)] (import-file-relations cfg) ;; Resolve external libraries by slug and auto-link single matches - (let [resolution (resolve-external-libraries cfg) - auto-linked (when (seq resolution) - (auto-link-libraries cfg resolution result)) + (let [resolution + (resolve-external-libraries-references cfg) - ;; Collect multi-match candidates for frontend resolution - candidates (when (seq resolution) - (into {} - (filter (fn [[_ candidates]] (> (count candidates) 1))) - resolution)) + resolution + (auto-link-libraries cfg resolution file-ids)] - ;; Build external-libraries lookup from manifest for the frontend - external-libs-info (when (seq resolution) - (->> (:external-libraries manifest) - (filter #(contains? candidates (:id %))) - (mapv (fn [{:keys [id name slug]}] - {:id id :name name :slug slug}))))] + (app.common.pprint/pprint resolution) (bfm/apply-pending-migrations! cfg) - - {:file-ids result - :auto-linked (or auto-linked []) - :library-candidates (or candidates {}) - :external-libs (or external-libs-info [])}))) + {:file-ids file-ids + :resolution resolution}))) (defn- import-file-and-overwrite* [{:keys [::manifest ::bfc/file-id] :as cfg}] @@ -1027,10 +1023,8 @@ (bfc/invalidate-thumbnails cfg file-id) (bfm/apply-pending-migrations! cfg) - {:file-ids [file-id] - :auto-linked [] - :library-candidates {} - :external-libs []}))) + {:file-ids [file-id] + :resolution {}}))) (defn- import-files [{:keys [::bfc/timestamp ::bfc/input] :or {timestamp (ct/now)} :as cfg}] @@ -1068,20 +1062,9 @@ (events/tap :progress {:section :manifest}) (binding [bfc/*state* (volatile! {:media [] :index {}})] - (let [result (if (::bfc/file-id cfg) - (db/tx-run! cfg import-file-and-overwrite*) - (db/tx-run! cfg import-files*))] - - ;; Emit library-candidates event for frontend resolution of - ;; multi-match cases (if any) - (when (seq (:library-candidates result)) - (events/tap :library-candidates - {:file-ids (:file-ids result) - :auto-linked (:auto-linked result) - :library-candidates (:library-candidates result) - :external-libs (:external-libs result)})) - - result)))) + (if (::bfc/file-id cfg) + (db/tx-run! cfg import-file-and-overwrite*) + (db/tx-run! cfg import-files*))))) ;; --- PUBLIC API @@ -1110,6 +1093,7 @@ tp (ct/tpoint) ab (volatile! false) cs (volatile! nil)] + (try (l/info :hint "start exportation" :export-id (str id)) (binding [bfc/*state* (volatile! (bfc/initial-state))] diff --git a/frontend/src/app/main/ui/dashboard/import.cljs b/frontend/src/app/main/ui/dashboard/import.cljs index a079a8a2b9..d82c3d2626 100644 --- a/frontend/src/app/main/ui/dashboard/import.cljs +++ b/frontend/src/app/main/ui/dashboard/import.cljs @@ -185,27 +185,27 @@ :project-id project-id :files entries :features features}) - (rx/filter (comp uuid? :file-id)) + (rx/filter some?) (rx/subs! (fn [message] ;; Capture library-resolution data if present (same for all ;; entries from the same zip, so first one wins) - (when-let [resolution (:library-resolution message)] - (when (nil? @library-resolution*) - (reset! library-resolution* resolution))) - (swap! state update-entry-status message)))))) + (if-let [resolution (-> (:libraries-resolution message) + (not-empty))] + (reset! library-resolution* resolution) + (swap! state update-entry-status message))))))) (mf/defc import-entry* {::mf/memo true ::mf/private true} - [{:keys [entries entry edition can-be-deleted importing? on-edit on-change on-delete]}] + [{:keys [entries entry edition can-be-deleted is-progress on-edit on-change on-delete]}] (let [status (:status entry) ;; FIXME: rename to format format (:type entry) loading? (or (= :analyze status) (= :import-progress status) - (and importing? (= :import-ready status))) + (and is-progress (= :import-ready status))) analyze-error? (= :analyze-error status) import-success? (= :import-success status) import-error? (= :import-error status) @@ -404,10 +404,11 @@ ;; Library resolution data from the backend (auto-linked + multi-match) library-resolution* (mf/use-state nil) - library-resolution (deref library-resolution*) + library-resolution (not-empty (deref library-resolution*)) ;; User selections for multi-match candidates: {old-lib-id candidate-id} library-selections* (mf/use-state {}) + library-selections (deref library-selections*) continue-entries (mf/use-fn @@ -520,6 +521,7 @@ pending-analysis? (some has-status-analyze? entries) + ;; TODO: check auto-linked-count (if (some? library-resolution) (count (:auto-linked library-resolution)) @@ -536,12 +538,7 @@ (and (seq entries) (every? #(= :import-success (:status %)) entries)) - ;; After all entries are imported, check if there are multi-match - ;; candidates that need user resolution - (if (and (some? @library-resolution*) - (seq (:library-candidates @library-resolution*))) - (reset! status* :library-resolution) - (reset! status* :import-success)) + (reset! status* :import-success) (and (seq entries) (and (every? #(not= :import-ready (:status %)) entries) @@ -584,7 +581,7 @@ :class (stl/css :context-notification-error) :content (tr "dashboard.import.import-error.disclaimer")}]) - (when (= :library-resolution status) + (when (some? library-resolution) [:* [:& context-notification {:level :success @@ -623,6 +620,7 @@ :else (tr "dashboard.import.import-error.unknown-error"))])]))] + [:div (tr "dashboard.import.import-error.message2")]] (when-not (= :library-resolution status) @@ -631,7 +629,7 @@ :key (dm/str (:uri entry) "/" (:file-id entry)) :entry entry :entries entries - :importing? (= :import-progress status) + :is-progress (= :import-progress status) :on-edit on-edit :on-change on-entry-change :on-delete on-entry-delete @@ -649,28 +647,29 @@ [:div {:class (stl/css :modal-footer)} [:div {:class (stl/css :action-buttons)} - (when (= :analyze status) + (cond + (some? library-resolution) + [:input {:class (stl/css :accept-btn) + :type "button" + :value (tr "dashboard.import.confirm-library-links") + :on-click on-confirm-library-links}] + + (= :analyze status) [:input {:class (stl/css :cancel-button) :type "button" :value (tr "labels.cancel") - :on-click on-cancel}]) + :on-click on-cancel}] - (when (= status :import-ready) + (= status :import-ready) [:input {:class (stl/css :accept-btn) :type "button" :value (tr "labels.continue") :disabled pending-analysis? - :on-click on-continue}]) + :on-click on-continue}] - (when (= :library-resolution status) - [:input {:class (stl/css :accept-btn) - :type "button" - :value (tr "dashboard.import.confirm-library-links") - :on-click on-confirm-library-links}]) - - (when (or (= :import-success status) - (= :import-error status) - (= :import-progress status)) + (or (= :import-success status) + (= :import-error status) + (= :import-progress status)) [:input {:class (stl/css :accept-btn) :type "button" :value (tr "labels.accept") diff --git a/frontend/src/app/worker/import.cljs b/frontend/src/app/worker/import.cljs index 0edcefa474..d78fd70ba1 100644 --- a/frontend/src/app/worker/import.cljs +++ b/frontend/src/app/worker/import.cljs @@ -165,8 +165,9 @@ (defmethod impl/handler :import-files [{:keys [project-id files]}] - (let [binfile-v1 (filter #(= :binfile-v1 (:type %)) files) - binfile-v3 (filter #(= :binfile-v3 (:type %)) files)] + (let [binfile-v1 (filter #(= :binfile-v1 (:type %)) files) + binfile-v3 (filter #(= :binfile-v3 (:type %)) files) + resolutions (volatile! {})] (rx/merge (->> (rx/from binfile-v1) @@ -197,50 +198,50 @@ :error (import-cause-message cause (tr "labels.error")) :file-id (:file-id data)}))))))) - (->> (rx/from binfile-v3) - (rx/reduce (fn [result file] - (update result (:uri file) (fnil conj []) file)) - {}) - (rx/mapcat identity) - (rx/merge-map - (fn [[uri entries]] - (let [library-resolution* (volatile! nil)] - (->> (import-blob-via-upload uri - {:name (-> entries first :name) - :version 3 - :project-id project-id}) - (rx/tap (fn [event] - (let [payload (sse/get-payload event) - type (sse/get-type event)] - (cond - (= type "progress") - (log/dbg :hint "import-binfile: progress" - :section (:section payload) - :name (:name payload)) - (= type "library-candidates") - (vreset! library-resolution* payload) + (rx/concat + (->> (rx/from binfile-v3) + (rx/reduce (fn [result file] + (update result (:uri file) (fnil conj []) file)) + {}) + (rx/mapcat identity) + (rx/merge-map + (fn [[uri entries]] + (->> (import-blob-via-upload uri + {:name (-> entries first :name) + :version 3 + :project-id project-id}) + (rx/tap (fn [event] + (let [payload (sse/get-payload event) + type (sse/get-type event)] + (cond + (= type "progress") + (log/dbg :hint "import-binfile: progress" + :section (:section payload) + :name (:name payload)) - :else - (log/dbg :hint "import-binfile: end"))))) - (rx/filter sse/end-of-stream?) - (rx/mapcat (fn [_] - (let [resolution @library-resolution*] - (->> (rx/from entries) - (rx/map (fn [entry] - (cond-> {:status :finish - :file-id (:file-id entry)} - (some? resolution) - (assoc :library-resolution resolution)))))))) - (rx/catch - (fn [cause] - (log/error :hint "import-binfile: unexpected error on importing" - :project-id project-id - ::log/sync? true - :cause cause) - (let [err (import-cause-message cause (tr "labels.error"))] - (->> (rx/from entries) - (rx/map (fn [entry] - {:status :error - :error err - :file-id (:file-id entry)})))))))))))))) + :else + (log/dbg :hint "import-binfile: end"))))) + (rx/filter sse/end-of-stream?) + (rx/mapcat (fn [message] + (let [{:keys [file-ids resolution]} (sse/get-payload message)] + (some->> (not-empty resolution) (vswap! resolutions merge)) + (->> (rx/from entries) + (rx/map (fn [entry] + {:status :finish + :file-id (:file-id entry)})))))) + (rx/catch (fn [cause] + (log/error :hint "import-binfile: unexpected error on importing" + :project-id project-id + ::log/sync? true + :cause cause) + (let [err (import-cause-message cause (tr "labels.error"))] + (->> (rx/from entries) + (rx/map (fn [entry] + {:status :error + :error err + :file-id (:file-id entry)})))))))))) + (->> (rx/defer #(deref resolutions)) + (rx/map (fn [resolutions] + {:type :libraries-resolution + :value resolutions}))))))) diff --git a/package.json b/package.json index 373949e3e7..abf05e593e 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "license": "MPL-2.0", "author": "Kaleidos INC Sucursal en EspaƱa SL", "private": true, - "packageManager": "pnpm@11.7.0+sha512.19cc852c120c7125760f2443ee6be0ca5b40f9f50598de1a09a1f177503e010e57c23c77646e01e761de59bf874fb22a3398c33ab9691fc13eb946b6f0f4d620", + "packageManager": "pnpm@11.9.0+sha512.bd682d5d03fe525ef7c9fd6780c6884d1e756ac4c9c9fe00c538782824310dcf90e3ddc4f53835f06dfaebd5085e41855e0bcbb3b60de2ac5bbab89e5036f03b", "repository": { "type": "git", "url": "https://github.com/penpot/penpot"