diff --git a/backend/src/app/features/fdata.clj b/backend/src/app/features/fdata.clj index 3ec5fdfb57..baa63f6936 100644 --- a/backend/src/app/features/fdata.clj +++ b/backend/src/app/features/fdata.clj @@ -62,6 +62,12 @@ {:id id :file-id file-id} {::sql/columns [:content] ::db/check-deleted false})] + + (l/trc :hint "load pointer" + :file-id (str file-id) + :id (str id) + :found (some? content)) + (when-not content (ex/raise :type :internal :code :fragment-not-found diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index 010a0aa7a3..38ea61dd89 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -35,15 +35,18 @@ (defn get-file "Get the migrated data of one file." - ([id] (get-file (or *system* main/system) id)) - ([system id] + ([id] (get-file (or *system* main/system) id nil)) + ([system id & {:keys [raw?] :as opts}] (db/run! system (fn [system] - (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)] - (-> (files/get-file system id :migrate? false) - (update :data feat.fdata/process-pointers deref) - (update :data feat.fdata/process-objects (partial into {})) - (fmg/migrate-file))))))) + (let [file (files/get-file system id :migrate? false)] + (if raw? + file + (binding [pmap/*load-fn* (partial feat.fdata/load-pointer system id)] + (-> file + (update :data feat.fdata/process-pointers deref) + (update :data feat.fdata/process-objects (partial into {})) + (fmg/migrate-file))))))))) (defn update-file! [system {:keys [id] :as file}] @@ -166,7 +169,7 @@ (fsnap/take-file-snapshot! system {:file-id file-id :label label})) (let [conn (db/get-connection system) - file (get-file system file-id) + file (get-file system file-id opts) libs (when with-libraries? (->> (files/get-file-libraries conn file-id) (into [file] (map (fn [{:keys [id]}] diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index 17df79a9a7..2f538d6f66 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -429,7 +429,9 @@ (try (l/trc :hint "process:file:start" :file-id (str file-id) :index idx) (let [system (assoc main/system ::db/rollback rollback?)] - (db/tx-run! system h/process-file! file-id update-fn opts)) + (db/tx-run! system (fn [system] + (binding [h/*system* system] + (h/process-file! system file-id update-fn opts))))) (catch Throwable cause (l/wrn :hint "unexpected error on processing file (skiping)" diff --git a/backend/src/app/tasks/file_gc.clj b/backend/src/app/tasks/file_gc.clj index 29c07f78b5..ed7815f688 100644 --- a/backend/src/app/tasks/file_gc.clj +++ b/backend/src/app/tasks/file_gc.clj @@ -65,19 +65,8 @@ :features (:features file) :version (:version file) :data (:data file)} - {:id id}))) - -(defn- process-file! - [cfg file] - (try - (let [file (decode-file cfg file) - file (clean-file! cfg file)] - (cfv/validate-file-schema! file) - (update-file! cfg file)) - (catch Throwable cause - (l/err :hint "error on cleaning file (skiping)" - :file-id (str (:id file)) - :cause cause)))) + {:id id} + {::db/return-keys true}))) (def ^:private sql:get-candidates @@ -118,13 +107,15 @@ unused (->> (db/exec! conn [sql:mark-file-media-object-deleted id ids]) (into #{} (map :id)))] + (l/dbg :hint "clean" :rel "file-media-object" :file-id (str id) :total (count unused)) + (doseq [id unused] (l/trc :hint "mark deleted" :rel "file-media-object" :id (str id) :file-id (str id))) - [(count unused) file])) + file)) (def ^:private sql:mark-file-object-thumbnails-deleted "UPDATE file_tagged_object_thumbnail @@ -149,13 +140,15 @@ unused (->> (db/exec! conn [sql:mark-file-object-thumbnails-deleted file-id ids]) (into #{} (map :object-id)))] + (l/dbg :hint "clean" :rel "file-object-thumbnail" :file-id (str file-id) :total (count unused)) + (doseq [object-id unused] (l/trc :hint "mark deleted" :rel "file-tagged-object-thumbnail" :object-id object-id :file-id (str file-id))) - [(count unused) file])) + file)) (def ^:private sql:mark-file-thumbnails-deleted "UPDATE file_thumbnail @@ -168,13 +161,15 @@ (let [unused (->> (db/exec! conn [sql:mark-file-thumbnails-deleted id revn]) (into #{} (map :revn)))] + (l/dbg :hint "clean" :rel "file-thumbnail" :file-id (str id) :total (count unused)) + (doseq [revn unused] (l/trc :hint "mark deleted" :rel "file-thumbnail" :revn revn :file-id (str id))) - [(count unused) file])) + file)) (def ^:private sql:get-files-for-library @@ -230,7 +225,9 @@ file (update file :data process-fdata unused)] - [(count unused) file])) + + (l/dbg :hint "clean" :rel "components" :file-id (str file-id) :total (count unused)) + file)) (def ^:private sql:get-changes "SELECT id, data FROM file_change @@ -242,47 +239,52 @@ SET deleted_at = now() WHERE file_id = ? AND id != ALL(?::uuid[]) + AND deleted_at IS NULL RETURNING id") +(def ^:private xf:collect-pointers + (comp (map :data) + (map blob/decode) + (mapcat feat.fdata/get-used-pointer-ids))) + (defn- clean-data-fragments! - [{:keys [::db/conn]} {:keys [id data] :as file}] - (let [used (->> (db/cursor conn [sql:get-changes id]) - (into (feat.fdata/get-used-pointer-ids data) - (comp (map :data) - (map blob/decode) - (mapcat feat.fdata/get-used-pointer-ids)))) + [{:keys [::db/conn]} {:keys [id] :as file}] + (let [used (into #{} xf:collect-pointers + (cons file (db/cursor conn [sql:get-changes id]))) unused (let [ids (db/create-array conn "uuid" used)] (->> (db/exec! conn [sql:mark-deleted-data-fragments id ids]) (into #{} (map :id))))] + (l/dbg :hint "clean" :rel "file-data-fragment" :file-id (str id) :total (count unused)) (doseq [id unused] (l/trc :hint "mark deleted" :rel "file-data-fragment" :id (str id) - :file-id (str id))) - - [(count unused) file])) - -(defn- clean-file! - [cfg {:keys [id] :as file}] - (let [[n1 file] (clean-file-media! cfg file) - [n2 file] (clean-file-thumbnails! cfg file) - [n3 file] (clean-file-object-thumbnails! cfg file) - [n4 file] (clean-deleted-components! cfg file) - [n5 file] (clean-data-fragments! cfg file)] - - (l/dbg :hint "file clened" - :file-id (str id) - :modified-at (dt/format-instant (:modified-at file)) - :media-objects n1 - :thumbnails n2 - :object-thumbnails n3 - :components n4 - :data-fragments n5) + :file-id (str id))))) +(defn- clean-media! + [cfg file] + (let [file (->> file + (clean-file-media! cfg) + (clean-file-thumbnails! cfg) + (clean-file-object-thumbnails! cfg) + (clean-deleted-components! cfg))] + (cfv/validate-file-schema! file) file)) +(defn- process-file! + [cfg file] + (try + (let [file (decode-file cfg file) + file (clean-media! cfg file) + file (update-file! cfg file)] + (clean-data-fragments! cfg file)) + (catch Throwable cause + (l/err :hint "error on cleaning file (skiping)" + :file-id (str (:id file)) + :cause cause)))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; HANDLER ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/backend/src/app/util/pointer_map.clj b/backend/src/app/util/pointer_map.clj index f5933ecd6a..16ce73bb0a 100644 --- a/backend/src/app/util/pointer_map.clj +++ b/backend/src/app/util/pointer_map.clj @@ -37,7 +37,6 @@ (:require [app.common.fressian :as fres] - [app.common.logging :as l] [app.common.transit :as t] [app.common.uuid :as uuid] [app.util.time :as dt] @@ -78,8 +77,6 @@ IPointerMap (load! [_] - (l/trace :hint "pointer-map:load" :id (str id)) - (when-not *load-fn* (throw (UnsupportedOperationException. "load is not supported when *load-fn* is not bind"))) diff --git a/backend/src/app/worker.clj b/backend/src/app/worker.clj index 3a6bfe897c..5c63ecfa0f 100644 --- a/backend/src/app/worker.clj +++ b/backend/src/app/worker.clj @@ -683,7 +683,6 @@ deleted (when dedupe (-> (db/exec-one! conn [sql:remove-not-started-tasks task queue label]) :next.jdbc/update-count))] - (l/trc :hint "submit task" :name task :queue queue diff --git a/frontend/src/app/main/ui/viewer/inspect/code.cljs b/frontend/src/app/main/ui/viewer/inspect/code.cljs index c1e5ebcd2f..bceac31ade 100644 --- a/frontend/src/app/main/ui/viewer/inspect/code.cljs +++ b/frontend/src/app/main/ui/viewer/inspect/code.cljs @@ -189,7 +189,14 @@ (mf/use-fn (mf/deps style-code markup-code images-data) (fn [] - (wapi/write-to-clipboard (gen-all-code style-code markup-code images-data)))) + (wapi/write-to-clipboard (gen-all-code style-code markup-code images-data)) + (let [origin (if (= :workspace from) + "workspace" + "viewer")] + (st/emit! (ptk/event ::ev/event + {::ev/name "copy-inspect-code" + ::ev/origin origin + :type "all"}))))) ;;handle-open-review ;;(mf/use-fn