Merge pull request #2236 from penpot/niwinz-minor-improvements

Minor improvements

CHANGES.md

@@ -3,6 +3,12 @@
 ## :rocket: Next
 
 ### :boom: Breaking changes & Deprecations
+
+- Removed the support for v2 internal file data blob format.  This
+  version has never been documented nor set as default value so
+  technicaly this is not a breaking change because we are removing
+  a "private API".
+
 ### :sparkles: New features
 
 - Add team hero in projects dashboard [Taiga #3863](https://tree.taiga.io/project/penpot/us/3863)

backend/deps.edn

@@ -6,7 +6,6 @@
   ;; Logging
   org.zeromq/jeromq {:mvn/version "0.5.2"}
 
-  com.taoensso/nippy {:mvn/version "3.1.1"}
   com.github.luben/zstd-jni {:mvn/version "1.5.2-3"}
   org.clojure/data.fressian {:mvn/version "1.0.0"}
 

backend/dev/user.clj

@@ -10,6 +10,7 @@
    [app.common.exceptions :as ex]
    [app.common.geom.matrix :as gmt]
    [app.common.perf :as perf]
+   [app.common.pprint :as pp]
    [app.common.transit :as t]
    [app.config :as cfg]
    [app.main :as main]
@@ -35,6 +36,24 @@
 
 (defonce system nil)
 
+;; --- Benchmarking Tools
+
+(defmacro run-quick-bench
+  [& exprs]
+  `(with-progress-reporting (quick-bench (do ~@exprs) :verbose)))
+
+(defmacro run-quick-bench'
+  [& exprs]
+  `(quick-bench (do ~@exprs)))
+
+(defmacro run-bench
+  [& exprs]
+  `(with-progress-reporting (bench (do ~@exprs) :verbose)))
+
+(defmacro run-bench'
+  [& exprs]
+  `(bench (do ~@exprs)))
+
 ;; --- Development Stuff
 
 (defn- run-tests

backend/src/app/srepl/helpers.clj

@@ -25,6 +25,7 @@
    [app.util.blob :as blob]
    [app.util.time :as dt]
    [clojure.spec.alpha :as s]
+   [clojure.stacktrace :as strace]
    [clojure.walk :as walk]
    [cuerdas.core :as str]
    [expound.alpha :as expound]))
@@ -78,7 +79,7 @@
       (update file :data blob/decode))))
 
 (def ^:private sql:retrieve-files-chunk
-  "SELECT id, name, modified_at, data FROM file
+  "SELECT id, name, created_at, data FROM file
     WHERE created_at < ? AND deleted_at is NULL
     ORDER BY created_at desc LIMIT ?")
 
@@ -88,26 +89,39 @@
 
   The `on-file` parameter should be a function that receives the file
   and the previous state and returns the new state."
-  [system & {:keys [chunk-size on-file] :or {chunk-size 10}}]
+  [system & {:keys [chunk-size max-chunks start-at on-file on-error on-end]
+             :or {chunk-size 10 max-chunks Long/MAX_VALUE}}]
   (letfn [(get-chunk [conn cursor]
             (let [rows (db/exec! conn [sql:retrieve-files-chunk cursor chunk-size])]
               [(some->> rows peek :created-at) (seq rows)]))
 
           (get-candidates [conn]
             (->> (d/iteration (partial get-chunk conn)
-                                       :vf second
+                              :vf second
-                                       :kf first
+                              :kf first
-                                       :initk (dt/now))
+                              :initk (or start-at (dt/now)))
-                 (sequence cat)
+                 (take max-chunks)
-                 (map #(update % :data blob/decode))))]
+                 (mapcat identity)
+                 (map #(update % :data blob/decode))))
+
+          (on-error* [file cause]
+            (println "unexpected exception happened on processing file: " (:id file))
+            (strace/print-stack-trace cause))]
 
     (db/with-atomic [conn (:app.db/pool system)]
       (loop [state {}
              files (get-candidates conn)]
         (if-let [file (first files)]
-          (let [state (on-file file state)]
+          (let [state' (try
-            (recur state (rest files)))
+                         (on-file file state)
-          state)))))
+                         (catch Throwable cause
+                           (let [on-error (or on-error on-error*)]
+                             (on-error file cause))))]
+            (recur (or state' state) (rest files)))
+
+          (if (fn? on-end)
+            (on-end state)
+            state))))))
 
 
 (defn analyze-file-data

backend/src/app/util/blob.clj

@@ -10,8 +10,7 @@
   (:require
    [app.common.transit :as t]
    [app.config :as cf]
-   [app.util.fressian :as fres]
+   [app.util.fressian :as fres])
-   [taoensso.nippy :as n])
   (:import
    java.io.ByteArrayInputStream
    java.io.ByteArrayOutputStream
@@ -27,21 +26,18 @@
 (def lz4-factory (LZ4Factory/fastestInstance))
 
 (declare decode-v1)
-(declare decode-v2)
 (declare decode-v3)
 (declare decode-v4)
 (declare encode-v1)
-(declare encode-v2)
 (declare encode-v3)
 (declare encode-v4)
 
 (defn encode
   ([data] (encode data nil))
   ([data {:keys [version]}]
-   (let [version (or version (cf/get :default-blob-version 3))]
+   (let [version (or version (cf/get :default-blob-version 4))]
      (case (long version)
        1 (encode-v1 data)
-       2 (encode-v2 data)
        3 (encode-v3 data)
        4 (encode-v4 data)
        (throw (ex-info "unsupported version" {:version version}))))))
@@ -55,7 +51,6 @@
           ulen    (.readInt dis)]
       (case version
         1 (decode-v1 data ulen)
-        2 (decode-v2 data ulen)
         3 (decode-v3 data ulen)
         4 (decode-v4 data ulen)
         (throw (ex-info "unsupported version" {:version version}))))))
@@ -84,29 +79,6 @@
     (.decompress ^LZ4FastDecompressor dcp cdata 6 ^bytes udata 0 ulen)
     (t/decode udata {:type :json})))
 
-(defn- encode-v2
-  [data]
-  (let [data  (n/fast-freeze data)
-        dlen  (alength ^bytes data)
-        mlen  (Zstd/compressBound dlen)
-        cdata (byte-array mlen)
-        clen  (Zstd/compressByteArray ^bytes cdata 0 mlen
-                                      ^bytes data 0 dlen
-                                      8)]
-    (with-open [^ByteArrayOutputStream baos (ByteArrayOutputStream. (+ (alength cdata) 2 4))
-                ^DataOutputStream dos (DataOutputStream. baos)]
-      (.writeShort dos (short 2)) ;; version number
-      (.writeInt dos (int dlen))
-      (.write dos ^bytes cdata (int 0) clen)
-      (.toByteArray baos))))
-
-(defn- decode-v2
-  [^bytes cdata ^long ulen]
-  (let [udata (byte-array ulen)]
-    (Zstd/decompressByteArray ^bytes udata 0 ulen
-                              ^bytes cdata 6 (- (alength cdata) 6))
-    (n/fast-thaw udata)))
-
 (defn- encode-v3
   [data]
   (let [data  (t/encode data {:type :json})