✨ Use embedded Ladybug Java API instead of CLI

backend/deps.edn

@@ -64,7 +64,9 @@
   ;; Pretty Print specs
   pretty-spec/pretty-spec {:mvn/version "0.1.4"}
   software.amazon.awssdk/s3 {:mvn/version "2.46.18"}
-  software.amazon.awssdk/sts {:mvn/version "2.46.18"}}
+  software.amazon.awssdk/sts {:mvn/version "2.46.18"}
+
+  com.ladybugdb/lbug {:mvn/version "0.18.0"}}
 
  :paths ["src" "resources" "target/classes"]
  :aliases

backend/src/app/graph/ingest.clj

@@ -40,7 +40,7 @@
           {:keys [statements stats]}
           (project.document/projection-statements data file)
           ingest-statements (conj (into ddl statements) "CHECKPOINT;")]
-      (ladybug/exec! system db-path ingest-statements)
+      (ladybug/exec! db-path ingest-statements)
       {:file-id        file-id
        :revn           (:revn file)
        :name           (or (:name data) (:name file))
@@ -48,4 +48,4 @@
        :schema-version schema/schema-version
        :projection     {:stats stats}
        :transforms     (project.transforms/apply-transforms! system db-path data file)
-       :stats          (stats/summarize system db-path)})))
+       :stats          (stats/summarize db-path)})))

backend/src/app/graph/ladybug.clj

@@ -5,24 +5,21 @@
 ;; Copyright (c) KALEIDOS INC Sucursal en España SL
 
 (ns app.graph.ladybug
-  "Thin Ladybug CLI access layer for graph-backed Penpot."
+  "Ladybug access layer for graph-backed Penpot.
+
+  Uses the embedded Java API (`com.ladybugdb/lbug`)."
   (:require
    [app.common.exceptions :as ex]
-   [app.util.shell :as shell]
    [clojure.string :as str]
    [datoteka.fs :as fs])
   (:import
-   java.util.concurrent.TimeUnit
+   com.ladybugdb.Connection
-   org.apache.commons.io.IOUtils))
+   com.ladybugdb.Database
+   com.ladybugdb.FlatTuple
+   com.ladybugdb.QueryResult
+   com.ladybugdb.Value))
 
-(defn lbug-bin
+(set! *warn-on-reflection* true)
-  "Resolved Ladybug CLI path (`PENPOT_LBUG_BIN`, `./lbug`, or `lbug`)."
-  []
-  (or (System/getenv "PENPOT_LBUG_BIN")
-      (let [local (java.io.File. "lbug")]
-        (when (.exists local)
-          (.getAbsolutePath local)))
-      "lbug"))
 
 (defn default-graph-dir
   []
@@ -32,9 +29,13 @@
   [file-id]
   (str (fs/path (default-graph-dir) (str file-id ".lbug"))))
 
+(defn- memory-db-path?
+  [db-path]
+  (= db-path ":memory:"))
+
 (defn reset-db-path!
   [db-path]
-  (when-not (= db-path ":memory:")
+  (when-not (memory-db-path? db-path)
     (when (fs/exists? db-path)
       (fs/delete db-path))))
 
@@ -71,127 +72,91 @@
   (let [s (str/trim (str statement))]
     (if (str/ends-with? s ";") s (str s ";"))))
 
-(defn- script-content
+(defn- value->clj
-  [statements]
+  [^Value value]
-  (str (str/join "\n" (map ensure-semicolon statements)) "\n"))
+  (when-not (.isNull value)
+    (let [v (.getValue value)]
+      (cond
+        (instance? Long v)    v
+        (instance? Integer v) (long v)
+        (instance? Double v)  v
+        :else                 v))))
 
-(defn- shell-quote
+(defn- check-success!
-  [s]
+  [^QueryResult result statement]
-  (str "\"" (str/replace s "\"" "\\\"") "\""))
+  (when-not (.isSuccess result)
+    (ex/raise :type :internal
+              :code :ladybug-query-failed
+              :hint "Ladybug query failed"
+              :statement statement
+              :err (.getErrorMessage result))))
 
-(defn- shell-single-quote
+(defn- with-connection
-  [s]
+  [db-path f]
-  (str "'" (str/replace s "'" "'\\''") "'"))
+  (let [^Database db (if (memory-db-path? db-path)
+                       (Database.)
+                       (Database. (str db-path)))]
+    (try
+      (let [^Connection conn (Connection. db)]
+        (try
+          (f conn)
+          (finally
+            (.close conn))))
+      (finally
+        (.close db)))))
 
-(defn- exec-sh-sync!
+(defn- scalar-value
-  "Run `sh -c` synchronously on the calling thread.
+  [^Connection conn statement]
+  (let [cypher (ensure-semicolon statement)]
+    (with-open [^QueryResult result (.query conn cypher)]
+      (check-success! result cypher)
+      (when (.hasNext result)
+        (with-open [^FlatTuple tuple (.getNext result)]
+          (with-open [^Value value (.getValue tuple 0)]
+            (value->clj value)))))))
 
-  `shell/exec!` can return empty stdout for very fast pipelines when used from
+(defn- run-statements!
-  the REPL executor; this path reads the merged stream before `waitFor` returns."
+  [^Connection conn statements]
-  [shell-cmd & {:keys [timeout] :or {timeout 120}}]
+  (doseq [statement statements]
-  (let [^Process proc (.start (doto (ProcessBuilder. (into-array String ["sh" "-c" shell-cmd]))
+    (let [cypher (ensure-semicolon statement)]
-                               (.redirectErrorStream true)))
+      (with-open [^QueryResult result (.query conn cypher)]
-        out        (IOUtils/toString (.getInputStream proc) "UTF-8")]
+        (check-success! result cypher)))))
-    (when-not (.waitFor proc (long timeout) TimeUnit/SECONDS)
-      (.destroyForcibly proc)
-      (ex/raise :type :internal
-                :code :ladybug-timeout
-                :hint "Ladybug query timed out"
-                :shell-cmd shell-cmd
-                :timeout timeout))
-    {:exit (.exitValue proc)
-     :out  out
-     :err  ""}))
 
-(defn- run-script!
+(def ^:private default-query-timeout-seconds 120)
-  "Run `lbug` with Cypher on stdin.
-
-  Writes use a heredoc via `shell/exec!`. Queries use `printf ... | lbug`
-  via a synchronous shell invocation (matches the working manual command)."
-  [system db-path flags script & {:keys [timeout] :or {timeout 120}}]
-  (let [cmd       (into [(lbug-bin) db-path] flags)
-        query?    (some #{"line"} flags)
-        shell-cmd (if query?
-                    (str "printf " (shell-single-quote script)
-                         " | " (str/join " " (map shell-quote cmd)))
-                    (str (str/join " " (map shell-quote cmd))
-                         " <<'LBUG_EOF'\n" script "\nLBUG_EOF"))]
-    (if query?
-      (exec-sh-sync! shell-cmd :timeout timeout)
-      (shell/exec! system {:cmd ["sh" "-c" shell-cmd] :timeout timeout}))))
 
 (defn exec!
-  "Execute Cypher statements. `:mode` is `:write` (default) or `:query`."
+  "Execute Cypher statements against a Ladybug database.
-  [system db-path statements & {:keys [timeout mode] :or {timeout 120 mode :write}}]
+
+  `db-path` is either `:memory:` or a filesystem path to a `.lbug` database."
+  [db-path statements]
   (assert (sequential? statements) "statements should be a sequential collection")
-  (when-not (= db-path ":memory:")
+  (when-not (memory-db-path? db-path)
     (fs/create-dir (fs/parent db-path)))
-  (let [flags  (if (= mode :query)
+  (with-connection db-path
-                 ["-m" "line" "-s"]
+    (fn [^Connection conn]
-                 ["-m" "trash" "-s" "-b"])
+      (.setQueryTimeout conn default-query-timeout-seconds)
-        script (if (= mode :query)
+      (run-statements! conn statements))))
-                 (str ":singleline\n" (script-content statements))
-                 (script-content statements))
-        result (run-script! system db-path flags script :timeout timeout)]
-    (when (not= 0 (:exit result))
-      (ex/raise :type :internal
-                :code :ladybug-exec-failed
-                :hint "Ladybug execution failed"
-                :db-path db-path
-                :exit (:exit result)
-                :out (:out result)
-                :err (:err result)))
-    result))
-
-(defn- lbug-noise-line?
-  [line]
-  (or (str/blank? line)
-      (str/starts-with? line "--")
-      (str/starts-with? line ":singleline")
-      (str/includes? line "Single line mode")
-      (str/includes? line "usage hints")
-      (str/includes? line "Processing:")
-      (str/includes? line "Pipeline")
-      (str/includes? line "Progress:")))
-
-(defn- data-lines
-  [out]
-  (->> (str/split-lines (str out))
-       (map str/trim)
-       (remove lbug-noise-line?)))
-
-(defn- parse-scalar-line
-  [line]
-  (let [line (str/trim line)]
-    (cond
-      (re-matches #"-?\d+" line) (Long/parseLong line)
-      :else (some->> (re-seq #"-?\d+" line) last Long/parseLong))))
-
-(defn- parse-equality-value
-  "Last `label = 123` tuple in output (results come after pipeline noise)."
-  [out]
-  (some->> (re-seq #"([A-Za-z][A-Za-z0-9_]*)\s*=\s*(-?\d+)" (str out))
-           (remove (fn [[_ label _]]
-                     (or (str/includes? label "Pipeline")
-                         (str/includes? label "Progress"))))
-           last
-           (nth 2)
-           Long/parseLong))
 
 (defn query-scalar!
-  [system db-path statement & {:keys [timeout] :or {timeout 120}}]
+  "Execute a query expected to return a single scalar value."
-  (let [out (:out (exec! system db-path [statement] :timeout timeout :mode :query))]
+  [db-path statement]
-    (or (some parse-scalar-line (data-lines out))
+  (with-connection db-path
-        (parse-equality-value out))))
+    (fn [^Connection conn]
-
+      (.setQueryTimeout conn default-query-timeout-seconds)
-(defn smoke-test-statements
+      (scalar-value conn statement))))
-  []
-  ["CREATE NODE TABLE Person(name STRING, age INT64, PRIMARY KEY(name));"
-   "CREATE (:Person {name: 'Alice', age: 25});"
-   "CREATE (:Person {name: 'Bob', age: 30});"
-   "MATCH (a:Person) RETURN a.name AS NAME, a.age AS AGE ORDER BY NAME;"])
 
 (defn smoke-test!
-  [system & {:keys [db-path] :or {db-path ":memory:"}}]
+  "Run a minimal CREATE + count against Ladybug."
-  {:db-path db-path
+  [& {:keys [db-path] :or {db-path ":memory:"}}]
-   :out     (:out (exec! system db-path (smoke-test-statements)))})
+  (when-not (memory-db-path? db-path)
+    (reset-db-path! db-path)
+    (fs/create-dir (fs/parent db-path)))
+  (with-connection db-path
+    (fn [^Connection conn]
+      (run-statements! conn
+                       ["CREATE NODE TABLE Person(name STRING, age INT64, PRIMARY KEY(name));"
+                        "CREATE (:Person {name: 'Alice', age: 25});"
+                        "CREATE (:Person {name: 'Bob', age: 30});"])
+      {:db-path      db-path
+       :person-count (scalar-value conn
+                                   "MATCH (a:Person) RETURN count(a) AS c;")})))

backend/src/app/graph/stats.clj

@@ -16,19 +16,19 @@
     table))
 
 (defn- count-query
-  [system db-path statement]
+  [db-path statement]
-  (or (ladybug/query-scalar! system db-path statement) 0))
+  (or (ladybug/query-scalar! db-path statement) 0))
 
 (defn summarize
   "Return node/edge counts from the graph database."
-  [system db-path]
+  [db-path]
   {:nodes (into {}
                 (map (fn [{:keys [name]}]
                        [name (count-query
-                              system db-path
+                              db-path
                               (str "MATCH (n:" (node-label-for-match name) ") "
                                    "RETURN count(n) AS " name "_c;"))])
                      schema/node-tables))
    :edges {:IsChildOf (count-query
-                        system db-path
+                        db-path
                         "MATCH ()-[e:IsChildOf]->() RETURN count(e) AS IsChildOf_c;")}})

backend/src/app/srepl/main.clj

@@ -406,12 +406,12 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (defn graph-smoke-test!
-  "Execute a basic Ladybug smoke test (CREATE + MATCH).
+  "Execute a basic Ladybug smoke test (CREATE + count).
 
-  Requires the `lbug` CLI on PATH, or set PENPOT_LBUG_BIN. Use :db-path
+  Uses the embedded Ladybug Java API. Use :db-path \":memory:\" (default)
-  \":memory:\" (default) or a filesystem path such as /tmp/test.lbug."
+  or a filesystem path such as /tmp/test.lbug."
   [& {:keys [db-path] :or {db-path ":memory:"}}]
-  (graph.ladybug/smoke-test! main/system :db-path db-path))
+  (graph.ladybug/smoke-test! :db-path db-path))
 
 (defn graph-query-test!
   "Query Document count for a file's graph db (REPL diagnostic)."
@@ -419,7 +419,7 @@
   (let [file-id (h/parse-uuid file-id)
         db-path (or db-path (graph.ladybug/db-path-for-file file-id))
         stmt    "MATCH (n:Document) RETURN count(n) AS Document_c;"]
-    (graph.ladybug/query-scalar! main/system db-path stmt)))
+    (graph.ladybug/query-scalar! db-path stmt)))
 
 (defn ingest-file-to-graph!
   "Project a Penpot file into a per-file Ladybug database.