diff --git a/backend/deps.edn b/backend/deps.edn index c60adf10ca..9bdf318dd7 100644 --- a/backend/deps.edn +++ b/backend/deps.edn @@ -64,7 +64,9 @@ ;; Pretty Print specs pretty-spec/pretty-spec {:mvn/version "0.1.4"} software.amazon.awssdk/s3 {:mvn/version "2.46.18"} - software.amazon.awssdk/sts {:mvn/version "2.46.18"}} + software.amazon.awssdk/sts {:mvn/version "2.46.18"} + + com.ladybugdb/lbug {:mvn/version "0.18.0"}} :paths ["src" "resources" "target/classes"] :aliases diff --git a/backend/src/app/graph/ingest.clj b/backend/src/app/graph/ingest.clj index 1762a1f4e2..8b7d29d81b 100644 --- a/backend/src/app/graph/ingest.clj +++ b/backend/src/app/graph/ingest.clj @@ -40,7 +40,7 @@ {:keys [statements stats]} (project.document/projection-statements data file) ingest-statements (conj (into ddl statements) "CHECKPOINT;")] - (ladybug/exec! system db-path ingest-statements) + (ladybug/exec! db-path ingest-statements) {:file-id file-id :revn (:revn file) :name (or (:name data) (:name file)) @@ -48,4 +48,4 @@ :schema-version schema/schema-version :projection {:stats stats} :transforms (project.transforms/apply-transforms! system db-path data file) - :stats (stats/summarize system db-path)}))) + :stats (stats/summarize db-path)}))) diff --git a/backend/src/app/graph/ladybug.clj b/backend/src/app/graph/ladybug.clj index 3aeb6b52c8..f984b6bba6 100644 --- a/backend/src/app/graph/ladybug.clj +++ b/backend/src/app/graph/ladybug.clj @@ -5,24 +5,21 @@ ;; Copyright (c) KALEIDOS INC Sucursal en España SL (ns app.graph.ladybug - "Thin Ladybug CLI access layer for graph-backed Penpot." + "Ladybug access layer for graph-backed Penpot. + + Uses the embedded Java API (`com.ladybugdb/lbug`)." (:require [app.common.exceptions :as ex] - [app.util.shell :as shell] [clojure.string :as str] [datoteka.fs :as fs]) (:import - java.util.concurrent.TimeUnit - org.apache.commons.io.IOUtils)) + com.ladybugdb.Connection + com.ladybugdb.Database + com.ladybugdb.FlatTuple + com.ladybugdb.QueryResult + com.ladybugdb.Value)) -(defn lbug-bin - "Resolved Ladybug CLI path (`PENPOT_LBUG_BIN`, `./lbug`, or `lbug`)." - [] - (or (System/getenv "PENPOT_LBUG_BIN") - (let [local (java.io.File. "lbug")] - (when (.exists local) - (.getAbsolutePath local))) - "lbug")) +(set! *warn-on-reflection* true) (defn default-graph-dir [] @@ -32,9 +29,13 @@ [file-id] (str (fs/path (default-graph-dir) (str file-id ".lbug")))) +(defn- memory-db-path? + [db-path] + (= db-path ":memory:")) + (defn reset-db-path! [db-path] - (when-not (= db-path ":memory:") + (when-not (memory-db-path? db-path) (when (fs/exists? db-path) (fs/delete db-path)))) @@ -71,127 +72,91 @@ (let [s (str/trim (str statement))] (if (str/ends-with? s ";") s (str s ";")))) -(defn- script-content - [statements] - (str (str/join "\n" (map ensure-semicolon statements)) "\n")) +(defn- value->clj + [^Value value] + (when-not (.isNull value) + (let [v (.getValue value)] + (cond + (instance? Long v) v + (instance? Integer v) (long v) + (instance? Double v) v + :else v)))) -(defn- shell-quote - [s] - (str "\"" (str/replace s "\"" "\\\"") "\"")) +(defn- check-success! + [^QueryResult result statement] + (when-not (.isSuccess result) + (ex/raise :type :internal + :code :ladybug-query-failed + :hint "Ladybug query failed" + :statement statement + :err (.getErrorMessage result)))) -(defn- shell-single-quote - [s] - (str "'" (str/replace s "'" "'\\''") "'")) +(defn- with-connection + [db-path f] + (let [^Database db (if (memory-db-path? db-path) + (Database.) + (Database. (str db-path)))] + (try + (let [^Connection conn (Connection. db)] + (try + (f conn) + (finally + (.close conn)))) + (finally + (.close db))))) -(defn- exec-sh-sync! - "Run `sh -c` synchronously on the calling thread. +(defn- scalar-value + [^Connection conn statement] + (let [cypher (ensure-semicolon statement)] + (with-open [^QueryResult result (.query conn cypher)] + (check-success! result cypher) + (when (.hasNext result) + (with-open [^FlatTuple tuple (.getNext result)] + (with-open [^Value value (.getValue tuple 0)] + (value->clj value))))))) - `shell/exec!` can return empty stdout for very fast pipelines when used from - the REPL executor; this path reads the merged stream before `waitFor` returns." - [shell-cmd & {:keys [timeout] :or {timeout 120}}] - (let [^Process proc (.start (doto (ProcessBuilder. (into-array String ["sh" "-c" shell-cmd])) - (.redirectErrorStream true))) - out (IOUtils/toString (.getInputStream proc) "UTF-8")] - (when-not (.waitFor proc (long timeout) TimeUnit/SECONDS) - (.destroyForcibly proc) - (ex/raise :type :internal - :code :ladybug-timeout - :hint "Ladybug query timed out" - :shell-cmd shell-cmd - :timeout timeout)) - {:exit (.exitValue proc) - :out out - :err ""})) +(defn- run-statements! + [^Connection conn statements] + (doseq [statement statements] + (let [cypher (ensure-semicolon statement)] + (with-open [^QueryResult result (.query conn cypher)] + (check-success! result cypher))))) -(defn- run-script! - "Run `lbug` with Cypher on stdin. - - Writes use a heredoc via `shell/exec!`. Queries use `printf ... | lbug` - via a synchronous shell invocation (matches the working manual command)." - [system db-path flags script & {:keys [timeout] :or {timeout 120}}] - (let [cmd (into [(lbug-bin) db-path] flags) - query? (some #{"line"} flags) - shell-cmd (if query? - (str "printf " (shell-single-quote script) - " | " (str/join " " (map shell-quote cmd))) - (str (str/join " " (map shell-quote cmd)) - " <<'LBUG_EOF'\n" script "\nLBUG_EOF"))] - (if query? - (exec-sh-sync! shell-cmd :timeout timeout) - (shell/exec! system {:cmd ["sh" "-c" shell-cmd] :timeout timeout})))) +(def ^:private default-query-timeout-seconds 120) (defn exec! - "Execute Cypher statements. `:mode` is `:write` (default) or `:query`." - [system db-path statements & {:keys [timeout mode] :or {timeout 120 mode :write}}] + "Execute Cypher statements against a Ladybug database. + + `db-path` is either `:memory:` or a filesystem path to a `.lbug` database." + [db-path statements] (assert (sequential? statements) "statements should be a sequential collection") - (when-not (= db-path ":memory:") + (when-not (memory-db-path? db-path) (fs/create-dir (fs/parent db-path))) - (let [flags (if (= mode :query) - ["-m" "line" "-s"] - ["-m" "trash" "-s" "-b"]) - script (if (= mode :query) - (str ":singleline\n" (script-content statements)) - (script-content statements)) - result (run-script! system db-path flags script :timeout timeout)] - (when (not= 0 (:exit result)) - (ex/raise :type :internal - :code :ladybug-exec-failed - :hint "Ladybug execution failed" - :db-path db-path - :exit (:exit result) - :out (:out result) - :err (:err result))) - result)) - -(defn- lbug-noise-line? - [line] - (or (str/blank? line) - (str/starts-with? line "--") - (str/starts-with? line ":singleline") - (str/includes? line "Single line mode") - (str/includes? line "usage hints") - (str/includes? line "Processing:") - (str/includes? line "Pipeline") - (str/includes? line "Progress:"))) - -(defn- data-lines - [out] - (->> (str/split-lines (str out)) - (map str/trim) - (remove lbug-noise-line?))) - -(defn- parse-scalar-line - [line] - (let [line (str/trim line)] - (cond - (re-matches #"-?\d+" line) (Long/parseLong line) - :else (some->> (re-seq #"-?\d+" line) last Long/parseLong)))) - -(defn- parse-equality-value - "Last `label = 123` tuple in output (results come after pipeline noise)." - [out] - (some->> (re-seq #"([A-Za-z][A-Za-z0-9_]*)\s*=\s*(-?\d+)" (str out)) - (remove (fn [[_ label _]] - (or (str/includes? label "Pipeline") - (str/includes? label "Progress")))) - last - (nth 2) - Long/parseLong)) + (with-connection db-path + (fn [^Connection conn] + (.setQueryTimeout conn default-query-timeout-seconds) + (run-statements! conn statements)))) (defn query-scalar! - [system db-path statement & {:keys [timeout] :or {timeout 120}}] - (let [out (:out (exec! system db-path [statement] :timeout timeout :mode :query))] - (or (some parse-scalar-line (data-lines out)) - (parse-equality-value out)))) - -(defn smoke-test-statements - [] - ["CREATE NODE TABLE Person(name STRING, age INT64, PRIMARY KEY(name));" - "CREATE (:Person {name: 'Alice', age: 25});" - "CREATE (:Person {name: 'Bob', age: 30});" - "MATCH (a:Person) RETURN a.name AS NAME, a.age AS AGE ORDER BY NAME;"]) + "Execute a query expected to return a single scalar value." + [db-path statement] + (with-connection db-path + (fn [^Connection conn] + (.setQueryTimeout conn default-query-timeout-seconds) + (scalar-value conn statement)))) (defn smoke-test! - [system & {:keys [db-path] :or {db-path ":memory:"}}] - {:db-path db-path - :out (:out (exec! system db-path (smoke-test-statements)))}) + "Run a minimal CREATE + count against Ladybug." + [& {:keys [db-path] :or {db-path ":memory:"}}] + (when-not (memory-db-path? db-path) + (reset-db-path! db-path) + (fs/create-dir (fs/parent db-path))) + (with-connection db-path + (fn [^Connection conn] + (run-statements! conn + ["CREATE NODE TABLE Person(name STRING, age INT64, PRIMARY KEY(name));" + "CREATE (:Person {name: 'Alice', age: 25});" + "CREATE (:Person {name: 'Bob', age: 30});"]) + {:db-path db-path + :person-count (scalar-value conn + "MATCH (a:Person) RETURN count(a) AS c;")}))) diff --git a/backend/src/app/graph/stats.clj b/backend/src/app/graph/stats.clj index 00ad5ef182..282f982e0d 100644 --- a/backend/src/app/graph/stats.clj +++ b/backend/src/app/graph/stats.clj @@ -16,19 +16,19 @@ table)) (defn- count-query - [system db-path statement] - (or (ladybug/query-scalar! system db-path statement) 0)) + [db-path statement] + (or (ladybug/query-scalar! db-path statement) 0)) (defn summarize "Return node/edge counts from the graph database." - [system db-path] + [db-path] {:nodes (into {} (map (fn [{:keys [name]}] [name (count-query - system db-path + db-path (str "MATCH (n:" (node-label-for-match name) ") " "RETURN count(n) AS " name "_c;"))]) schema/node-tables)) :edges {:IsChildOf (count-query - system db-path + db-path "MATCH ()-[e:IsChildOf]->() RETURN count(e) AS IsChildOf_c;")}}) diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index dcefefccb6..a30663a21a 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -406,12 +406,12 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (defn graph-smoke-test! - "Execute a basic Ladybug smoke test (CREATE + MATCH). + "Execute a basic Ladybug smoke test (CREATE + count). - Requires the `lbug` CLI on PATH, or set PENPOT_LBUG_BIN. Use :db-path - \":memory:\" (default) or a filesystem path such as /tmp/test.lbug." + Uses the embedded Ladybug Java API. Use :db-path \":memory:\" (default) + or a filesystem path such as /tmp/test.lbug." [& {:keys [db-path] :or {db-path ":memory:"}}] - (graph.ladybug/smoke-test! main/system :db-path db-path)) + (graph.ladybug/smoke-test! :db-path db-path)) (defn graph-query-test! "Query Document count for a file's graph db (REPL diagnostic)." @@ -419,7 +419,7 @@ (let [file-id (h/parse-uuid file-id) db-path (or db-path (graph.ladybug/db-path-for-file file-id)) stmt "MATCH (n:Document) RETURN count(n) AS Document_c;"] - (graph.ladybug/query-scalar! main/system db-path stmt))) + (graph.ladybug/query-scalar! db-path stmt))) (defn ingest-file-to-graph! "Project a Penpot file into a per-file Ladybug database.