✨ Add debug graph console for in-memory Cypher queries

backend/resources/app/templates/debug.tmpl

@@ -233,6 +233,7 @@ Debug Main Page
         </div>
         <div class="row">
           <input type="submit" value="Download .lbug" />
+          <a href="/dbg/graph">Open graph console</a>
         </div>
       </form>
     </fieldset>

backend/resources/app/templates/graph-console.tmpl

@@ -0,0 +1,108 @@
+{% extends "app/templates/base.tmpl" %}
+
+{% block title %}
+Graph Console
+{% endblock %}
+
+{% block content %}
+<nav>
+  <div class="title">
+    <h1>GRAPH CONSOLE (VERSION: {{version}})</h1>
+  </div>
+</nav>
+<main class="dashboard">
+  <section class="widget">
+    <p><a href="/dbg">&larr; Back to debug</a></p>
+
+    <fieldset>
+      <legend>Load graph in memory</legend>
+      <desc>
+        Projects the Penpot file into an in-memory Ladybug database for this
+        admin session. Loading a new file replaces the previous one.
+      </desc>
+      <form method="post" action="/dbg/actions/graph-load">
+        <div class="row">
+          <input type="text" style="width:420px" name="file-id"
+                 placeholder="file-id"
+                 value="{% if session %}{{session.file-id}}{% endif %}" />
+        </div>
+        <div class="row">                 
+          <input type="submit" value="Load" />
+        </div>
+      </form>
+      {% if session %}
+      <form method="post" action="/dbg/actions/graph-unload">
+        <div class="row">
+          <input type="submit" value="Unload" />
+        </div>
+      </form>
+      {% endif %}
+    </fieldset>
+
+    {% if session %}
+    <fieldset>
+      <legend>Loaded session</legend>
+      <desc>
+        <p>
+          File: <b>{{session.name}}</b> ({{session.file-id}})<br />
+          Revision: <b>{{session.revn}}</b><br />
+          Schema: <b>{{session.schema-version}}</b><br />
+          Loaded at: <b>{{session.loaded-at}}</b>
+        </p>
+        {% if session.projection.stats %}
+        <p>
+          Projection:
+          documents={{session.projection.stats.documents}},
+          pages={{session.projection.stats.pages}},
+          shapes={{session.projection.stats.shapes}}
+        </p>
+        {% endif %}
+      </desc>
+    </fieldset>
+
+    <fieldset>
+      <legend>Cypher query</legend>
+      <form method="post" action="/dbg/actions/graph-query">
+        <div class="row">
+          <textarea name="query" rows="8" style="width:100%; font-family: monospace;">{{query}}</textarea>
+        </div>
+        <div class="row">
+          <input type="submit" value="Run query" />
+        </div>
+      </form>
+    </fieldset>
+    {% endif %}
+
+    {% if error %}
+    <fieldset>
+      <legend>Error</legend>
+      <pre>{{error}}</pre>
+    </fieldset>
+    {% endif %}
+
+    {% if query-result %}
+    <fieldset>
+      <legend>Results ({{query-result.row-count}} rows{% if query-result.truncated? %}, truncated{% endif %})</legend>
+      <table border="1" cellpadding="4" cellspacing="0" style="border-collapse: collapse; width: 100%;">
+        <thead>
+          <tr>
+            {% for column in query-result.columns %}
+            <th>{{column}}</th>
+            {% endfor %}
+          </tr>
+        </thead>
+        <tbody>
+          {% for row in query-result.rows %}
+          <tr>
+            {% for cell in row %}
+            <td><code>{{cell}}</code></td>
+            {% endfor %}
+          </tr>
+          {% endfor %}
+        </tbody>
+      </table>
+    </fieldset>
+    {% endif %}
+  </section>
+</main>
+{% endblock %}

backend/src/app/graph/bulk.clj

@@ -72,8 +72,8 @@
     (.write w "from,to,position\n")
     (doseq [{:keys [from-id to-id position]} edges]
       (.write w (str (csv-cell from-id) ","
-                      (csv-cell to-id) ","
+                     (csv-cell to-id) ","
-                      (csv-cell position) "\n")))))
+                     (csv-cell position) "\n")))))
 
 (defn- delete-tree!
   [path]

backend/src/app/graph/debug.clj

@@ -0,0 +1,117 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) KALEIDOS INC Sucursal en España SL
+
+(ns app.graph.debug
+  "In-memory Ladybug sessions for the debug graph console."
+  (:require
+   [app.common.exceptions :as ex]
+   [app.common.time :as ct]
+   [app.graph.ingest :as graph.ingest]
+   [app.graph.ladybug :as ladybug]
+   [clojure.string :as str])
+  (:import
+   com.ladybugdb.Connection
+   com.ladybugdb.Database))
+
+(set! *warn-on-reflection* true)
+
+(def default-query
+  "MATCH (n:Document) RETURN n.id AS id, n.name AS name;")
+
+(defonce ^:private sessions
+  (atom {}))
+
+(defn- session-key
+  [profile-id]
+  (str profile-id))
+
+(defn- destroy-session!
+  [{:keys [conn db]}]
+  (when conn
+    (ex/ignoring (.close ^Connection conn)))
+  (when db
+    (ex/ignoring (.close ^Database db))))
+
+(defn- format-cell
+  [value]
+  (cond
+    (nil? value) "NULL"
+    (string? value) value
+    :else (str value)))
+
+(defn- format-query-result
+  [{:keys [columns rows truncated?]}]
+  {:columns (mapv str columns)
+   :rows    (mapv (fn [row]
+                    (mapv format-cell row))
+                  rows)
+   :truncated? truncated?
+   :row-count (count rows)})
+
+(defn session-info
+  "Return a public view of the current session for `profile-id`, if any."
+  [profile-id]
+  (when-let [{:keys [file-id meta loaded-at]} (get @sessions (session-key profile-id))]
+    {:file-id        file-id
+     :name           (:name meta)
+     :revn           (:revn meta)
+     :schema-version (:schema-version meta)
+     :projection     (:projection meta)
+     :loaded-at      (ct/format-inst loaded-at :iso)}))
+
+(defn unload-session!
+  "Close and discard the in-memory graph for `profile-id`."
+  [profile-id]
+  (when-let [session (get @sessions (session-key profile-id))]
+    (destroy-session! session))
+  (swap! sessions dissoc (session-key profile-id)))
+
+(defn load-session!
+  "Ingest `file-id` into a new in-memory Ladybug database for `profile-id`."
+  [cfg profile-id file-id]
+  (unload-session! profile-id)
+  (let [^Database db (Database.)
+        ^Connection conn (Connection. db)]
+    (.setQueryTimeout conn 0)
+    (try
+      (let [meta (graph.ingest/ingest-on-connection! cfg conn file-id
+                                                     :db-path ":memory:"
+                                                     :skip-stats? true
+                                                     :skip-validation? true)]
+        (swap! sessions assoc (session-key profile-id)
+               {:db db
+                :conn conn
+                :file-id file-id
+                :meta meta
+                :loaded-at (ct/now)})
+        meta)
+      (catch Throwable cause
+        (destroy-session! {:conn conn :db db})
+        (throw cause)))))
+
+(defn query-session!
+  "Run `statement` against the in-memory graph for `profile-id`."
+  [profile-id statement]
+  (when (str/blank? statement)
+    (ex/raise :type :validation
+              :code :missing-query
+              :hint "cypher query is required"))
+  (if-let [{:keys [conn]} (get @sessions (session-key profile-id))]
+    (-> (ladybug/query-on-connection! conn statement)
+        format-query-result)
+    (ex/raise :type :not-found
+              :code :graph-session-not-loaded
+              :hint "load a file graph before running queries")))
+
+(defn console-context
+  "Build template data for the graph debug console page."
+  [profile-id & {:keys [query query-result error message]}]
+  {:session       (session-info profile-id)
+   :query         (or query default-query)
+   :query-result  query-result
+   :error         error
+   :message       message
+   :default-query default-query})

backend/src/app/graph/ingest.clj

@@ -18,14 +18,14 @@
    [app.graph.project.transforms :as project.transforms]
    [app.graph.schema :as schema]
    [app.graph.stats :as stats]
-   [app.srepl.helpers :as h]))
+   [app.srepl.helpers :as h])
+  (:import
+   com.ladybugdb.Connection))
 
-(defn ingest-file!
+(defn- fetch-file!
-  [system file-id & {:keys [db-path reset-db? skip-stats?]
+  [system file-id]
-                     :or   {reset-db? true}}]
   (let [file-id (h/parse-uuid file-id)
-        file    (db/run! system #(bfc/get-file % file-id :realize? true))
+        file    (db/run! system #(bfc/get-file % file-id :realize? true))]
-        db-path (or db-path (ladybug/db-path-for-file file-id))]
     (when-not file
       (ex/raise :type :not-found
                 :code :file-not-found
@@ -35,30 +35,48 @@
                 :code :file-without-data
                 :hint "file has no data to project"
                 :file-id (str file-id)))
-    (ctf/check-file-data (:data file))
+    [file-id file]))
-    (when reset-db?
+
-      (ladybug/reset-db-path! db-path))
+(defn ingest-on-connection!
+  "Project `file-id` into an already open Ladybug `conn`."
+  [system ^Connection conn file-id & {:keys [db-path skip-stats? skip-validation?]
+                                      :or   {skip-stats? true}}]
+  (let [[file-id file] (fetch-file! system file-id)
+        db-path        (or db-path (ladybug/db-path-for-file file-id))
+        data           (:data file)]
+    (when-not skip-validation?
+      (ctf/check-file-data data))
     (l/inf :hint "graph ingest"
            :file-id (str file-id)
            :revn (:revn file)
            :db-path db-path
            :schema schema/schema-version)
-    (let [data              (:data file)
+    (let [ddl          (schema/ddl-statements)
-          ddl               (schema/ddl-statements)
           {:keys [nodes edges stats]}
           (project.document/projection-data data file)
-          staging-path      (bulk/staging-dir db-path file-id)]
+          staging-path (bulk/staging-dir db-path file-id)]
-      (ladybug/with-connection! db-path
+      (ladybug/exec-on-connection! conn ddl)
-        (fn [conn]
+      (bulk/load-projection! conn {:nodes nodes :edges edges} staging-path)
-          (ladybug/exec-on-connection! conn ddl)
+      (ladybug/exec-on-connection! conn ["CHECKPOINT;"])
-          (bulk/load-projection! conn {:nodes nodes :edges edges} staging-path)
+      {:file-id        file-id
-          (ladybug/exec-on-connection! conn ["CHECKPOINT;"])
+       :revn           (:revn file)
-          {:file-id        file-id
+       :name           (or (:name data) (:name file))
-           :revn           (:revn file)
+       :db-path        db-path
-           :name           (or (:name data) (:name file))
+       :schema-version schema/schema-version
-           :db-path        db-path
+       :projection     {:stats stats}
-           :schema-version schema/schema-version
+       :transforms     (project.transforms/apply-transforms! system db-path data file)
-           :projection     {:stats stats}
+       :stats          (when-not skip-stats?
-           :transforms     (project.transforms/apply-transforms! system db-path data file)
+                         (stats/summarize-connection conn))})))
-           :stats          (when-not skip-stats?
+
-                             (stats/summarize-connection conn))})))))
+(defn ingest-file!
+  [system file-id & {:keys [db-path reset-db? skip-stats? skip-validation?]
+                     :or   {reset-db? true}}]
+  (let [db-path (or db-path (ladybug/db-path-for-file (h/parse-uuid file-id)))]
+    (when reset-db?
+      (ladybug/reset-db-path! db-path))
+    (ladybug/with-connection! db-path
+      (fn [conn]
+        (ingest-on-connection! system conn file-id
+                               :db-path db-path
+                               :skip-stats? skip-stats?
+                               :skip-validation? skip-validation?)))))

backend/src/app/graph/ladybug.clj

@@ -92,6 +92,46 @@
                 :statement statement
                 :err err))))
 
+(defn- query-columns
+  [^QueryResult result]
+  (let [ncols (.getNumColumns result)]
+    (vec (for [i (range ncols)]
+           (.getColumnName result (long i))))))
+
+(defn- query-row
+  [^FlatTuple tuple ncols]
+  (vec (for [i (range ncols)]
+         (with-open [^Value value (.getValue tuple (long i))]
+           (value->clj value)))))
+
+(def ^:private default-query-max-rows 200)
+
+(defn- read-query-rows
+  [^QueryResult result ncols max-rows]
+  (loop [rows [] n 0]
+    (if (and (< n max-rows) (.hasNext result))
+      (let [row (with-open [^FlatTuple tuple (.getNext result)]
+                  (query-row tuple ncols))]
+        (recur (conj rows row) (inc n)))
+      rows)))
+
+(defn query-on-connection!
+  "Execute a Cypher query on `conn` and return tabular results.
+
+  Returns `{:columns [...] :rows [[...] ...] :truncated? bool}`."
+  [^Connection conn statement & {:keys [max-rows]
+                                 :or   {max-rows default-query-max-rows}}]
+  (let [cypher (ensure-semicolon statement)]
+    (with-open [^QueryResult result (.query conn cypher)]
+      (check-success! result cypher)
+      (let [ncols   (long (.getNumColumns result))
+            columns (query-columns result)
+            rows    (read-query-rows result ncols max-rows)
+            total   (long (.getNumTuples result))]
+        {:columns    columns
+         :rows       rows
+         :truncated? (and (pos? total) (> total (count rows)))}))))
+
 (def ^:private default-query-timeout-ms
   "0 disables query timeout (recommended for bulk COPY ingest)."
   0)

backend/src/app/graph/stats.clj

@@ -30,8 +30,8 @@
                                    "RETURN count(n) AS " name "_c;"))])
                      schema/node-tables))
    :edges {:IsChildOf (count-on-connection
-                        conn
+                       conn
-                        "MATCH ()-[e:IsChildOf]->() RETURN count(e) AS IsChildOf_c;")}})
+                       "MATCH ()-[e:IsChildOf]->() RETURN count(e) AS IsChildOf_c;")}})
 
 (defn summarize
   "Return node/edge counts from the graph database."

backend/src/app/http/debug.clj

@@ -21,6 +21,7 @@
    [app.config :as cf]
    [app.db :as db]
    [app.features.file-migrations :as feat.fmig]
+   [app.graph.debug :as graph.debug]
    [app.graph.ingest :as graph.ingest]
    [app.http.session :as session]
    [app.rpc.commands.auth :as auth]
@@ -354,6 +355,52 @@
        ::yres/headers {"content-type" "application/octet-stream"
                        "content-disposition" (str "attachment; filename=" file-id ".lbug")}})))
 
+(defn- graph-console-response
+  [profile-id data]
+  {::yres/status  200
+   ::yres/headers {"content-type" "text/html; charset=utf-8"
+                   "x-robots-tag" "noindex"}
+   ::yres/body    (-> (io/resource "app/templates/graph-console.tmpl")
+                      (tmpl/render (assoc data :version (:full cf/version))))})
+
+(defn graph-console-handler
+  [_cfg {:keys [::session/profile-id]}]
+  (graph-console-response profile-id
+                          (graph.debug/console-context profile-id)))
+
+(defn graph-load-handler
+  [cfg {:keys [params ::session/profile-id]}]
+  (let [file-id (some-> (:file-id params) parse-uuid)]
+    (when-not file-id
+      (ex/raise :type :validation
+                :code :missing-arguments
+                :hint "missing file-id"))
+    (graph.debug/load-session! cfg profile-id file-id)
+    {::yres/status  302
+     ::yres/headers {"location" "/dbg/graph"}}))
+
+(defn graph-unload-handler
+  [_cfg {:keys [::session/profile-id]}]
+  (graph.debug/unload-session! profile-id)
+  {::yres/status  302
+   ::yres/headers {"location" "/dbg/graph"}})
+
+(defn graph-query-handler
+  [_cfg {:keys [params ::session/profile-id]}]
+  (let [query (:query params)]
+    (try
+      (let [result (graph.debug/query-session! profile-id query)]
+        (graph-console-response profile-id
+                                (graph.debug/console-context profile-id
+                                                             :query query
+                                                             :query-result result)))
+      (catch Throwable e
+        (graph-console-response profile-id
+                                (graph.debug/console-context profile-id
+                                                             :query query
+                                                             :error (or (:hint (ex-data e))
+                                                                        (ex-message e))))))))
+
 (defn import-handler
   [{:keys [::db/pool] :as cfg} {:keys [params ::session/profile-id] :as request}]
   (when-not (contains? params :file)
@@ -591,6 +638,7 @@
     ["" {:handler (partial index-handler cfg)}]
     ["/health" {:handler (partial health-handler cfg)}]
     ["/changelog" {:handler (partial changelog-handler cfg)}]
+    ["/graph" {:handler (partial graph-console-handler cfg)}]
     ["/error/:id" {:handler (partial error-handler cfg)}]
     ["/error" {:handler (partial error-list-handler cfg)}]
     ["/actions" {:middleware [[errors]]}
@@ -602,6 +650,9 @@
       {:handler (partial handle-team-features cfg)}]
      ["/file-export" {:handler (partial export-handler cfg)}]
      ["/graph-export" {:handler (partial graph-export-handler cfg)}]
+     ["/graph-load" {:handler (partial graph-load-handler cfg)}]
+     ["/graph-query" {:handler (partial graph-query-handler cfg)}]
+     ["/graph-unload" {:handler (partial graph-unload-handler cfg)}]
      ["/file-import" {:handler (partial import-handler cfg)}]
      ["/file-raw-export-import" {:handler (partial raw-export-import-handler cfg)}]]]])