From c8ec4cc159a667ac7c8446e90a5c13a384e8160a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Tejero=20Cantero?= Date: Sun, 26 Jul 2026 19:35:06 +0200 Subject: [PATCH] :sparkles: Add the Arrow prerequisites for in-memory bulk load MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lbug pulls arrow-memory-core and arrow-vector but no allocation-manager implementation, so RootAllocator cannot be constructed; arrow-memory-netty 18.2.0 matches the arrow-vector lbug already brings and pulls only netty-buffer, netty-common, jackson and slf4j-api, all of which the backend already has. --add-opens=java.base/java.nio=ALL-UNNAMED is the second half: without it MemoryUtil's static initializer dies with an InaccessibleObjectException that surfaces as an unhelpful NoClassDefFoundError from anything touching RootAllocator. It has to be present at JVM start, hence all three places. Note app.main/restart will not pick it up — it restarts integrant inside the same JVM, so the process must be restarted. Worth a reviewer's attention: this is a JVM-wide flag added for one subsystem. It is the standard Arrow requirement and grants nothing beyond reflective access to java.nio, but it strengthens the case for putting the whole graph subsystem behind a feature flag. --- backend/deps.edn | 8 ++++++-- backend/scripts/_env | 3 ++- backend/scripts/run.template.sh | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/deps.edn b/backend/deps.edn index 623eaab86b..9fd4b90e66 100644 --- a/backend/deps.edn +++ b/backend/deps.edn @@ -67,13 +67,17 @@ software.amazon.awssdk/s3 {:mvn/version "2.50.1"} software.amazon.awssdk/sts {:mvn/version "2.50.1"} - com.ladybugdb/lbug {:mvn/version "0.19.1"}} + com.ladybugdb/lbug {:mvn/version "0.19.1"} + ;; Required by Arrow RootAllocator (lbug only pulls arrow-memory-core). + org.apache.arrow/arrow-memory-netty {:mvn/version "18.2.0"}} :paths ["src" "resources" "target/classes"] :aliases {:dev {:jvm-opts ["--sun-misc-unsafe-memory-access=allow" - "--enable-native-access=ALL-UNNAMED"] + "--enable-native-access=ALL-UNNAMED" + ;; Arrow jars are on the classpath (unnamed module), not module-path. + "--add-opens=java.base/java.nio=ALL-UNNAMED"] :extra-deps {com.bhauman/rebel-readline {:mvn/version "0.1.11"} clojure-humanize/clojure-humanize {:mvn/version "0.2.2"} diff --git a/backend/scripts/_env b/backend/scripts/_env index 724b55f05b..5e4b02b80a 100644 --- a/backend/scripts/_env +++ b/backend/scripts/_env @@ -93,7 +93,8 @@ export JAVA_OPTS="\ -XX:-OmitStackTraceInFastThrow \ --sun-misc-unsafe-memory-access=allow \ --enable-preview \ - --enable-native-access=ALL-UNNAMED"; + --enable-native-access=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED"; function setup_minio() { if [ "${PENPOT_OBJECTS_STORAGE_BACKEND}" != "s3" ]; then diff --git a/backend/scripts/run.template.sh b/backend/scripts/run.template.sh index cff4afc870..19f47e6c0a 100644 --- a/backend/scripts/run.template.sh +++ b/backend/scripts/run.template.sh @@ -18,7 +18,7 @@ if [ -f ./environ ]; then source ./environ fi -export JAVA_OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED --enable-preview $JVM_OPTS $JAVA_OPTS" +export JAVA_OPTS="-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dlog4j2.configurationFile=log4j2.xml -XX:-OmitStackTraceInFastThrow --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --enable-preview $JVM_OPTS $JAVA_OPTS" ENTRYPOINT=${1:-app.main};