From 3f610ad793815de401354b2175d0700d2bf8d88e Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 1 Oct 2026 19:22:01 +0200 Subject: [PATCH] :sparkles: Isolate backend-test database per devenv instance (#12048) * :sparkles: Isolate backend-test database per devenv instance Each wsN now gets its own backend-test database (penpot_test_wsN) and Valkey DB (6+N) on the shared infra, so parallel test runs no longer wipe each other. The main database stays shared on purpose. manage.sh creates the test database on bring-up and passes PENPOT_TEST_* into the main container. Test helpers already read those vars, so no runtime code changes. scripts/psql and db-schema gain --ws. AI-assisted-by: muse-spark-1.3-contributor-free * :bug: Fix per-instance test database creation on bring-up psql -c does not reliably mix SQL with psql-only commands, so the CREATE DATABASE ... WHERE NOT EXISTS ... \gexec one-liner never ran. Use two plain SQL round-trips instead (check pg_database, then CREATE DATABASE) and print what happens, so a future failure shows up in the bring-up output. AI-assisted-by: muse-spark-1.3-contributor-free --- .serena/memories/devenv/core.md | 5 +- .serena/memories/scripts/psql.md | 9 +++- backend/test/backend_tests/helpers.clj | 7 +++ docker/devenv/docker-compose.main.yml | 5 ++ manage.sh | 73 ++++++++++++++++++++++++++ scripts/db-schema | 31 +++++++++++ scripts/psql | 31 +++++++++++ 7 files changed, 158 insertions(+), 3 deletions(-) diff --git a/.serena/memories/devenv/core.md b/.serena/memories/devenv/core.md index ddc83b1845..7bd436221a 100644 --- a/.serena/memories/devenv/core.md +++ b/.serena/memories/devenv/core.md @@ -1,6 +1,6 @@ # Devenv startup and configuration -Compose-based dev environment under `docker/devenv/`, driven by `manage.sh`. Parallel instances share infra + Postgres + RustFS; each instance has its own `main` container, Valkey, source checkout, tmux session. +Compose-based dev environment under `docker/devenv/`, driven by `manage.sh`. Parallel instances share infra + Postgres + RustFS; each instance has its own `main` container, Valkey, source checkout, tmux session. The main database (`penpot`) and the S3 bucket (`penpot`) are shared by all instances on purpose; backend-test databases are isolated per instance (`penpot_test` on ws0, `penpot_test_wsN` on wsN) with per-instance Valkey DB numbers (`6+N`, since runtime backends occupy `0..5`). ## Compose project layout @@ -12,7 +12,8 @@ Compose-based dev environment under `docker/devenv/`, driven by `manage.sh`. Par ## Source-of-truth files - `docker/devenv/defaults.env`: ws0 baseline — container/volume names, runtime env, published host ports, tmux defaults. `manage.sh` aborts if unreadable. -- For ws1+, `instance-env-overrides` computes the per-instance overrides (container/volume names, host ports offset `10000·N`, `PENPOT_PUBLIC_URI`, `PENPOT_REDIS_URI`, `PENPOT_BACKEND_WORKER=false`) and `instance-compose` injects them as env vars at compose time — never written to disk, recomputed each call so they can't drift. ws0 uses `defaults.env` as-is. +- For ws1+, `instance-env-overrides` computes the per-instance overrides (container/volume names, host ports offset `10000·N`, `PENPOT_PUBLIC_URI`, `PENPOT_REDIS_URI`, `PENPOT_TEST_DATABASE_URI`, `PENPOT_TEST_REDIS_URI`) and `instance-compose` injects them as env vars at compose time — never written to disk, recomputed each call so they can't drift. ws0 uses `defaults.env` as-is. +- Per-instance test databases are created by `ensure-instance-test-database` (called from `start-instance`): `CREATE DATABASE` guarded by a `pg_database` existence check, so it is idempotent. Needed because `postgresql_init.sql` only runs on first volume creation. The backend applies migrations itself on first use, so an empty database is enough. - `backend/scripts/_env`: backend-internal only — secret keys, `PENPOT_FLAGS` (with `enable-backend-worker` gated on `PENPOT_BACKEND_WORKER`), `JAVA_OPTS`, `setup_s3_bucket()`. Never duplicates `defaults.env`. - Compose files use pure `${VAR}` substitution; missing var = compose fails. diff --git a/.serena/memories/scripts/psql.md b/.serena/memories/scripts/psql.md index 810905f7c6..8b527cb0c6 100644 --- a/.serena/memories/scripts/psql.md +++ b/.serena/memories/scripts/psql.md @@ -16,13 +16,20 @@ development. # Default connection (penpot db, localhost) scripts/psql -c "SELECT version();" -# Test database +# Test database (ws0) scripts/psql --test -c "SELECT * FROM migrations;" +# Isolated test database of another instance (ws1+) +scripts/psql --test --ws 1 -c "SELECT * FROM migrations;" + # Custom host/user/database scripts/psql --host myhost --user myuser --db mydb ``` +`--ws N` only applies together with `--test` (the main database is +shared by all instances): it selects `penpot_test` on `--ws 0` and +`penpot_test_wsN` on `--ws N`. `scripts/db-schema` accepts the same flags. + `scripts/psql` must be invoked from the repo root so the path resolves. ## Native Tool Available (OpenCode V2) diff --git a/backend/test/backend_tests/helpers.clj b/backend/test/backend_tests/helpers.clj index b41bde7770..1ee446af6a 100644 --- a/backend/test/backend_tests/helpers.clj +++ b/backend/test/backend_tests/helpers.clj @@ -60,6 +60,13 @@ (def ^:dynamic *system* nil) (def ^:dynamic *pool* nil) +;; Fallback values used when no PENPOT_TEST_* env vars are set (e.g. CI sets +;; them explicitly, see .github/workflows/tests-backend.yml). Inside the +;; devenv each wsN container receives per-instance values via +;; manage.sh (PENPOT_TEST_DATABASE_URI=postgresql://postgres/penpot_test_wsN, +;; PENPOT_TEST_REDIS_URI=redis://valkey/<6+N>), so parallel test runs on +;; different workspaces never share a database. cf/read-config picks the env +;; vars up through the "penpot-test" prefix. (def default {:database-uri "postgresql://postgres/penpot_test" :redis-uri "redis://valkey/1" diff --git a/docker/devenv/docker-compose.main.yml b/docker/devenv/docker-compose.main.yml index e593ad1ba7..491c76b729 100644 --- a/docker/devenv/docker-compose.main.yml +++ b/docker/devenv/docker-compose.main.yml @@ -81,6 +81,11 @@ services: - PENPOT_DATABASE_PASSWORD=${PENPOT_DATABASE_PASSWORD} - PENPOT_DATABASE_MAX_POOL_SIZE=${PENPOT_DATABASE_MAX_POOL_SIZE} - PENPOT_REDIS_URI=${PENPOT_REDIS_URI} + # Per-instance backend-test config. Values come from + # instance-env-overrides in manage.sh; the main database above stays + # shared by all instances on purpose. + - PENPOT_TEST_DATABASE_URI=${PENPOT_TEST_DATABASE_URI} + - PENPOT_TEST_REDIS_URI=${PENPOT_TEST_REDIS_URI} - PENPOT_OBJECTS_STORAGE_BACKEND=${PENPOT_OBJECTS_STORAGE_BACKEND} - PENPOT_OBJECTS_STORAGE_S3_ENDPOINT=${PENPOT_OBJECTS_STORAGE_S3_ENDPOINT} - PENPOT_OBJECTS_STORAGE_S3_BUCKET=${PENPOT_OBJECTS_STORAGE_S3_BUCKET} diff --git a/manage.sh b/manage.sh index 0cbc4e7cd7..408792d781 100755 --- a/manage.sh +++ b/manage.sh @@ -94,6 +94,9 @@ set -e # instance-compose wrap 'docker compose' for one instance's main # project, injecting that instance's overrides # instance-env-overrides the per-instance KEY=VALUE overrides +# instance-test-db-name test database name for one instance +# instance-test-redis-db Valkey DB number for one instance's tests +# ensure-instance-test-database create one instance's test database # devenv-main-container resolve the 'main' container id via compose ps # devenv-main-running true if 'main' is up # @@ -301,6 +304,43 @@ function ensure-infra-up { infra-compose up -d --wait --wait-timeout 60 --remove-orphans } +# Create the backend-test database for on the shared Postgres +# if it does not exist yet. Idempotent: safe to run on every bring-up. +# Needed because postgresql_init.sql only runs when the Postgres volume is +# created for the first time, so per-instance test databases (which did not +# exist back then) would otherwise never be created on existing volumes. +# The backend applies migrations itself on first use, so an empty database +# is enough here. (Two plain SQL round-trips on purpose: psql -c does not +# reliably mix SQL with psql-only commands like \gexec.) +function ensure-instance-test-database { + local instance="$1" + local dbname + dbname=$(instance-test-db-name "$instance") + + local pg_container + pg_container=$(infra-compose ps -q postgres 2>/dev/null) + if [[ -z "$pg_container" ]]; then + echo "[${instance}] postgres container not found; is shared infra up?" >&2 + return 1 + fi + + local -a psql_base=(docker exec + -e "PGPASSWORD=${PENPOT_DATABASE_PASSWORD:-penpot}" + "$pg_container" + psql -h 127.0.0.1 -U "${PENPOT_DATABASE_USERNAME:-penpot}" + -d postgres -v ON_ERROR_STOP=1 -tA) + + local exists + exists=$("${psql_base[@]}" -c "SELECT 1 FROM pg_database WHERE datname = '${dbname}'") + if [[ "$exists" == "1" ]]; then + echo "[${instance}] test database ${dbname} already exists." + return 0 + fi + + echo "[${instance}] creating test database ${dbname} ..." + "${psql_base[@]}" -c "CREATE DATABASE \"${dbname}\"" +} + # Refuse to sync workspaces if the live repo is in a fragile Git state. # Copying a partial rebase/merge/cherry-pick into all workspaces would leave # every instance in the same broken state. @@ -333,6 +373,33 @@ function instance-port { echo $(( base + n * PENPOT_INSTANCE_PORT_STRIDE )) } +# Echo the backend-test database name for . ws0 keeps the +# historical name so existing flows stay untouched; ws1+ get an isolated +# database each on the shared Postgres server. The main database stays +# shared by all instances on purpose. +function instance-test-db-name { + local instance="$1" + local n=0 + [[ "$instance" =~ ^ws([0-9]+)$ ]] && n="${BASH_REMATCH[1]}" + if (( n == 0 )); then + echo "penpot_test" + else + echo "penpot_test_ws${n}" + fi +} + +# Echo the Valkey DB number reserved for the backend tests of . +# Runtime backends already occupy 0..PENPOT_MAX_WS_INDEX on the shared +# Valkey, so test DBs start right after to never share a DB with a +# running backend. (Resolved at call time; PENPOT_MAX_WS_INDEX is defined +# further below but always set before any call.) +function instance-test-redis-db { + local instance="$1" + local n=0 + [[ "$instance" =~ ^ws([0-9]+)$ ]] && n="${BASH_REMATCH[1]}" + echo $(( PENPOT_MAX_WS_INDEX + 1 + n )) +} + # Echo the per-instance Compose variable overrides for a workspace, one # KEY=VALUE per line, for instance-compose to inject into its `env -i` line. # Compose gives shell-env precedence over --env-file, so these override the @@ -358,11 +425,15 @@ function instance-env-overrides { opencode=$(instance-port "$instance" "$PENPOT_PORT_BASE_OPENCODE") mdts=$(instance-port "$instance" "$PENPOT_PORT_BASE_MDTS") storybook=$(instance-port "$instance" "$PENPOT_PORT_BASE_STORYBOOK") + test_db=$(instance-test-db-name "$instance") + test_redis_db=$(instance-test-redis-db "$instance") printf '%s\n' \ "PENPOT_MAIN_CONTAINER_NAME=penpot-devenv-${instance}-main" \ "PENPOT_USER_DATA_VOLUME=penpotdev_${instance}_user_data" \ "PENPOT_PUBLIC_URI=https://localhost:${public_https}" \ "PENPOT_REDIS_URI=redis://valkey/${n}" \ + "PENPOT_TEST_DATABASE_URI=postgresql://postgres/${test_db}" \ + "PENPOT_TEST_REDIS_URI=redis://valkey/${test_redis_db}" \ "PENPOT_PUBLIC_HTTPS_PORT=${public_https}" \ "PENPOT_PUBLIC_HTTP_PORT=${public}" \ "PENPOT_MCP_SERVER_PORT=${mcp}" \ @@ -709,6 +780,8 @@ function start-instance { local git_user_email="${4:-}" local agentic="${5:-true}" + ensure-instance-test-database "$instance" + instance-compose "$instance" up -d main # Wait briefly for main to be reachable; the tmux session lives inside. diff --git a/scripts/db-schema b/scripts/db-schema index 6cf2a8c710..3557cc859e 100755 --- a/scripts/db-schema +++ b/scripts/db-schema @@ -5,11 +5,22 @@ HOST="${PENPOT_DB_HOST:-postgres}" USER="${PENPOT_DB_USER:-penpot}" PASSWORD="${PENPOT_DB_PASSWORD:-penpot}" DB="${PENPOT_DB_NAME:-penpot}" +TEST=false +WS="" while [[ $# -gt 0 ]]; do case "$1" in --test) DB="penpot_test" + TEST=true + shift + ;; + --ws) + WS="$2" + shift 2 + ;; + --ws*) + WS="${1#--ws}" shift ;; --host|-h) @@ -30,5 +41,25 @@ while [[ $# -gt 0 ]]; do esac done +# --ws selects one instance's isolated backend-test database +# (penpot_test on ws0, penpot_test_wsN on wsN). It only applies together +# with --test because the main database is shared by all instances. +if [[ -n "$WS" ]]; then + if [[ "$WS" =~ ^ws([0-9]+)$ ]]; then + WS="${BASH_REMATCH[1]}" + fi + if [[ ! "$WS" =~ ^[0-9]+$ ]]; then + echo "Invalid --ws value: '$WS' (expected a non-negative integer, e.g. --ws 1)" >&2 + exit 1 + fi + if [[ "$TEST" != "true" ]]; then + echo "--ws only applies together with --test (the main database is shared by all instances)." >&2 + exit 1 + fi + if [[ "$WS" != "0" ]]; then + DB="penpot_test_ws${WS}" + fi +fi + export PGPASSWORD="$PASSWORD" exec pg_dump -h "$HOST" -U "$USER" -d "$DB" --schema-only --no-owner --no-privileges "$@" diff --git a/scripts/psql b/scripts/psql index 03dee3be19..3efac9c591 100755 --- a/scripts/psql +++ b/scripts/psql @@ -5,11 +5,22 @@ HOST="${PENPOT_DB_HOST:-postgres}" USER="${PENPOT_DB_USER:-penpot}" PASSWORD="${PENPOT_DB_PASSWORD:-penpot}" DB="${PENPOT_DB_NAME:-penpot}" +TEST=false +WS="" while [[ $# -gt 0 ]]; do case "$1" in --test) DB="penpot_test" + TEST=true + shift + ;; + --ws) + WS="$2" + shift 2 + ;; + --ws*) + WS="${1#--ws}" shift ;; --host|-h) @@ -30,5 +41,25 @@ while [[ $# -gt 0 ]]; do esac done +# --ws selects one instance's isolated backend-test database +# (penpot_test on ws0, penpot_test_wsN on wsN). It only applies together +# with --test because the main database is shared by all instances. +if [[ -n "$WS" ]]; then + if [[ "$WS" =~ ^ws([0-9]+)$ ]]; then + WS="${BASH_REMATCH[1]}" + fi + if [[ ! "$WS" =~ ^[0-9]+$ ]]; then + echo "Invalid --ws value: '$WS' (expected a non-negative integer, e.g. --ws 1)" >&2 + exit 1 + fi + if [[ "$TEST" != "true" ]]; then + echo "--ws only applies together with --test (the main database is shared by all instances)." >&2 + exit 1 + fi + if [[ "$WS" != "0" ]]; then + DB="penpot_test_ws${WS}" + fi +fi + export PGPASSWORD="$PASSWORD" exec psql -h "$HOST" -U "$USER" -d "$DB" "$@"