From 0915f894195df3d0a43552c014756217e974754e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Tue, 17 Nov 2026 12:24:49 +0100 Subject: [PATCH] :bug: Size frontend nginx workers from the container CPU limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit worker_processes auto starts one worker per host CPU, ignoring the container CPU limit, and since worker_connections went from 2048 to 65535 each worker reserves about 30 MB at startup. A frontend limited to a few CPUs on a 64-core node needs close to 2 GB before serving a request and gets OOM killed. Default the worker count to the CPUs the container can use, its cpuset and its cgroup CPU quota rounded up, when they are fewer than the CPUs of the host, and keep auto otherwise. Make both settings configurable through PENPOT_HTTP_SERVER_WORKER_PROCESSES and PENPOT_HTTP_SERVER_WORKER_CONNECTIONS, and refuse to start on values that are not valid. Refs #13103 Signed-off-by: David Barragán Merino --- docker/images/files/nginx-entrypoint.sh | 38 ++++++++++++++++++++++++- docker/images/files/nginx.conf.template | 4 +-- docs/technical-guide/configuration.md | 21 ++++++++++++++ 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/docker/images/files/nginx-entrypoint.sh b/docker/images/files/nginx-entrypoint.sh index f809591827..c6ad751c00 100644 --- a/docker/images/files/nginx-entrypoint.sh +++ b/docker/images/files/nginx-entrypoint.sh @@ -68,6 +68,42 @@ else export PENPOT_PUBLIC_PATH_DIRECTIVES="" fi +# nginx's `auto` starts one worker per CPU of the host, not of the container, +# and each worker reserves the memory for all its connections at startup. A +# container limited to a few CPUs on a large node can run out of memory before +# serving a request, so size the workers from the CPUs the container can use +# (its cpuset, and its CPU quota rounded up) when they are fewer than the host's. +container_cpu_limit() { + local quota="" period="" cpus + cpus="$(nproc 2>/dev/null)" || return 1 + if [ -r /sys/fs/cgroup/cpu.max ]; then + read -r quota period < /sys/fs/cgroup/cpu.max + elif [ -r /sys/fs/cgroup/cpu/cpu.cfs_quota_us ]; then + quota="$(< /sys/fs/cgroup/cpu/cpu.cfs_quota_us)" + period="$(< /sys/fs/cgroup/cpu/cpu.cfs_period_us)" + fi + # "max" (cgroup v2) and -1 (cgroup v1) mean no quota, and 0 is not one. + if [[ "$quota" =~ ^[1-9][0-9]*$ && "$period" =~ ^[1-9][0-9]*$ ]]; then + quota=$(( (quota + period - 1) / period )) + (( quota < cpus )) && cpus=$quota + fi + (( cpus < $(nproc --all) )) || return 1 + echo "$cpus" +} + +export PENPOT_HTTP_SERVER_WORKER_PROCESSES=${PENPOT_HTTP_SERVER_WORKER_PROCESSES:-$(container_cpu_limit || echo auto)} +export PENPOT_HTTP_SERVER_WORKER_CONNECTIONS=${PENPOT_HTTP_SERVER_WORKER_CONNECTIONS:-65535} + +if [[ ! "$PENPOT_HTTP_SERVER_WORKER_PROCESSES" =~ ^(auto|[1-9][0-9]*)$ ]]; then + echo "penpot: invalid PENPOT_HTTP_SERVER_WORKER_PROCESSES '${PENPOT_HTTP_SERVER_WORKER_PROCESSES}'; expected auto or a positive integer" >&2 + exit 1 +fi + +if [[ ! "$PENPOT_HTTP_SERVER_WORKER_CONNECTIONS" =~ ^[1-9][0-9]*$ ]]; then + echo "penpot: invalid PENPOT_HTTP_SERVER_WORKER_CONNECTIONS '${PENPOT_HTTP_SERVER_WORKER_CONNECTIONS}'; expected a positive integer" >&2 + exit 1 +fi + export PENPOT_BACKEND_URI=${PENPOT_BACKEND_URI:-http://penpot-backend:6060} export PENPOT_EXPORTER_URI=${PENPOT_EXPORTER_URI:-http://penpot-exporter:6061} export PENPOT_HTTP_SERVER_MAX_BODY_SIZE=${PENPOT_HTTP_SERVER_MAX_BODY_SIZE:-367001600} # Default to 350MiB @@ -75,7 +111,7 @@ export PENPOT_IPV6_LISTEN_DIRECTIVE=${PENPOT_IPV6_LISTEN_DIRECTIVE:-"listen [::] if is_truthy "${PENPOT_DISABLE_IPV6_LISTEN:-}"; then export PENPOT_IPV6_LISTEN_DIRECTIVE="" fi -envsubst "\$PENPOT_BACKEND_URI,\$PENPOT_EXPORTER_URI,\$PENPOT_HTTP_SERVER_MAX_BODY_SIZE,\$PENPOT_IPV6_LISTEN_DIRECTIVE,\$PENPOT_PUBLIC_PATH,\$PENPOT_PUBLIC_PATH_DIRECTIVES" \ +envsubst "\$PENPOT_HTTP_SERVER_WORKER_PROCESSES,\$PENPOT_HTTP_SERVER_WORKER_CONNECTIONS,\$PENPOT_BACKEND_URI,\$PENPOT_EXPORTER_URI,\$PENPOT_HTTP_SERVER_MAX_BODY_SIZE,\$PENPOT_IPV6_LISTEN_DIRECTIVE,\$PENPOT_PUBLIC_PATH,\$PENPOT_PUBLIC_PATH_DIRECTIVES" \ < /tmp/nginx.conf.template > /etc/nginx/nginx.conf if [[ " $PENPOT_FLAGS " != *" disable-admin-console "* ]]; then diff --git a/docker/images/files/nginx.conf.template b/docker/images/files/nginx.conf.template index 54d62e18eb..544d0fcdfd 100644 --- a/docker/images/files/nginx.conf.template +++ b/docker/images/files/nginx.conf.template @@ -1,9 +1,9 @@ -worker_processes auto; +worker_processes ${PENPOT_HTTP_SERVER_WORKER_PROCESSES}; pid /tmp/nginx.pid; include /etc/nginx/overrides/main.d/*.conf; events { - worker_connections 65535; + worker_connections ${PENPOT_HTTP_SERVER_WORKER_CONNECTIONS}; multi_accept on; } diff --git a/docs/technical-guide/configuration.md b/docs/technical-guide/configuration.md index 17e0914e15..5e9731badd 100644 --- a/docs/technical-guide/configuration.md +++ b/docs/technical-guide/configuration.md @@ -1076,6 +1076,27 @@ PENPOT_INTERNAL_RESOLVER: 127.0.0.11 names. Defaults to the nameservers found in `/etc/resolv.conf`. `127.0.0.11` is the embedded Docker DNS server; use the address of your own resolver on other setups. +### Nginx workers + +The frontend container sizes its nginx workers from the CPUs it can use, its CPU quota or +its cpuset, when they are fewer than the CPUs of the host. Otherwise it starts one worker +per CPU of the host. Each worker reserves the memory for all its connections when it +starts, so tune these if the frontend uses more memory than you expect: + +```bash +# Frontend +PENPOT_HTTP_SERVER_WORKER_PROCESSES: 2 +PENPOT_HTTP_SERVER_WORKER_CONNECTIONS: 2048 +``` + +- `PENPOT_HTTP_SERVER_WORKER_PROCESSES`: Number of nginx worker processes, a positive + integer or `auto` for one per host CPU. Defaults to the CPUs the container can use, with + the quota rounded up, or `auto` when it has no limit. +- `PENPOT_HTTP_SERVER_WORKER_CONNECTIONS`: Maximum simultaneous connections per worker, + a positive integer. Defaults to `65535`, which takes about 30 MB of memory per worker. + +The container refuses to start if either value is not valid. + ## Other flags There are other flags that are useful for a more customized Penpot experience. This section has the list of the flags meant