From 240e7575869088ead04b257f55f031c45952ce0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Tue, 17 Nov 2026 12:24:49 +0100 Subject: [PATCH] :bug: Size frontend nginx workers from the container CPU limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit worker_processes auto starts one worker per host CPU, ignoring the container CPU limit, and since worker_connections went from 2048 to 65535 each worker reserves about 30 MB at startup. A frontend limited to a few CPUs on a 64-core node needs close to 2 GB before serving a request and gets OOM killed. Default the worker count to the CPUs the container can use, its cpuset and its cgroup CPU quota rounded up, when they are fewer than the CPUs of the host, and keep auto otherwise. Make both settings configurable through PENPOT_HTTP_SERVER_WORKER_PROCESSES and PENPOT_HTTP_SERVER_WORKER_CONNECTIONS, and refuse to start on values that are not valid. Refs #13103 Signed-off-by: David Barragán Merino --- docker/images/files/nginx-entrypoint.sh | 39 ++++++++++++++++++++++++- docker/images/files/nginx.conf.template | 4 +-- docs/technical-guide/configuration.md | 21 +++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/docker/images/files/nginx-entrypoint.sh b/docker/images/files/nginx-entrypoint.sh index e76e581f1c..4132871f07 100644 --- a/docker/images/files/nginx-entrypoint.sh +++ b/docker/images/files/nginx-entrypoint.sh @@ -51,6 +51,42 @@ update_oidc_name /var/www/app/js/config.js ## Nginx Config ######################################### +# nginx's `auto` starts one worker per CPU of the host, not of the container, +# and each worker reserves the memory for all its connections at startup. A +# container limited to a few CPUs on a large node can run out of memory before +# serving a request, so size the workers from the CPUs the container can use +# (its cpuset, and its CPU quota rounded up) when they are fewer than the host's. +container_cpu_limit() { + local quota="" period="" cpus + cpus="$(nproc 2>/dev/null)" || return 1 + if [ -r /sys/fs/cgroup/cpu.max ]; then + read -r quota period < /sys/fs/cgroup/cpu.max + elif [ -r /sys/fs/cgroup/cpu/cpu.cfs_quota_us ]; then + quota="$(< /sys/fs/cgroup/cpu/cpu.cfs_quota_us)" + period="$(< /sys/fs/cgroup/cpu/cpu.cfs_period_us)" + fi + # "max" (cgroup v2) and -1 (cgroup v1) mean no quota, and 0 is not one. + if [[ "$quota" =~ ^[1-9][0-9]*$ && "$period" =~ ^[1-9][0-9]*$ ]]; then + quota=$(( (quota + period - 1) / period )) + (( quota < cpus )) && cpus=$quota + fi + (( cpus < $(nproc --all) )) || return 1 + echo "$cpus" +} + +export PENPOT_HTTP_SERVER_WORKER_PROCESSES=${PENPOT_HTTP_SERVER_WORKER_PROCESSES:-$(container_cpu_limit || echo auto)} +export PENPOT_HTTP_SERVER_WORKER_CONNECTIONS=${PENPOT_HTTP_SERVER_WORKER_CONNECTIONS:-65535} + +if [[ ! "$PENPOT_HTTP_SERVER_WORKER_PROCESSES" =~ ^(auto|[1-9][0-9]*)$ ]]; then + echo "penpot: invalid PENPOT_HTTP_SERVER_WORKER_PROCESSES '${PENPOT_HTTP_SERVER_WORKER_PROCESSES}'; expected auto or a positive integer" >&2 + exit 1 +fi + +if [[ ! "$PENPOT_HTTP_SERVER_WORKER_CONNECTIONS" =~ ^[1-9][0-9]*$ ]]; then + echo "penpot: invalid PENPOT_HTTP_SERVER_WORKER_CONNECTIONS '${PENPOT_HTTP_SERVER_WORKER_CONNECTIONS}'; expected a positive integer" >&2 + exit 1 +fi + export PENPOT_BACKEND_URI=${PENPOT_BACKEND_URI:-http://penpot-backend:6060} export PENPOT_EXPORTER_URI=${PENPOT_EXPORTER_URI:-http://penpot-exporter:6061} export PENPOT_HTTP_SERVER_MAX_BODY_SIZE=${PENPOT_HTTP_SERVER_MAX_BODY_SIZE:-367001600} # Default to 350MiB @@ -58,7 +94,8 @@ export PENPOT_IPV6_LISTEN_DIRECTIVE=${PENPOT_IPV6_LISTEN_DIRECTIVE:-"listen [::] if is_truthy "${PENPOT_DISABLE_IPV6_LISTEN:-}"; then export PENPOT_IPV6_LISTEN_DIRECTIVE="" fi -envsubst "\$PENPOT_BACKEND_URI,\$PENPOT_EXPORTER_URI,\$PENPOT_HTTP_SERVER_MAX_BODY_SIZE,\$PENPOT_IPV6_LISTEN_DIRECTIVE" \ + +envsubst "\$PENPOT_HTTP_SERVER_WORKER_PROCESSES,\$PENPOT_HTTP_SERVER_WORKER_CONNECTIONS,\$PENPOT_BACKEND_URI,\$PENPOT_EXPORTER_URI,\$PENPOT_HTTP_SERVER_MAX_BODY_SIZE,\$PENPOT_IPV6_LISTEN_DIRECTIVE" \ < /tmp/nginx.conf.template > /etc/nginx/nginx.conf if [[ " $PENPOT_FLAGS " != *" disable-admin-console "* ]]; then diff --git a/docker/images/files/nginx.conf.template b/docker/images/files/nginx.conf.template index 0451e7a24a..bae5030007 100644 --- a/docker/images/files/nginx.conf.template +++ b/docker/images/files/nginx.conf.template @@ -1,9 +1,9 @@ -worker_processes auto; +worker_processes ${PENPOT_HTTP_SERVER_WORKER_PROCESSES}; pid /tmp/nginx.pid; include /etc/nginx/overrides/main.d/*.conf; events { - worker_connections 65535; + worker_connections ${PENPOT_HTTP_SERVER_WORKER_CONNECTIONS}; multi_accept on; } diff --git a/docs/technical-guide/configuration.md b/docs/technical-guide/configuration.md index 37ea377bec..33dffaa0eb 100644 --- a/docs/technical-guide/configuration.md +++ b/docs/technical-guide/configuration.md @@ -957,6 +957,27 @@ PENPOT_INTERNAL_RESOLVER: 127.0.0.11 names. Defaults to the nameservers found in `/etc/resolv.conf`. `127.0.0.11` is the embedded Docker DNS server; use the address of your own resolver on other setups. +### Nginx workers + +The frontend container sizes its nginx workers from the CPUs it can use, its CPU quota or +its cpuset, when they are fewer than the CPUs of the host. Otherwise it starts one worker +per CPU of the host. Each worker reserves the memory for all its connections when it +starts, so tune these if the frontend uses more memory than you expect: + +```bash +# Frontend +PENPOT_HTTP_SERVER_WORKER_PROCESSES: 2 +PENPOT_HTTP_SERVER_WORKER_CONNECTIONS: 2048 +``` + +- `PENPOT_HTTP_SERVER_WORKER_PROCESSES`: Number of nginx worker processes, a positive + integer or `auto` for one per host CPU. Defaults to the CPUs the container can use, with + the quota rounded up, or `auto` when it has no limit. +- `PENPOT_HTTP_SERVER_WORKER_CONNECTIONS`: Maximum simultaneous connections per worker, + a positive integer. Defaults to `65535`, which takes about 30 MB of memory per worker. + +The container refuses to start if either value is not valid. + ## Other flags There are other flags that are useful for a more customized Penpot experience. This section has the list of the flags meant