mirror of
https://github.com/penpot/penpot.git
synced 2026-09-19 02:16:14 +00:00
76763ddd6d
The bundle and docker-image build/dedup checks used different cache keys: the bundle was cached by ref name (`penpot-<gh_ref>.zip`) while the docker image marker was cached by commit sha (`markers/images-sha-<sha>`). A tag built from a commit already promoted under another ref (e.g. `develop`) would rebuild the bundle unnecessarily, while `build-docker`'s `promote` job silently inherited the skip from `build` and never created that ref's branch tags (`backend:<gh_ref>`, `frontend:<gh_ref>`, ...), even though the underlying sha-tagged images already existed. - Key the bundle S3 object by commit sha (`penpot-sha-<sha>.zip`) instead of by ref name, matching the docker marker's semantics. - Drop the S3 metadata round-trip for `bundle_version` in build-docker.yml; compute it locally with `git describe`, same as build-bundle.yml (requires fetch-depth: 0 on that checkout). - Split `promote` into two mutually-exclusive jobs, `promote` (needs `build` to succeed) and `retag` (needs only `prepare`, runs when `prepare.outputs.exists == 'true'`), each moving the `:<gh_ref>` branch tags to the current sha. This replaces relying on `build`'s skip/success state with two explicit conditions, so the tags always get moved regardless of which path built the images. Signed-off-by: David Barragán Merino <david.barragan@kaleidos.net>
357 lines
14 KiB
YAML
357 lines
14 KiB
YAML
name: Docker Images Builder
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
gh_ref:
|
|
description: 'Name of the branch or ref'
|
|
type: string
|
|
required: true
|
|
default: 'develop'
|
|
force:
|
|
description: 'Rebuild and overwrite even if this sha is already promoted'
|
|
type: boolean
|
|
required: false
|
|
default: false
|
|
workflow_call:
|
|
inputs:
|
|
gh_ref:
|
|
description: 'Name of the branch or ref'
|
|
type: string
|
|
required: true
|
|
default: 'develop'
|
|
force:
|
|
description: 'Rebuild and overwrite even if this sha is already promoted'
|
|
type: boolean
|
|
required: false
|
|
default: false
|
|
|
|
# Literal group name: under `workflow_call`, `github.workflow` resolves to the
|
|
# caller's workflow, which put this workflow and the other reusable one called
|
|
# by the same caller into a single shared group, and left a manual dispatch of
|
|
# the same ref in a group of its own, free to race on the same artifacts.
|
|
concurrency:
|
|
group: build-docker-${{ inputs.gh_ref }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
ALL_IMAGES: backend frontend exporter storybook mcp
|
|
# All runner instances live on the same server, so the bundle is
|
|
# downloaded from S3 once and shared between build jobs through this
|
|
# host-local directory. Each build job falls back to S3 if the file is
|
|
# missing (e.g. if runners ever move to separate machines).
|
|
BUNDLE_CACHE: /var/tmp/penpot-bundle-cache
|
|
|
|
jobs:
|
|
# ── 1. Resolve the build key and check the whole set at once ───────────
|
|
prepare:
|
|
name: Prepare
|
|
runs-on: penpot-standar-runner
|
|
timeout-minutes: 15
|
|
outputs:
|
|
gh_ref: ${{ steps.vars.outputs.gh_ref }}
|
|
bundle_version: ${{ steps.vars.outputs.bundle_version }}
|
|
sha: ${{ steps.vars.outputs.sha }}
|
|
commit_title: ${{ steps.vars.outputs.commit_title }}
|
|
exists: ${{ steps.check.outputs.exists }}
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ inputs.gh_ref }}
|
|
|
|
- name: Extract some useful variables
|
|
id: vars
|
|
run: |
|
|
GH_REF="${{ inputs.gh_ref || github.ref_name }}"
|
|
echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT
|
|
echo "sha=$(git rev-parse --short=12 HEAD)" >> $GITHUB_OUTPUT
|
|
echo "commit_title=$(git log -1 --pretty=%s)" >> $GITHUB_OUTPUT
|
|
echo "bundle_version=$(git describe --tags --always)" >> $GITHUB_OUTPUT
|
|
|
|
# The image set is a single block, so a single set-level check is
|
|
# enough: `promote` drops a marker object in S3 only after every
|
|
# image was built AND every branch tag was moved. Marker present
|
|
# means there is nothing at all to do for this commit. `force`
|
|
# bypasses this check entirely.
|
|
- name: Check if this image set is already built
|
|
id: check
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
|
|
run: |
|
|
if [ "${{ inputs.force }}" = "true" ]; then
|
|
echo "exists=false" >> $GITHUB_OUTPUT
|
|
mkdir -p "$BUNDLE_CACHE"
|
|
find "$BUNDLE_CACHE" -type f -mtime +1 -delete || true
|
|
ZIP="$BUNDLE_CACHE/penpot-sha-${{ steps.vars.outputs.sha }}.zip"
|
|
aws s3 cp "s3://${{ secrets.S3_BUCKET }}/penpot-sha-${{ steps.vars.outputs.sha }}.zip" "$ZIP.$$.tmp"
|
|
mv "$ZIP.$$.tmp" "$ZIP"
|
|
{
|
|
echo "### 🔁 Image set build forced"
|
|
echo ""
|
|
echo "\`force: true\` — skipping the S3 marker check."
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
exit 0
|
|
fi
|
|
|
|
if aws s3api head-object \
|
|
--bucket ${{ secrets.S3_BUCKET }} \
|
|
--key "markers/images-sha-${{ steps.vars.outputs.sha }}" \
|
|
> /dev/null 2>&1; then
|
|
echo "exists=true" >> $GITHUB_OUTPUT
|
|
{
|
|
echo "### ⏭️ Image set build skipped"
|
|
echo ""
|
|
echo "The whole set was already built and promoted for \`sha-${{ steps.vars.outputs.sha }}\`."
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
else
|
|
echo "exists=false" >> $GITHUB_OUTPUT
|
|
|
|
# Stage the bundle in the host-local cache, once, for all the
|
|
# build jobs. Download to a temp name and mv for atomicity;
|
|
# prune stale bundles while at it.
|
|
mkdir -p "$BUNDLE_CACHE"
|
|
find "$BUNDLE_CACHE" -type f -mtime +1 -delete || true
|
|
ZIP="$BUNDLE_CACHE/penpot-sha-${{ steps.vars.outputs.sha }}.zip"
|
|
if [ ! -f "$ZIP" ]; then
|
|
aws s3 cp "s3://${{ secrets.S3_BUCKET }}/penpot-sha-${{ steps.vars.outputs.sha }}.zip" "$ZIP.$$.tmp"
|
|
mv "$ZIP.$$.tmp" "$ZIP"
|
|
fi
|
|
fi
|
|
|
|
# ── 2. One build per image, in parallel, only when needed ──────────────
|
|
build:
|
|
name: Build ${{ matrix.image }}
|
|
runs-on: penpot-standar-runner
|
|
timeout-minutes: 60
|
|
needs: prepare
|
|
if: needs.prepare.outputs.exists == 'false'
|
|
strategy:
|
|
fail-fast: true
|
|
# 4 runner slots are available for build jobs on this server; cap the
|
|
# matrix at 3 so short jobs (prepare and other workflows' checks)
|
|
# never queue behind long builds.
|
|
max-parallel: 3
|
|
matrix:
|
|
image: [backend, frontend, exporter, storybook, mcp]
|
|
|
|
steps:
|
|
- name: Set common environment variables
|
|
run: |
|
|
# Each job execution will use its own docker configuration.
|
|
echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}-${{ matrix.image }}" >> $GITHUB_ENV
|
|
|
|
- name: Checkout code
|
|
uses: actions/checkout@v6
|
|
with:
|
|
ref: ${{ inputs.gh_ref }}
|
|
|
|
- name: Login to Docker Registry
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: ${{ secrets.DOCKER_REGISTRY }}
|
|
username: ${{ secrets.DOCKER_USERNAME }}
|
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
|
|
|
# To avoid the "429 Too Many Requests" error when downloading
|
|
# images from DockerHub for unregistered users.
|
|
# https://docs.docker.com/docker-hub/usage/
|
|
- name: Login to DockerHub Registry
|
|
uses: docker/login-action@v4
|
|
with:
|
|
username: ${{ secrets.PUB_DOCKER_USERNAME }}
|
|
password: ${{ secrets.PUB_DOCKER_PASSWORD }}
|
|
|
|
# Images now build FROM Docker Hardened Images (dhi.io). DHI
|
|
# is free (Apache 2.0, no subscription), but pulling from it
|
|
# still requires an authenticated login -- a separate `docker
|
|
# login` against a different registry host, even though it
|
|
# reuses the same PUB_DOCKER_* credentials as the DockerHub
|
|
# login above.
|
|
- name: Login to Docker Hardened Images registry (base image pull)
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: dhi.io
|
|
username: ${{ secrets.PUB_DOCKER_USERNAME }}
|
|
password: ${{ secrets.PUB_DOCKER_PASSWORD }}
|
|
|
|
# Bundle staged once by `prepare` on this host; the S3 fallback only
|
|
# triggers if the cache is unavailable (runners on another machine,
|
|
# cache pruned mid-run, ...).
|
|
- name: Prepare Penpot bundle
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
|
|
run: |
|
|
ZIP="$BUNDLE_CACHE/penpot-sha-${{ needs.prepare.outputs.sha }}.zip"
|
|
if [ ! -f "$ZIP" ]; then
|
|
echo "Bundle not found in host cache; falling back to S3."
|
|
mkdir -p "$BUNDLE_CACHE"
|
|
aws s3 cp "s3://${{ secrets.S3_BUCKET }}/penpot-sha-${{ needs.prepare.outputs.sha }}.zip" "$ZIP.$$.tmp"
|
|
mv "$ZIP.$$.tmp" "$ZIP"
|
|
fi
|
|
# Extract only the bundle this job needs.
|
|
pushd docker/images
|
|
unzip -q "$ZIP" "penpot/${{ matrix.image }}/*"
|
|
mv "penpot/${{ matrix.image }}" "bundle-${{ matrix.image }}"
|
|
popd
|
|
|
|
- name: Set up QEMU (stable)
|
|
uses: docker/setup-qemu-action@v4
|
|
with:
|
|
platforms: linux/amd64,linux/arm64
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v4
|
|
|
|
- name: Extract metadata (tags, labels)
|
|
id: meta
|
|
uses: docker/metadata-action@v6
|
|
with:
|
|
images: ${{ matrix.image }}
|
|
labels: |
|
|
bundle_version=${{ needs.prepare.outputs.bundle_version }}
|
|
|
|
- name: Build and push Docker image
|
|
uses: docker/build-push-action@v7
|
|
with:
|
|
context: ./docker/images/
|
|
file: ./docker/images/Dockerfile.${{ matrix.image }}
|
|
platforms: linux/amd64,linux/arm64
|
|
push: true
|
|
provenance: mode=max
|
|
sbom: true
|
|
# Immutable tag only; branch tags are moved atomically for the
|
|
# whole image set by the `promote` job.
|
|
tags: ${{ secrets.DOCKER_REGISTRY }}/${{ matrix.image }}:sha-${{ needs.prepare.outputs.sha }}
|
|
labels: ${{ steps.meta.outputs.labels }}
|
|
cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ matrix.image }}:buildcache
|
|
cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ matrix.image }}:buildcache,mode=max
|
|
|
|
# ── 3a. Move the branch tags of ALL images together (fresh build) ──────
|
|
promote:
|
|
name: Promote image set
|
|
runs-on: penpot-standar-runner
|
|
timeout-minutes: 10
|
|
needs: [prepare, build]
|
|
|
|
steps:
|
|
- name: Set common environment variables
|
|
run: |
|
|
echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v4
|
|
|
|
- name: Login to Docker Registry
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: ${{ secrets.DOCKER_REGISTRY }}
|
|
username: ${{ secrets.DOCKER_USERNAME }}
|
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
|
|
|
- name: Point branch tags to the new build key
|
|
run: |
|
|
set -e
|
|
for image in $ALL_IMAGES; do
|
|
docker buildx imagetools create \
|
|
-t "${{ secrets.DOCKER_REGISTRY }}/$image:${{ needs.prepare.outputs.gh_ref }}" \
|
|
"${{ secrets.DOCKER_REGISTRY }}/$image:sha-${{ needs.prepare.outputs.sha }}"
|
|
done
|
|
|
|
# The marker is written LAST: its presence certifies that all five
|
|
# images exist and all branch tags point to this build key.
|
|
- name: Write set-completed marker
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
|
|
run: |
|
|
echo "${{ github.run_id }}" | aws s3 cp - \
|
|
"s3://${{ secrets.S3_BUCKET }}/markers/images-sha-${{ needs.prepare.outputs.sha }}"
|
|
|
|
- name: Write step summary
|
|
run: |
|
|
{
|
|
echo "### ✅ Image set promoted"
|
|
echo ""
|
|
echo "- Version: \`${{ needs.prepare.outputs.bundle_version }}\` (\`git describe --tags --always\`)"
|
|
echo "- Commit: [\`${{ needs.prepare.outputs.sha }}\`](https://github.com/${{ github.repository }}/commit/${{ needs.prepare.outputs.sha }}) — ${{ needs.prepare.outputs.commit_title }}"
|
|
echo "- Built at: $(date -u +'%Y-%m-%d %H:%M:%S UTC')"
|
|
echo ""
|
|
echo "All \`:${{ needs.prepare.outputs.gh_ref }}\` tags now point to \`sha-${{ needs.prepare.outputs.sha }}\`."
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
# ── 3b. Skip path: make sure THIS ref's tags point to the existing
|
|
# images (another ref may have built and promoted the same commit
|
|
# first — the sha marker says the images exist, not that this ref's
|
|
# tags were ever created).
|
|
retag:
|
|
name: Ensure branch tags (already-built commit)
|
|
runs-on: penpot-standar-runner
|
|
timeout-minutes: 10
|
|
needs: prepare
|
|
if: needs.prepare.outputs.exists == 'true'
|
|
|
|
steps:
|
|
- name: Set common environment variables
|
|
run: |
|
|
echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v4
|
|
|
|
- name: Login to Docker Registry
|
|
uses: docker/login-action@v4
|
|
with:
|
|
registry: ${{ secrets.DOCKER_REGISTRY }}
|
|
username: ${{ secrets.DOCKER_USERNAME }}
|
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
|
|
|
- name: Point branch tags to the existing build key
|
|
run: |
|
|
set -e
|
|
for image in $ALL_IMAGES; do
|
|
docker buildx imagetools create \
|
|
-t "${{ secrets.DOCKER_REGISTRY }}/$image:${{ needs.prepare.outputs.gh_ref }}" \
|
|
"${{ secrets.DOCKER_REGISTRY }}/$image:sha-${{ needs.prepare.outputs.sha }}"
|
|
done
|
|
|
|
- name: Write step summary
|
|
run: |
|
|
{
|
|
echo "### ✅ Image set already built (branch tags ensured)"
|
|
echo ""
|
|
echo "- Version: \`${{ needs.prepare.outputs.bundle_version }}\` (\`git describe --tags --always\`)"
|
|
echo "- Commit: [\`${{ needs.prepare.outputs.sha }}\`](https://github.com/${{ github.repository }}/commit/${{ needs.prepare.outputs.sha }}) — ${{ needs.prepare.outputs.commit_title }}"
|
|
echo "- Checked at: $(date -u +'%Y-%m-%d %H:%M:%S UTC')"
|
|
echo ""
|
|
echo "All \`:${{ needs.prepare.outputs.gh_ref }}\` tags now point to \`sha-${{ needs.prepare.outputs.sha }}\`."
|
|
} >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
# ── 4. Single failure notification for the whole workflow ─────────────
|
|
notify:
|
|
name: Notify failure
|
|
runs-on: penpot-standar-runner
|
|
timeout-minutes: 5
|
|
needs: [prepare, build, promote, retag]
|
|
if: failure()
|
|
|
|
steps:
|
|
- name: Notify Mattermost
|
|
uses: mattermost/action-mattermost-notify@ae31bb6f9e26a54336e79696f108a2c91cf55b4e # v2.1.0
|
|
with:
|
|
MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }}
|
|
MATTERMOST_CHANNEL: bot-alerts-cicd
|
|
TEXT: |
|
|
❌ 🐳 *[PENPOT] Error building/promoting the penpot docker image set.*
|
|
📄 Triggered from ref: `${{ needs.prepare.outputs.gh_ref || inputs.gh_ref }}`
|
|
📦 Bundle: `${{ needs.prepare.outputs.bundle_version || 'n/a' }}`
|
|
🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
@infra
|