diff --git a/.github/workflows/build-bundle.yml b/.github/workflows/build-bundle.yml index 441587eea3..e785f2c84e 100644 --- a/.github/workflows/build-bundle.yml +++ b/.github/workflows/build-bundle.yml @@ -9,16 +9,6 @@ on: type: string required: true default: 'develop' - build_wasm: - description: 'BUILD_WASM. Valid values: yes, no' - type: string - required: false - default: 'yes' - build_storybook: - description: 'BUILD_STORYBOOK. Valid values: yes, no' - type: string - required: false - default: 'yes' workflow_call: inputs: gh_ref: @@ -26,29 +16,21 @@ on: type: string required: true default: 'develop' - build_wasm: - description: 'BUILD_WASM. Valid values: yes, no' - type: string - required: false - default: 'yes' - build_storybook: - description: 'BUILD_STORYBOOK. Valid values: yes, no' - type: string - required: false - default: 'yes' concurrency: group: ${{ github.workflow }}-${{ inputs.gh_ref }} cancel-in-progress: true jobs: - build-bundle: - name: Build and Upload Penpot Bundle + # ── 1. Decide whether there is anything to build ─────────────────────── + check: + name: Check current bundle runs-on: penpot-runner-01 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + timeout-minutes: 10 + outputs: + gh_ref: ${{ steps.vars.outputs.gh_ref }} + bundle_version: ${{ steps.vars.outputs.bundle_version }} + exists: ${{ steps.check.outputs.exists }} steps: - name: Checkout repository @@ -63,10 +45,52 @@ jobs: echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT echo "bundle_version=$(git describe --tags --always)" >> $GITHUB_OUTPUT + # The uploaded zip carries its version as S3 metadata. If the + # existing object was already built from this same commit, the + # whole build job is skipped. + - name: Check if this bundle is already built + id: check + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + run: | + EXISTING_VERSION=$(aws s3api head-object \ + --bucket ${{ secrets.S3_BUCKET }} \ + --key "penpot-${{ steps.vars.outputs.gh_ref }}.zip" \ + --query 'Metadata."bundle-version"' \ + --output text 2>/dev/null || echo "none") + + if [ "$EXISTING_VERSION" = "${{ steps.vars.outputs.bundle_version }}" ]; then + echo "exists=true" >> $GITHUB_OUTPUT + { + echo "### ⏭️ Bundle build skipped" + echo "" + echo "The bundle in S3 was already built from \`${{ steps.vars.outputs.bundle_version }}\`." + } >> "$GITHUB_STEP_SUMMARY" + else + echo "exists=false" >> $GITHUB_OUTPUT + fi + + # ── 2. Build and upload, only when needed ────────────────────────────── + build: + name: Build and Upload Penpot Bundle + runs-on: penpot-runner-01 + timeout-minutes: 90 + needs: check + if: needs.check.outputs.exists == 'false' + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ inputs.gh_ref }} + - name: Build bundle env: - BUILD_WASM: ${{ inputs.build_wasm }} - BUILD_STORYBOOK: ${{ inputs.build_storybook }} + BUILD_WASM: 'yes' + BUILD_STORYBOOK: 'yes' run: ./manage.sh build-bundle - name: Prepare directories for zipping @@ -80,18 +104,32 @@ jobs: zip -r zips/penpot.zip penpot - name: Upload Penpot bundle to S3 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} run: | - aws s3 cp zips/penpot.zip s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.gh_ref }}.zip --metadata bundle-version=${{ steps.vars.outputs.bundle_version }} + aws s3 cp zips/penpot.zip \ + s3://${{ secrets.S3_BUCKET }}/penpot-${{ needs.check.outputs.gh_ref }}.zip \ + --metadata bundle-version=${{ needs.check.outputs.bundle_version }} + # ── 3. Single failure notification for the whole workflow ───────────── + notify: + name: Notify failure + runs-on: penpot-runner-01 + timeout-minutes: 5 + needs: [check, build] + if: failure() + + steps: - name: Notify Mattermost - if: failure() uses: mattermost/action-mattermost-notify@ae31bb6f9e26a54336e79696f108a2c91cf55b4e # v2.1.0 with: MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }} MATTERMOST_CHANNEL: bot-alerts-cicd TEXT: | ❌ 📦 *[PENPOT] Error building penpot bundles.* - 📄 Triggered from ref: `${{ steps.vars.outputs.gh_ref }}` - Bundle version: `${{ steps.vars.outputs.bundle_version }}` + 📄 Triggered from ref: `${{ needs.check.outputs.gh_ref || inputs.gh_ref }}` + Bundle version: `${{ needs.check.outputs.bundle_version || 'n/a' }}` 🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} @infra diff --git a/.github/workflows/build-develop.yml b/.github/workflows/build-develop.yml index 559c18ef32..2da1ab2a31 100644 --- a/.github/workflows/build-develop.yml +++ b/.github/workflows/build-develop.yml @@ -11,8 +11,6 @@ jobs: secrets: inherit with: gh_ref: "develop" - build_wasm: "yes" - build_storybook: "yes" build-docker: needs: build-bundle diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 56940f4bff..0d03490194 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -20,16 +20,25 @@ concurrency: group: ${{ github.workflow }}-${{ inputs.gh_ref }} cancel-in-progress: true +env: + ALL_IMAGES: backend frontend exporter storybook mcp + # All runner instances live on the same server, so the bundle is + # downloaded from S3 once and shared between build jobs through this + # host-local directory. Each build job falls back to S3 if the file is + # missing (e.g. if runners ever move to separate machines). + BUNDLE_CACHE: /var/tmp/penpot-bundle-cache + jobs: - # ── 1. Resolve the build key shared by the whole image set ───────────── + # ── 1. Resolve the build key and check the whole set at once ─────────── prepare: name: Prepare runs-on: penpot-runner-02 - timeout-minutes: 10 + timeout-minutes: 15 outputs: gh_ref: ${{ steps.vars.outputs.gh_ref }} bundle_version: ${{ steps.vars.outputs.bundle_version }} build_key: ${{ steps.vars.outputs.build_key }} + exists: ${{ steps.check.outputs.exists }} steps: - name: Checkout code @@ -55,30 +64,61 @@ jobs: echo "bundle_version=$BUNDLE_VERSION" >> $GITHUB_OUTPUT # Image content = bundle + docker build context, so the build key - # (used for immutable tags and the skip check) combines both. + # combines both. CTX_HASH=$(git rev-parse "HEAD:docker/images" | cut -c1-12) echo "build_key=${BUNDLE_VERSION}-${CTX_HASH}" >> $GITHUB_OUTPUT - # ── 2. One build per image, in parallel ──────────────────────────────── + # The image set is a single block, so a single set-level check is + # enough: `promote` drops a marker object in S3 only after every + # image was built AND every branch tag was moved. Marker present + # means there is nothing at all to do for this build key. + - name: Check if this image set is already built + id: check + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + run: | + if aws s3api head-object \ + --bucket ${{ secrets.S3_BUCKET }} \ + --key "markers/images-${{ steps.vars.outputs.build_key }}" \ + > /dev/null 2>&1; then + echo "exists=true" >> $GITHUB_OUTPUT + { + echo "### ⏭️ Image set build skipped" + echo "" + echo "The whole set was already built and promoted for \`${{ steps.vars.outputs.build_key }}\`." + } >> "$GITHUB_STEP_SUMMARY" + else + echo "exists=false" >> $GITHUB_OUTPUT + + # Stage the bundle in the host-local cache, once, for all the + # build jobs. Download to a temp name and mv for atomicity; + # prune stale bundles while at it. + mkdir -p "$BUNDLE_CACHE" + find "$BUNDLE_CACHE" -type f -mtime +1 -delete || true + ZIP="$BUNDLE_CACHE/penpot-${{ steps.vars.outputs.build_key }}.zip" + if [ ! -f "$ZIP" ]; then + aws s3 cp "s3://${{ secrets.S3_BUCKET }}/penpot-${{ steps.vars.outputs.gh_ref }}.zip" "$ZIP.$$.tmp" + mv "$ZIP.$$.tmp" "$ZIP" + fi + fi + + # ── 2. One build per image, in parallel, only when needed ────────────── build: name: Build ${{ matrix.image }} runs-on: penpot-runner-02 timeout-minutes: 60 needs: prepare + if: needs.prepare.outputs.exists == 'false' strategy: fail-fast: true + # 4 runner slots are available for build jobs on this server; cap the + # matrix at 3 so short jobs (prepare and other workflows' checks) + # never queue behind long builds. + max-parallel: 3 matrix: - include: - - image: backend - bundle: backend - - image: frontend - bundle: frontend - - image: exporter - bundle: exporter - - image: storybook - bundle: storybook - - image: mcp - bundle: mcp + image: [backend, frontend, exporter, storybook, mcp] steps: - name: Set common environment variables @@ -107,12 +147,12 @@ jobs: username: ${{ secrets.PUB_DOCKER_USERNAME }} password: ${{ secrets.PUB_DOCKER_PASSWORD }} - # Frontend, backend, exporter, storybook and mcp now build FROM - # Docker Hardened Images (dhi.io). DHI is free (Apache 2.0, no - # subscription), but pulling from it still requires an - # authenticated login -- a separate `docker login` against a - # different registry host, even though it reuses the same - # PUB_DOCKER_* credentials as the DockerHub login above. + # Images now build FROM Docker Hardened Images (dhi.io). DHI + # is free (Apache 2.0, no subscription), but pulling from it + # still requires an authenticated login -- a separate `docker + # login` against a different registry host, even though it + # reuses the same PUB_DOCKER_* credentials as the DockerHub + # login above. - name: Login to Docker Hardened Images registry (base image pull) uses: docker/login-action@v4 with: @@ -120,48 +160,37 @@ jobs: username: ${{ secrets.PUB_DOCKER_USERNAME }} password: ${{ secrets.PUB_DOCKER_PASSWORD }} - # Every build pushes an immutable `build-` tag; if it already - # exists in the registry, this exact image was already built and the - # whole build can be skipped. Moving tags are handled by `promote`. - - name: Check if this image is already built - id: check - run: | - IMAGE="${{ secrets.DOCKER_REGISTRY }}/${{ matrix.image }}:build-${{ needs.prepare.outputs.build_key }}" - if docker manifest inspect "$IMAGE" > /dev/null 2>&1; then - echo "exists=true" >> $GITHUB_OUTPUT - echo "### ⏭️ ${{ matrix.image }}: already built (\`${{ needs.prepare.outputs.build_key }}\`)" >> "$GITHUB_STEP_SUMMARY" - else - echo "exists=false" >> $GITHUB_OUTPUT - fi - - - name: Download Penpot bundle - if: steps.check.outputs.exists == 'false' + # Bundle staged once by `prepare` on this host; the S3 fallback only + # triggers if the cache is unavailable (runners on another machine, + # cache pruned mid-run, ...). + - name: Prepare Penpot bundle env: - FILE_NAME: penpot-${{ needs.prepare.outputs.gh_ref }}.zip AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} run: | - pushd docker/images - aws s3 cp "s3://${{ secrets.S3_BUCKET }}/$FILE_NAME" . + ZIP="$BUNDLE_CACHE/penpot-${{ needs.prepare.outputs.build_key }}.zip" + if [ ! -f "$ZIP" ]; then + echo "Bundle not found in host cache; falling back to S3." + mkdir -p "$BUNDLE_CACHE" + aws s3 cp "s3://${{ secrets.S3_BUCKET }}/penpot-${{ needs.prepare.outputs.gh_ref }}.zip" "$ZIP.$$.tmp" + mv "$ZIP.$$.tmp" "$ZIP" + fi # Extract only the bundle this job needs. - unzip -q "$FILE_NAME" "penpot/${{ matrix.bundle }}/*" - mv "penpot/${{ matrix.bundle }}" "bundle-${{ matrix.bundle }}" - rm -f "$FILE_NAME" + pushd docker/images + unzip -q "$ZIP" "penpot/${{ matrix.image }}/*" + mv "penpot/${{ matrix.image }}" "bundle-${{ matrix.image }}" popd - name: Set up QEMU (stable) - if: steps.check.outputs.exists == 'false' uses: docker/setup-qemu-action@v4 with: platforms: linux/amd64,linux/arm64 - name: Set up Docker Buildx - if: steps.check.outputs.exists == 'false' uses: docker/setup-buildx-action@v4 - name: Extract metadata (tags, labels) - if: steps.check.outputs.exists == 'false' id: meta uses: docker/metadata-action@v6 with: @@ -170,7 +199,6 @@ jobs: bundle_version=${{ needs.prepare.outputs.bundle_version }} - name: Build and push Docker image - if: steps.check.outputs.exists == 'false' uses: docker/build-push-action@v7 with: context: ./docker/images/ @@ -186,23 +214,10 @@ jobs: cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ matrix.image }}:buildcache cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ matrix.image }}:buildcache,mode=max - - name: Notify Mattermost - if: failure() - uses: mattermost/action-mattermost-notify@ae31bb6f9e26a54336e79696f108a2c91cf55b4e # v2.1.0 - with: - MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }} - MATTERMOST_CHANNEL: bot-alerts-cicd - TEXT: | - ❌ 🐳 *[PENPOT] Error building the ${{ matrix.image }} docker image.* - 📄 Triggered from ref: `${{ needs.prepare.outputs.gh_ref }}` - 📦 Bundle: `${{ needs.prepare.outputs.bundle_version }}` - 🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - @infra - # ── 3. Move the branch tags of ALL images together ───────────────────── - # This is what makes the set behave as a single block: either every image - # was built (or already existed) and all branch tags advance to the same - # build key, or none of them move. + # Runs only when every build succeeded (default `needs` semantics); if + # the set was already complete, `build` is skipped and so is this job — + # the S3 marker guarantees the branch tags were already moved. promote: name: Promote image set runs-on: penpot-runner-02 @@ -227,26 +242,45 @@ jobs: - name: Point branch tags to the new build key run: | set -e - for image in backend frontend exporter storybook mcp; do + for image in $ALL_IMAGES; do docker buildx imagetools create \ -t "${{ secrets.DOCKER_REGISTRY }}/$image:${{ needs.prepare.outputs.gh_ref }}" \ "${{ secrets.DOCKER_REGISTRY }}/$image:build-${{ needs.prepare.outputs.build_key }}" done + + # The marker is written LAST: its presence certifies that all five + # images exist and all branch tags point to this build key. + - name: Write set-completed marker + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + run: | + echo "${{ github.run_id }}" | aws s3 cp - \ + "s3://${{ secrets.S3_BUCKET }}/markers/images-${{ needs.prepare.outputs.build_key }}" { echo "### ✅ Image set promoted" echo "" echo "All \`:${{ needs.prepare.outputs.gh_ref }}\` tags now point to \`build-${{ needs.prepare.outputs.build_key }}\`." } >> "$GITHUB_STEP_SUMMARY" + # ── 4. Single failure notification for the whole workflow ───────────── + notify: + name: Notify failure + runs-on: penpot-runner-02 + timeout-minutes: 5 + needs: [prepare, build, promote] + if: failure() + + steps: - name: Notify Mattermost - if: failure() uses: mattermost/action-mattermost-notify@ae31bb6f9e26a54336e79696f108a2c91cf55b4e # v2.1.0 with: MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK }} MATTERMOST_CHANNEL: bot-alerts-cicd TEXT: | - ❌ 🐳 *[PENPOT] Error promoting the penpot docker image set.* - 📄 Triggered from ref: `${{ needs.prepare.outputs.gh_ref }}` - 📦 Bundle: `${{ needs.prepare.outputs.bundle_version }}` + ❌ 🐳 *[PENPOT] Error building/promoting the penpot docker image set.* + 📄 Triggered from ref: `${{ needs.prepare.outputs.gh_ref || inputs.gh_ref }}` + 📦 Bundle: `${{ needs.prepare.outputs.bundle_version || 'n/a' }}` 🔗 Run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} @infra diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml index 05c5af1cf0..2ae5ee13b0 100644 --- a/.github/workflows/build-staging.yml +++ b/.github/workflows/build-staging.yml @@ -11,8 +11,6 @@ jobs: secrets: inherit with: gh_ref: "staging" - build_wasm: "yes" - build_storybook: "yes" build-docker: needs: build-bundle diff --git a/.github/workflows/build-tag.yml b/.github/workflows/build-tag.yml index 8a440b5ef9..f488c911a7 100644 --- a/.github/workflows/build-tag.yml +++ b/.github/workflows/build-tag.yml @@ -12,8 +12,6 @@ jobs: secrets: inherit with: gh_ref: ${{ github.ref_name }} - build_wasm: "yes" - build_storybook: "yes" build-docker: needs: build-bundle