🐛 Make integration test summary non-fatal on staging

The summary step reads a filter from .github/scripts, which never
landed on this branch, so every run failed with exit code 2 even when
all tests passed. Add the filter and tolerate both a missing file and
a jq error.

Use `!cancelled()` instead of `always()` on the merge job, so it no
longer runs after cancel-in-progress killed the shards without
uploading their blobs.

Lower the JSON report retention to 7 days, the repository maximum.

Signed-off-by: David Barragán Merino <david.barragan@kaleidos.net>
This commit is contained in:
David Barragán Merino 2026-09-15 18:55:11 +02:00
parent 39ca4c264c
commit 1f64cc3e52
2 changed files with 48 additions and 5 deletions

41
.github/scripts/playwright-summary.jq vendored Normal file
View File

@ -0,0 +1,41 @@
def specs: [.. | objects | select(has("tests") and has("file"))];
def dur: [.tests[].results[]?.duration // 0] | add;
specs as $s
| ($s | map(select(any(.tests[]; .status == "unexpected")))) as $failed
| ($s | map(select(any(.tests[]; .status == "flaky")))) as $flaky
| ($s | map(select(any(.tests[]; .status == "skipped")))) as $skipped
| ($s | length) as $total
| ($s | map(dur) | add // 0 | . / 1000 | floor) as $cpu
| (if ($failed | length) > 0 then "❌"
elif ($flaky | length) > 0 then "⚠️"
else "✅" end) as $icon
| "## \($icon) Integration tests\n\n"
+ "| Total | Passed | Flaky | Failed | Skipped | Test time |\n"
+ "|---|---|---|---|---|---|\n"
+ "| \($total) | \($total - ($failed|length) - ($flaky|length) - ($skipped|length)) "
+ "| \($flaky|length) | \($failed|length) | \($skipped|length) | \($cpu / 60 | floor)m |\n"
+ (if ($failed | length) > 0 then
"\n### Failed\n\n"
+ ($failed | map("- `\(.file):\(.line)` — \(.title)") | join("\n")) + "\n"
else "" end)
+ (if ($flaky | length) > 0 then
"\n### Flaky (passed on retry)\n\n"
+ ($flaky
| map({ t: "`\(.file):\(.line)` — \(.title)",
r: ([.tests[].results[]? | select(.status == "failed")] | length) })
| sort_by(-.r)
| map("- \(.t) _(\(.r) \(if .r == 1 then "retry" else "retries" end))_")
| join("\n")) + "\n"
else "" end)
+ (if $total > 0 then
"\n<details><summary>Slowest specs</summary>\n\n"
+ ($s | map({ t: "`\(.file)` — \(.title)", d: (dur / 1000 | floor) })
| sort_by(-.d) | .[0:5]
| map("- \(.t) — \(.d)s") | join("\n"))
+ "\n\n</details>\n"
else "" end)

View File

@ -178,7 +178,7 @@ jobs:
retention-days: 3
merge-reports:
if: ${{ always() && !github.event.pull_request.draft && needs.test-integration.result != 'skipped' }}
if: ${{ !cancelled() && !github.event.pull_request.draft && needs.test-integration.result != 'skipped' }}
name: "Merge Integration Reports"
runs-on: penpot-extended-runner
timeout-minutes: 15
@ -221,16 +221,18 @@ jobs:
- name: Test summary
if: always()
continue-on-error: true
working-directory: ./frontend
run: |
if [ ! -f report.json ]; then
echo "No report produced (all shards failed early)." >> "$GITHUB_STEP_SUMMARY"
exit 0
fi
jq -r -f ../.github/scripts/playwright-summary.jq report.json >> "$GITHUB_STEP_SUMMARY"
jq -r -f ../.github/scripts/playwright-summary.jq report.json \
>> "$GITHUB_STEP_SUMMARY" \
|| echo "Summary generation failed; see the HTML report artifact." \
>> "$GITHUB_STEP_SUMMARY"
# Kept for 30 days so flakiness rates can be aggregated across runs
# without scraping job logs.
- name: Upload JSON report
uses: actions/upload-artifact@v7
if: always()
@ -239,7 +241,7 @@ jobs:
path: frontend/report.json
overwrite: true
if-no-files-found: ignore
retention-days: 30
retention-days: 7
- name: Upload HTML report
uses: actions/upload-artifact@v7