mirror of
https://github.com/penpot/penpot.git
synced 2026-08-13 16:28:59 +00:00
🔧 Report flaky e2e tests in integration workflow
Enable Playwright's JSON reporter alongside `list` and publish a summary of flaky tests to the job step summary. The JSON report is kept as an artifact for 30 days so flakiness rates can be aggregated over time. CI already runs with `retries: 2`, so unstable tests have been passing silently on retry. This only surfaces what the suite already absorbs; no test behaviour changes. The reporter in `frontend/scripts/test-e2e` becomes overridable via `PLAYWRIGHT_REPORTER` so the local developer default stays untouched.
This commit is contained in:
parent
290b14167a
commit
9f17aa6216
31
.github/workflows/tests-integration.yml
vendored
31
.github/workflows/tests-integration.yml
vendored
@ -80,9 +80,40 @@ jobs:
|
||||
|
||||
- name: Run Tests
|
||||
working-directory: ./frontend
|
||||
env:
|
||||
PLAYWRIGHT_REPORTER: list,json
|
||||
PLAYWRIGHT_JSON_OUTPUT_NAME: report.json
|
||||
run: |
|
||||
./scripts/test-e2e
|
||||
|
||||
- name: Flaky summary
|
||||
if: always()
|
||||
working-directory: ./frontend
|
||||
run: |
|
||||
if [ ! -f report.json ]; then
|
||||
echo "No report.json produced (the run failed early)." >> "$GITHUB_STEP_SUMMARY"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
jq -r '
|
||||
[ .. | objects
|
||||
| select(has("tests") and has("file"))
|
||||
| select(any(.tests[]; .status == "flaky"))
|
||||
| "- `\(.file):\(.line)` — \(.title)"
|
||||
] as $f
|
||||
| "## Flaky tests: \($f | length)\n"
|
||||
+ (if ($f | length) == 0 then "_none_" else ($f | join("\n")) end)
|
||||
' report.json >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload JSON report
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: integration-json-report
|
||||
path: frontend/report.json
|
||||
overwrite: true
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload test result
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
|
||||
@ -1,8 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
SCRIPT_DIR=$(dirname $0);
|
||||
REPORTER=${PLAYWRIGHT_REPORTER:-list};
|
||||
|
||||
set -ex
|
||||
|
||||
$SCRIPT_DIR/setup;
|
||||
pnpm run test:e2e -x --workers=1 --reporter=list "$@";
|
||||
pnpm run test:e2e -x --workers=1 --reporter="$REPORTER" "$@";
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user