mirror of
https://github.com/penpot/penpot.git
synced 2026-11-10 20:10:37 +00:00
ee3e2a0e61
The Playwright config gives the retrying expects 40 seconds on CI and 5 seconds elsewhere, but the default and ds projects set their own expect block for the screenshot tolerance, and a project block replaces the top-level one instead of merging with it. On CI those projects have run with Playwright's 5 second default. That is too short for the first dashboard load on a busy runner, where the dashboard module is fetched lazily: specs that wait for the dashboard heading time out and pass on the retry (flaky), or fail after the retries. In the shard of one recent run that was 7 flaky and 4 failed specs out of 112, and develop itself failed the same families of specs on several runs the same day. Name the timeout once and repeat it in both projects. A probe config that imports this one and polls a condition that never holds reports 40000 ms for both projects with this change, and 5000 ms without it. The render-wasm project already repeats its own value. AI-assisted-by: claude-opus-5-5 AI-assisted-by: claude-sonnet-5-5 Signed-off-by: Filip Sajdak <filip.sajdak@siili.com>
117 lines
3.5 KiB
JavaScript
117 lines
3.5 KiB
JavaScript
import { defineConfig, devices } from "@playwright/test";
|
|
import { platform } from "os";
|
|
|
|
/**
|
|
* Read environment variables from file.
|
|
* https://github.com/motdotla/dotenv
|
|
*/
|
|
// require('dotenv').config();
|
|
|
|
const userAgent = platform === 'darwin' ?
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" :
|
|
undefined;
|
|
|
|
// Default timeout of the retrying expects (toBeVisible and friends). CI
|
|
// runners are slower and the first load of a page fetches its module, so
|
|
// they get more time. A project that sets its own `expect` replaces the
|
|
// top-level one instead of merging with it, so every such project has to
|
|
// repeat this timeout or it falls back to Playwright's 5 seconds.
|
|
const expectTimeout = process.env.CI ? 40000 : 5000;
|
|
|
|
/**
|
|
* @see https://playwright.dev/docs/test-configuration
|
|
*/
|
|
export default defineConfig({
|
|
testDir: "./playwright",
|
|
outputDir: './test-results',
|
|
/* Run tests in files in parallel */
|
|
fullyParallel: true,
|
|
/* Fail the build on CI if you accidentally left test.only in the source code. */
|
|
forbidOnly: !!process.env.CI,
|
|
/* Retry on CI only */
|
|
retries: process.env.CI ? 2 : 0,
|
|
/* Opt out of parallel tests by default; can be overriden with --workers */
|
|
workers: 1,
|
|
/* Timeout for expects (longer in CI) */
|
|
|
|
timeout: 80000,
|
|
expect: {
|
|
timeout: expectTimeout,
|
|
},
|
|
|
|
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
|
|
reporter: "list",
|
|
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
|
|
use: {
|
|
/* Base URL to use in actions like `await page.goto('/')`. */
|
|
baseURL: "http://localhost:3000",
|
|
|
|
locale: "en-US",
|
|
|
|
permissions: ["clipboard-write", "clipboard-read"],
|
|
},
|
|
|
|
/* Configure projects for major browsers */
|
|
projects: [
|
|
{
|
|
name: "default",
|
|
testDir: "./playwright/ui/specs",
|
|
use: {
|
|
...devices["Desktop Chrome"],
|
|
viewport: { width: 1920, height: 1080 }, // Add custom viewport size
|
|
video: 'retain-on-failure',
|
|
trace: 'retain-on-failure',
|
|
userAgent,
|
|
},
|
|
snapshotPathTemplate: "{testDir}/{testFilePath}-snapshots/{arg}.png",
|
|
expect: {
|
|
timeout: expectTimeout,
|
|
toHaveScreenshot: {
|
|
maxDiffPixelRatio: 0.001,
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "ds",
|
|
// Accounts without a theme follow the system one; keep the visual
|
|
// specs on the dark theme so their screenshots stay comparable.
|
|
use: { ...devices["Desktop Chrome"], colorScheme: "dark" },
|
|
testDir: "./playwright/ui/visual-specs",
|
|
expect: {
|
|
timeout: expectTimeout,
|
|
toHaveScreenshot: { maxDiffPixelRatio: 0.005 },
|
|
},
|
|
},
|
|
{
|
|
name: "render-wasm",
|
|
use: {
|
|
...devices["Desktop Chrome"],
|
|
viewport: { width: 1920, height: 1080 }, // Add custom viewport size
|
|
deviceScaleFactor: 2,
|
|
launchOptions: {
|
|
args: [
|
|
'--enable-gpu',
|
|
],
|
|
}
|
|
},
|
|
testDir: "./playwright/ui/render-wasm-specs",
|
|
snapshotPathTemplate: "{testDir}/{testFilePath}-snapshots/{arg}.png",
|
|
timeout: 2 * 60 * 1000,
|
|
expect: {
|
|
timeout: process.env.CI ? 20000 : 10000,
|
|
toHaveScreenshot: {
|
|
maxDiffPixelRatio: 0.001,
|
|
},
|
|
},
|
|
},
|
|
],
|
|
|
|
/* Run your local dev server before starting the tests */
|
|
webServer: {
|
|
timeout: 2 * 60 * 1000,
|
|
command: "node ./scripts/e2e-server.js",
|
|
url: "http://localhost:3000",
|
|
reuseExistingServer: !process.env.CI,
|
|
},
|
|
});
|