Author: Theodore Brown (theodorejb)
Committer: Derick Rethans (derickr)
Date: 2026-06-12T08:56:41+01:00
Commit: Remove pixel-perfect screenshot tests · php/web-php@cf7c6d2 · GitHub
Raw diff: https://github.com/php/web-php/commit/cf7c6d2b3ea48b157b8991a85fea010db77fb5ed.diff
Remove pixel-perfect screenshot tests
Per feedback from other maintainers, these tests will just be a waste of time as we make numerous necessary design fixes and improvements.
Changed paths:
D .github/workflows/update-screenshots.yaml
D tests/Visual/SearchModal.css
D tests/Visual/SearchModal.spec.ts
D tests/Visual/SearchModal.spec.ts-snapshots/tests-screenshots-search-modal-chromium-linux.png
D tests/Visual/SmokeTest.spec.ts
D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-archive-1998-php-chromium.png
D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-conferences-index-php-chromium.png
D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-index-php-chromium.png
D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-index-php-chromium.png
D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-php5-php-chromium.png
D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-0-index-php-chromium.png
D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-1-index-php-chromium.png
D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-2-index-php-chromium.png
D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-6-php-chromium.png
D tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-index-php-chromium.png
M .github/workflows/preview-tests.yml
M Makefile
M playwright.config.ts
Diff:
diff --git a/.github/workflows/preview-tests.yml b/.github/workflows/preview-tests.yml
index a9e69c6e0b..4a9c5ede8c 100644
--- a/.github/workflows/preview-tests.yml
+++ b/.github/workflows/preview-tests.yml
@@ -4,8 +4,8 @@ on:
types: [ labeled ]
jobs:
- tests_visual:
- name: "Visual Tests"
+ tests_e2e:
+ name: "E2E Tests"
runs-on: "ubuntu-latest"
if: "github.repository_owner == 'php' && contains(github.event.pull_request.labels.*.name, 'Status: Preview Allowed')"
@@ -77,8 +77,8 @@ jobs:
- name: "Install Playwright"
run: "npx playwright install"
- - name: "Run visual tests"
- run: "make tests_visual"
+ - name: "Run E2E tests"
+ run: "make tests_e2e"
- name: Upload playwright report
uses: actions/upload-artifact@v7
diff --git a/.github/workflows/update-screenshots.yaml b/.github/workflows/update-screenshots.yaml
deleted file mode 100644
index f62197aaca..0000000000
--- a/.github/workflows/update-screenshots.yaml
+++ /dev/null
@@ -1,74 +0,0 @@
-# GitHub Actions documentation - GitHub Docs
-
-name: "Update screenshots"
-
-on:
- workflow_dispatch:
-
-jobs:
- tests_update_snapshots:
- name: "Update Tests snapshots"
-
- runs-on: "ubuntu-latest"
-
- strategy:
- matrix:
- php-version:
- - "8.2"
- node-version:
- - "22.x"
-
- env:
- HTTP_HOST: "localhost:8080"
-
- steps:
- - name: "Checkout"
- uses: "actions/checkout@v6"
-
- - name: "Set up PHP"
- uses: "shivammathur/setup-php@v2"
- with:
- coverage: "none"
- extensions: "none, curl, dom, json, mbstring, tokenizer, xml, xmlwriter, iconv"
- php-version: "${{ matrix.php-version }}"
-
- - name: Use Node.js ${{ matrix.node-version }}
- uses: actions/setup-node@v6
- with:
- node-version: ${{ matrix.node-version }}
-
- - name: "Set up problem matchers for PHP"
- run: "echo \"::add-matcher::${{ runner.tool_cache }}/php.json\""
-
- - name: "Set up problem matchers for phpunit/phpunit"
- run: "echo \"::add-matcher::${{ runner.tool_cache }}/phpunit.json\""
-
- - name: "Determine composer cache directory"
- run: "echo \"COMPOSER_CACHE_DIR=$(composer config cache-dir)\" >> $GITHUB_ENV"
-
- - name: "Cache dependencies installed with composer"
- uses: "actions/cache@v5"
- with:
- path: "${{ env.COMPOSER_CACHE_DIR }}"
- key: "php-${{ matrix.php-version }}-composer-${{ hashFiles('composer.lock') }}"
- restore-keys: "php-${{ matrix.php-version }}-composer-"
-
- - name: "Install dependencies with composer"
- run: "composer install --ansi --no-interaction --no-progress"
-
- - name: "Install dependencies"
- run: "yarn install"
-
- - name: "Install Playwright"
- run: "npx playwright install"
-
- - name: "Run visual tests"
- run: "make tests_update_snapshots"
-
- - name: Update snapshots
- uses: test-room-7/action-update-file@v2
- if: ${{ !cancelled() }}
- with:
- file-path: tests/Visual/**/*
- commit-msg: Update snapshots
- github-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/Makefile b/Makefile
index 2d6e571924..12de398fe1 100644
--- a/Makefile
+++ b/Makefile
@@ -26,14 +26,9 @@ tests: vendor ## Runs unit and end-to-end tests with phpunit/phpunit
vendor/bin/phpunit --configuration=tests/phpunit.xml --testsuite=end-to-end;
tests/server stop
-tests_visual:
+tests_e2e:
tests/server start;
- npx playwright test --workers=$(CORES)
- tests/server stop
-
-tests_update_snapshots:
- tests/server start;
- npx playwright test --update-snapshots
+ npx playwright test
tests/server stop
vendor: composer.json composer.lock
diff --git a/playwright.config.ts b/playwright.config.ts
index 0675282122..adf514e93b 100644
--- a/playwright.config.ts
+++ b/playwright.config.ts
@@ -30,11 +30,6 @@ export default defineConfig({
timeout: 0,
projects: [
- {
- name: 'chromium',
- use: {...devices['Desktop Chrome']},
- testDir: './tests/Visual',
- },
{
name: 'End-to-End Chromium',
use: {...devices['Desktop Chrome']},
diff --git a/tests/Visual/SearchModal.css b/tests/Visual/SearchModal.css
deleted file mode 100644
index fa3a03bc6f..0000000000
--- a/tests/Visual/SearchModal.css
+++ /dev/null
@@ -1,3 +0,0 @@
-.hero__versions {
- visibility: hidden !important;
-}
diff --git a/tests/Visual/SearchModal.spec.ts b/tests/Visual/SearchModal.spec.ts
deleted file mode 100644
index 4a753c9f0a..0000000000
--- a/tests/Visual/SearchModal.spec.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-import { test, expect } from "@playwright/test";
-import path from "path";
-
-const httpHost = process.env.HTTP_HOST;
-
-if (typeof httpHost !== "string") {
- throw new Error('Environment variable "HTTP_HOST" is not set.');
-}
-
-test.beforeEach(async ({ page }) => {
- await page.goto(httpHost);
-});
-
-const openSearchModal = async (page) => {
- await page.getByRole("button", { name: "Search" }).click();
- const modal = await page.getByRole("dialog", { name: "Search modal" });
-
- // Wait for the modal animation to finish
- await expect(page.locator("#search-modal__backdrop.show")).not.toHaveClass(
- "showing",
- );
-
- expect(modal).toBeVisible();
- return modal;
-};
-
-test("should match search modal visual snapshot", async ({ page }) => {
- const modal = await openSearchModal(page);
- await modal.getByRole("searchbox").fill("array");
- await expect(page).toHaveScreenshot(`tests/screenshots/search-modal.png`, {
- // Cannot use mask as it ignores z-index
- // See [BUG] Masked Elements masking elements in fore front · Issue #19002 · microsoft/playwright · GitHub
- stylePath: path.join(__dirname, "SearchModal.css"),
- });
-});
diff --git a/tests/Visual/SearchModal.spec.ts-snapshots/tests-screenshots-search-modal-chromium-linux.png b/tests/Visual/SearchModal.spec.ts-snapshots/tests-screenshots-search-modal-chromium-linux.png
deleted file mode 100644
index 089cd07b76..0000000000
Binary files a/tests/Visual/SearchModal.spec.ts-snapshots/tests-screenshots-search-modal-chromium-linux.png and /dev/null differ
diff --git a/tests/Visual/SmokeTest.spec.ts b/tests/Visual/SmokeTest.spec.ts
deleted file mode 100644
index 7d8979edcc..0000000000
--- a/tests/Visual/SmokeTest.spec.ts
+++ /dev/null
@@ -1,76 +0,0 @@
-import {expect, test} from '@playwright/test';
-
-export type TestPageOptions = {
- path: string
- options?: object
- evaluate?: () => any
- mask?: string[]
-}
-
-const items: TestPageOptions[] = [
- {
- path: 'index.php',
- evaluate: () => {
- const selector = document.querySelector('.elephpants');
- selector.remove()
- },
- options: {
- fullPage: true,
- timeout: 10000,
- },
- mask: ['.hero__versions'],
- },
- {
- path: 'archive/1998.php',
- evaluate: () => {
- const selector = document.querySelector('.elephpants');
- selector.remove()
- }
- },
- {path: 'releases/8_3_6.php'},
- {path: 'releases/8.0/index.php'},
- {path: 'releases/8.1/index.php'},
- {path: 'releases/8.2/index.php'},
- {path: 'releases/8.3/index.php'},
- {path: 'manual/index.php'},
- {path: 'manual/php5.php'},
- {
- path: 'conferences/index.php',
- options: {
- fullPage: false,
- }
- },
-]
-
-for (const item of items) {
- test(`testing with ${item.path}`, async ({page}, testInfo) => {
- testInfo.snapshotSuffix = '';
-
- const httpHost = process.env.HTTP_HOST
-
- if (typeof httpHost !== 'string') {
- throw new Error('Environment variable "HTTP_HOST" is not set.')
- }
-
- await page.goto(`${httpHost}/${item.path}`)
-
- if (typeof item.evaluate === 'function') {
- await page.evaluate(item.evaluate)
- }
-
- if (typeof item.mask === 'object') {
- item.options = {
- ...item.options,
- mask: item.mask.map((selector) => page.locator(selector))
- }
- }
-
- await expect(page).toHaveScreenshot(
- `tests/screenshots/${item.path}.png`,
- item.options ?? {
- fullPage: true,
- timeout: 10000,
- }
- )
- })
-}
diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-archive-1998-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-archive-1998-php-chromium.png
deleted file mode 100644
index 847b1bae90..0000000000
Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-archive-1998-php-chromium.png and /dev/null differ
diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-conferences-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-conferences-index-php-chromium.png
deleted file mode 100644
index 4c299f7b51..0000000000
Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-conferences-index-php-chromium.png and /dev/null differ
diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-index-php-chromium.png
deleted file mode 100644
index 43ff6b6ca7..0000000000
Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-index-php-chromium.png and /dev/null differ
diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-index-php-chromium.png
deleted file mode 100644
index 8ea58d12ca..0000000000
Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-index-php-chromium.png and /dev/null differ
diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-php5-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-php5-php-chromium.png
deleted file mode 100644
index 4c615c577e..0000000000
Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-manual-php5-php-chromium.png and /dev/null differ
diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-0-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-0-index-php-chromium.png
deleted file mode 100644
index 0c08cc4f3b..0000000000
Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-0-index-php-chromium.png and /dev/null differ
diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-1-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-1-index-php-chromium.png
deleted file mode 100644
index 8aa25dd736..0000000000
Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-1-index-php-chromium.png and /dev/null differ
diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-2-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-2-index-php-chromium.png
deleted file mode 100644
index 078fc5f7d8..0000000000
Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-2-index-php-chromium.png and /dev/null differ
diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-6-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-6-php-chromium.png
deleted file mode 100644
index 26c6ff70c9..0000000000
Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-6-php-chromium.png and /dev/null differ
diff --git a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-index-php-chromium.png b/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-index-php-chromium.png
deleted file mode 100644
index 8cb5b81304..0000000000
Binary files a/tests/Visual/SmokeTest.spec.ts-snapshots/tests-screenshots-releases-8-3-index-php-chromium.png and /dev/null differ