From ee5d4bf85c12602a44b3c9ecf374da3940841625 Mon Sep 17 00:00:00 2001 From: Vitor Mattos Date: Mon, 21 Sep 2026 19:47:14 -0300 Subject: [PATCH 1/2] ci: include expected screenshot in failure artifacts --- scripts/check-screenshots.mjs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scripts/check-screenshots.mjs b/scripts/check-screenshots.mjs index cb15ffe..21b2d31 100644 --- a/scripts/check-screenshots.mjs +++ b/scripts/check-screenshots.mjs @@ -13,19 +13,27 @@ const maxDiffRatio = 0.001 const diffPath = path.join(packageRoot, 'test-results', 'demo-screenshot-diff.png') const currentPath = path.join(packageRoot, 'test-results', 'demo-screenshot-current.png') +const expectedPath = path.join(packageRoot, 'test-results', 'demo-screenshot-expected.png') + +async function writeComparisonImages(committedImage, currentImage) { + await mkdir(path.dirname(currentPath), { recursive: true }) + await writeFile(expectedPath, committedImage) + await writeFile(currentPath, currentImage) +} async function main() { - const committed = PNG.sync.read(await readFile(screenshotPath)) + const committedImage = await readFile(screenshotPath) + const committed = PNG.sync.read(committedImage) const currentImage = await generateScreenshot() const current = PNG.sync.read(currentImage) if (committed.width !== current.width || committed.height !== current.height) { - await mkdir(path.dirname(currentPath), { recursive: true }) - await writeFile(currentPath, currentImage) + await writeComparisonImages(committedImage, currentImage) throw new Error( `Screenshot size changed: committed ${committed.width}x${committed.height}, ` + `generated ${current.width}x${current.height}.\n` + + `Expected screenshot written to ${path.relative(packageRoot, expectedPath)}.\n` + `Generated screenshot written to ${path.relative(packageRoot, currentPath)}.\n` + 'Run "npm run screenshots:update" and commit the result.' ) @@ -46,12 +54,12 @@ async function main() { return } - await mkdir(path.dirname(diffPath), {recursive: true}) - await writeFile(currentPath, currentImage) + await writeComparisonImages(committedImage, currentImage) await writeFile(diffPath, PNG.sync.write(diff)) throw new Error( `${report}, above the allowed ${(maxDiffRatio * 100).toFixed(4)}%.\n` + + `Expected screenshot written to ${path.relative(packageRoot, expectedPath)}.\n` + `Generated screenshot written to ${path.relative(packageRoot, currentPath)}.\n` + `Visual diff written to ${path.relative(packageRoot, diffPath)}.\n` + 'If the change is expected, run "npm run screenshots:update" and commit the result.' From ed615e213e322c0f695d76135bb50817b7235483 Mon Sep 17 00:00:00 2001 From: Vitor Mattos Date: Mon, 21 Sep 2026 19:47:21 -0300 Subject: [PATCH 2/2] ci: comment screenshot differences on pull requests --- .../workflows/screenshot-failure-comment.yml | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .github/workflows/screenshot-failure-comment.yml diff --git a/.github/workflows/screenshot-failure-comment.yml b/.github/workflows/screenshot-failure-comment.yml new file mode 100644 index 0000000..a9f948c --- /dev/null +++ b/.github/workflows/screenshot-failure-comment.yml @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: 2026 LibreCode coop and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +name: Screenshot failure comment + +on: + workflow_run: + workflows: + - Playwright + types: + - completed + +permissions: + actions: read + contents: write + pull-requests: write + +jobs: + comment: + if: >- + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'failure' && + github.event.workflow_run.pull_requests[0].number != null + runs-on: ubuntu-latest + steps: + - name: Download screenshot artifacts + id: artifacts + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + GH_REPO: ${{ github.repository }} + run: | + if gh run download "$RUN_ID" --repo "$GH_REPO" --name screenshot-diff --dir screenshot-artifacts; then + echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "found=false" >> "$GITHUB_OUTPUT" + fi + + - name: Comment on pull request + if: steps.artifacts.outputs.found == 'true' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} + shell: bash + run: | + cat > screenshot-comment.md <<'EOF' + ## README screenshot check failed + + The generated README screenshot differs from the committed version. + + ### Expected + + ![Expected screenshot](./screenshot-artifacts/demo-screenshot-expected.png) + + ### Generated + + ![Generated screenshot](./screenshot-artifacts/demo-screenshot-current.png) + EOF + + attachments=( + "--attach" "./screenshot-artifacts/demo-screenshot-expected.png#Expected screenshot" + "--attach" "./screenshot-artifacts/demo-screenshot-current.png#Generated screenshot" + ) + + if [[ -f screenshot-artifacts/demo-screenshot-diff.png ]]; then + cat >> screenshot-comment.md <<'EOF' + + ### Diff + + ![Visual diff](./screenshot-artifacts/demo-screenshot-diff.png) + EOF + attachments+=("--attach" "./screenshot-artifacts/demo-screenshot-diff.png#Visual diff") + fi + + cat >> screenshot-comment.md <<'EOF' + + If the visual change is expected, run `npm run screenshots:update` and commit the updated screenshot. + EOF + + gh pr comment "$PR_NUMBER" --repo "$GH_REPO" --body-file screenshot-comment.md "${attachments[@]}"