Headless Testing

Automated WebGL Testing in Headless Chrome

A shader edit can break a scene without any error, so test what the page draws. Playwright 26,280 (github.com/microsoft/playwright (https://github.com/microsoft/playwright 96,776 ), Apache-2.0, 1.63.0) drives your installed Chrome 1 , and its toHaveScreenshot() compares a capture with a stored baseline. With SwiftShader 2,546 (SwiftShader), a laptop and a GPU-less CI server render the same pixels:

shelf.spec.mjs: a WebGL regression test with a screenshot baselineJavaScript
// shelf.spec.mjs: npm install -D @playwright/test, then npx playwright test
import { test, expect } from '@playwright/test';
import { pathToFileURL } from 'node:url';
test.use({ channel: 'chrome', launchOptions: { args: ['--use-angle=swiftshader',
  '--enable-unsafe-swiftshader', '--allow-file-access-from-files'] } });   // identical pixels
test('lit shelf: no GL errors, matches baseline', async ({ page }) => {
  const problems = [];
  page.on('console', (m) => /WebGL|GL_/.test(m.text()) && problems.push(m.text()));
  page.on('pageerror', (error) => problems.push(error.message));
  await page.goto(pathToFileURL('demos/ch03/shelf-lit.html').href);
  await page.waitForFunction(() => window.__done === true);
  expect(problems).toEqual([]);
  await expect(page.locator('canvas')).toHaveScreenshot('shelf-lit.png', { maxDiffPixels: 0 });
});

The first run fails with "A snapshot doesn't exist ... writing actual" and saves the baseline; commit it. Later runs compare against it:

Output of 150
Running 1 test using 1 worker
  ok 1 shelf.spec.mjs:8:1 › lit shelf: no GL errors, matches baseline (2.1s)
  1 passed (3.6s)

Pin the renderer: on the GTX 1650 the test passed only thanks to the default color threshold of 0.2; with threshold: 0, 26,243 pixels (25%) differed from the SwiftShader baseline, while SwiftShader runs matched exactly. A page drawing with no program failed the console check.

A screenshot regression test in miniature: baseline render, a render after a small shader edit, and the pixel diff that catches itHTMLLive
<!doctype html>
<style>
  body { margin: 0; font: 11px system-ui, sans-serif; background: #f7f4ee; color: #333; }
  canvas { display: block; width: 100%; max-width: 600px; }
  .names { display: flex; max-width: 600px; text-align: center; font: 11px monospace; }
  .names div { flex: 1; padding: 4px 2px; }
  #verdict { max-width: 600px; margin: 4px 0; padding: 4px 8px; font: 12px monospace; color: #fff; }
</style>
<canvas id="c" width="600" height="160"></canvas>
<div class="names"><div>baseline (committed)</div><div>after the edit</div><div>diff: red = changed pixel</div></div>
<div id="verdict"></div>
<script>
const gl = document.getElementById('c').getContext('webgl2', { preserveDrawingBuffer: true });
const W = 200, H = 160;
function program(fs) {
  const p = gl.createProgram();
  for (const [type, src] of [[gl.VERTEX_SHADER, `#version 300 es
void main() { gl_Position = vec4(gl_VertexID == 1 ? 3.0 : -1.0, gl_VertexID == 2 ? 3.0 : -1.0, 0.0, 1.0); }`], [gl.FRAGMENT_SHADER, fs]]) {
    const s = gl.createShader(type); gl.shaderSource(s, src); gl.compileShader(s);
    if (!gl.getShaderParameter(s, gl.COMPILE_STATUS)) throw new Error(gl.getShaderInfoLog(s));
    gl.attachShader(p, s);
  }
  gl.linkProgram(p);
  return p;
}
// The page under test: a shelf painted by one fragment shader. The "edit" nudges one constant.
const shelf = (spine) => `#version 300 es
precision highp float; uniform vec2 uOrigin; out vec4 c;
const vec3 covers[6] = vec3[](vec3(0.12,0.37,0.55), vec3(0.36,0.25,0.6), vec3(0.88,0.6,0.06), vec3(0.25,0.49,0.23), vec3(0.71,0.27,0.18), vec3(0.16,0.62,0.56));
void main() {
  vec2 p = gl_FragCoord.xy - uOrigin;
  vec3 color = p.y < 30.0 ? vec3(0.54, 0.38, 0.25) : vec3(0.93, 0.91, 0.87);
  int book = int((p.x - 10.0) / 30.0);
  float local = mod(p.x - 10.0, 30.0);
  if (book >= 0 && book < 6 && local > 3.0 && p.y >= 30.0 && p.y < 110.0 + float(book % 3) * 10.0)
    color = covers[book] * (local < ${spine} ? 0.6 : 1.0);   // the spine shade
  c = vec4(color, 1.0);
}`;
const baseline = program(shelf('7.0')), edited = program(shelf('8.0'));   // a one-pixel-wider spine
gl.bindVertexArray(gl.createVertexArray());
gl.enable(gl.SCISSOR_TEST);
function render(p, x) {
  gl.viewport(x, 0, W, H); gl.scissor(x, 0, W, H);
  gl.useProgram(p); gl.uniform2f(gl.getUniformLocation(p, 'uOrigin'), x, 0);
  gl.drawArrays(gl.TRIANGLES, 0, 3);
  const pixels = new Uint8Array(W * H * 4);
  gl.readPixels(x, 0, W, H, gl.RGBA, gl.UNSIGNED_BYTE, pixels);   // the "screenshot"
  return pixels;
}
const a = render(baseline, 0), b = render(edited, W);
// Compare like toHaveScreenshot(): count pixels whose channels differ by more than a threshold.
const diff = new Uint8Array(W * H * 4);
let changed = 0;
for (let i = 0; i < a.length; i += 4) {
  const d = Math.max(Math.abs(a[i] - b[i]), Math.abs(a[i + 1] - b[i + 1]), Math.abs(a[i + 2] - b[i + 2]));
  const bad = d > 0.2 * 255 * 0.1;                               // a small tolerance
  if (bad) changed++;
  diff.set(bad ? [220, 30, 30, 255] : [a[i] * 0.3 + 170, a[i + 1] * 0.3 + 170, a[i + 2] * 0.3 + 170, 255], i);
}
const t = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, t);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA8, W, H, 0, gl.RGBA, gl.UNSIGNED_BYTE, diff);   // rows already bottom-up
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
const show = program(`#version 300 es
precision mediump float; uniform sampler2D uDiff; uniform vec2 uOrigin; out vec4 c;
void main() { c = texture(uDiff, (gl_FragCoord.xy - uOrigin) / vec2(${W}.0, ${H}.0)); }`);
gl.viewport(2 * W, 0, W, H); gl.scissor(2 * W, 0, W, H);
gl.useProgram(show); gl.uniform2f(gl.getUniformLocation(show, 'uOrigin'), 2 * W, 0);
gl.drawArrays(gl.TRIANGLES, 0, 3);
const verdict = document.getElementById('verdict');
verdict.style.background = changed ? '#b5452f' : '#3f7d3a';
verdict.textContent = changed ? `FAIL: ${changed} of ${W * H} pixels differ from the baseline (maxDiffPixels: 0)` : 'PASS: identical to the baseline';
</script>