Color-coded picking draws the scene again with each object in a flat "color" that encodes its ID and reads the pixel under the pointer. Instead of RGB bytes in the visible canvas, WebGL 2 can use an offscreen R32UI target: 32-bit integers, no blending or antialiasing, and nothing on screen. The thirtieth part:
// Section 3.17: color-coded picking into an R32UI target the size of the canvas: objects
// write object * 16 + face (face = gl_VertexID / 4 in GLH.box meshes; 0 means nothing)
GLH.picker = function (gl) {
const target = GLH.targets(gl, gl.canvas.width, gl.canvas.height, [gl.R32UI]);
const pass = GLH.load(gl, `#version 300 es
in vec3 aPosition; uniform mat4 uViewProjection, uModel; flat out uint vFace;
void main() { vFace = uint(gl_VertexID / 4); // four vertices per box face
gl_Position = uViewProjection * uModel * vec4(aPosition, 1.0); }`, `#version 300 es
flat in uint vFace; uniform uint uObject; out uint id;
void main() { id = uObject * 16u + vFace; }`);
pass.render = (draw) => { // draw(uniforms) draws the IDs
gl.bindFramebuffer(gl.FRAMEBUFFER, target.fbo);
gl.viewport(0, 0, target.width, target.height);
gl.clearBufferuiv(gl.COLOR, 0, [0, 0, 0, 0]); // clear() is invalid on integer targets
gl.clearBufferfv(gl.DEPTH, 0, [1]);
gl.useProgram(pass.program); draw(pass.uniforms);
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
};
pass.read = ([x, y]) => { // the ID under a buffer pixel
const id = new Uint32Array(4); // RGBA_INTEGER always returns four
gl.bindFramebuffer(gl.FRAMEBUFFER, target.fbo);
gl.readPixels(x, y, 1, 1, gl.RGBA_INTEGER, gl.UNSIGNED_INT, id);
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
return id[0];
};
return pass;
};Reading back at points projected from known spots on four objects, plus an empty corner:
const gl = Object.assign(document.createElement('canvas'), { width: 620, height: 300 })
.getContext('webgl2');
const picker = GLH.picker(gl), shelf = GLH.shelf(gl, picker.attributes);
const camera = shelf.camera(620 / 300), { vec3 } = glMatrix;
gl.enable(gl.DEPTH_TEST);
picker.render((u) => { // one ID pass, redrawn only on clicks
u.uViewProjection(camera);
shelf.items.forEach((item, i) => { u.uObject(i + 1); shelf.draw(u, [item]); }); // plank: 7
});
const faces = ['fore-edge', 'spine', 'top', 'bottom', 'front cover', 'back cover'];
const pixelOf = (item, point) => { // a point on an item -> buffer pixel
const p = vec3.transformMat4([], vec3.transformMat4([], point, item.model), camera);
return [Math.floor((p[0] + 1) * 310), Math.floor((p[1] + 1) * 150)];
};
const probes = { 'book 2 cover': [1, [0, 0.2, 0.5]], 'book 5 fore-edge': [4, [0.5, 0, 0]],
'book 1 top': [0, [0, 0.5, 0]], 'plank top': [6, [0.3, 0.5, 0.3]] };
for (const [name, [item, point]] of Object.entries(probes)) {
const pixel = pixelOf(shelf.items[item], point), id = picker.read(pixel);
console.log(`${name} at ${pixel}: id ${id} -> object ${id >> 4}, ${faces[id & 15]}`);
}
console.log('corner at 0,299: id', picker.read([0, 299]));Output
book 2 cover at 201,205: id 36 -> object 2, front cover book 5 fore-edge at 425,162: id 80 -> object 5, fore-edge book 1 top at 142,246: id 18 -> object 1, top plank top at 433,86: id 114 -> object 7, top corner at 0,299: id 0
A plain clear() on the integer target failed with INVALID_OPERATION, hence clearBufferuiv(). readPixels() waits for the GPU (SwiftShader 2,546 even logs "GPU stall due to ReadPixels"), so render IDs on clicks only.
<!doctype html>
<style>
body { margin: 0; font: 12px Georgia, serif; background: #f7f4ee; color: #333; }
.wrap { position: relative; width: 100%; max-width: 600px; }
canvas { display: block; width: 100%; cursor: grab; }
.names { display: flex; max-width: 600px; text-align: center; font: 11px monospace; }
.names div { flex: 1; padding: 4px 2px; }
#card { position: absolute; right: 10px; bottom: 10px; padding: 6px 10px; background: #fffdf7; border: 1px solid #c9b99a; border-radius: 6px; }
#log { position: absolute; left: 10px; top: 8px; font: 11px monospace; }
</style>
<div class="wrap">
<canvas id="c" width="1200" height="480"></canvas>
</div>
<div class="names"><div>what you see</div><div>the ID pass (never shown normally): hue = object, shade = face</div></div>
<pre id="log" style="position:static;max-width:600px;margin:4px 0"></pre>
<script>
const m4 = {
perspective(fovy, aspect, near, far) {
const f = 1 / Math.tan(fovy * Math.PI / 360), d = near - far;
return [f / aspect, 0, 0, 0, 0, f, 0, 0, 0, 0, (far + near) / d, -1, 0, 0, 2 * far * near / d, 0];
},
lookAt(eye, target, up = [0, 1, 0]) {
const cross = (a, b) => [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]];
const unit = (v) => v.map((c) => c / Math.hypot(...v));
const dot = (a, b) => a[0] * b[0] + a[1] * b[1] + a[2] * b[2];
const z = unit(eye.map((c, i) => c - target[i])), x = unit(cross(up, z)), y = cross(z, x);
return [x[0], y[0], z[0], 0, x[1], y[1], z[1], 0, x[2], y[2], z[2], 0, -dot(x, eye), -dot(y, eye), -dot(z, eye), 1];
},
multiply(a, b) {
const out = new Array(16).fill(0);
for (let c = 0; c < 4; c++) for (let r = 0; r < 4; r++) for (let k = 0; k < 4; k++) out[c * 4 + r] += a[k * 4 + r] * b[c * 4 + k];
return out;
},
model(x, y, z, sx, sy, sz, turn = 0) { // T * Ry(turn) * S
const c = Math.cos(turn), s = Math.sin(turn);
return [c * sx, 0, -s * sx, 0, 0, sy, 0, 0, s * sz, 0, c * sz, 0, x, y, z, 1];
},
normal(m) {
const [a, b, c, , d, e, f, , g, h, i] = m;
const n = [e * i - f * h, f * g - d * i, d * h - e * g, c * h - b * i, a * i - c * g, b * g - a * h, b * f - c * e, c * d - a * f, a * e - b * d];
const det = a * n[0] + b * n[1] + c * n[2];
return n.map((v) => v / det);
},
};
function program(vs, fs) {
const p = gl.createProgram();
for (const [type, src] of [[gl.VERTEX_SHADER, vs], [gl.FRAGMENT_SHADER, fs]]) {
const s = gl.createShader(type); gl.shaderSource(s, src); gl.compileShader(s);
if (!gl.getShaderParameter(s, gl.COMPILE_STATUS)) throw new Error(gl.getShaderInfoLog(s));
gl.attachShader(p, s);
}
gl.linkProgram(p);
return p;
}
const canvas = document.getElementById('c');
const gl = canvas.getContext('webgl2');
const u = (p, n) => gl.getUniformLocation(p, n);
// A box whose faces come in book order (6 vertices each, so face = gl_VertexID / 6):
// +x fore-edge, -x spine, +y top, -y bottom, +z front cover, -z back cover.
const FACES = ['fore-edge', 'spine', 'top', 'bottom', 'front cover', 'back cover'];
const cube = [];
for (const [n, a1, b1] of [[[1,0,0],[0,0,-1],[0,1,0]], [[-1,0,0],[0,0,1],[0,1,0]], [[0,1,0],[1,0,0],[0,0,-1]],
[[0,-1,0],[1,0,0],[0,0,1]], [[0,0,1],[1,0,0],[0,1,0]], [[0,0,-1],[-1,0,0],[0,1,0]]])
for (const [a, b] of [[-1,-1],[1,-1],[1,1],[-1,-1],[1,1],[-1,1]])
cube.push(...[0, 1, 2].map(i => (n[i] + a * a1[i] + b * b1[i]) / 2), ...n, (a + 1) / 2, (b + 1) / 2);
const sceneVAO = gl.createVertexArray();
gl.bindVertexArray(sceneVAO);
gl.bindBuffer(gl.ARRAY_BUFFER, gl.createBuffer());
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(cube), gl.STATIC_DRAW);
gl.vertexAttribPointer(0, 3, gl.FLOAT, false, 32, 0); gl.enableVertexAttribArray(0);
gl.vertexAttribPointer(1, 3, gl.FLOAT, false, 32, 12); gl.enableVertexAttribArray(1);
gl.vertexAttribPointer(2, 2, gl.FLOAT, false, 32, 24); gl.enableVertexAttribArray(2);
// BookNest's catalog, its covers (drawn on 2D canvases) and the shelf.
const books = [['The Quiet Harbor', 'Mara Ellison', 14.99, 4.6, '#1f5f8b'], ['Patterns of the Deep Web', 'Tomas Reyes', 39.5, 4.3, '#5b3f99'],
['Salt and Saffron', 'Priya Nair', 24, 4.8, '#e09a10'], ['Small Steps to Big Summits', 'Jonas Berg', 18.75, 4.1, '#3f7d3a'],
["The Clockmaker's Paradox", 'Elena Sokolova', 16.2, 4.5, '#b5452f'], ['Gardens in Glass', 'Aiko Tanaka', 21.3, 4.4, '#2a9d8f']]
.map(([title, author, price, rating, color]) => ({ title, author, price, rating, color }));
const rgb = (hex) => [1, 3, 5].map((k) => parseInt(hex.slice(k, k + 2), 16) / 255);
const covers = books.map(({ title, color }) => {
const c = Object.assign(document.createElement('canvas'), { width: 128, height: 192 }), g = c.getContext('2d');
g.fillStyle = color; g.fillRect(0, 0, 128, 192);
g.strokeStyle = '#f2ead8'; g.lineWidth = 3; g.strokeRect(8, 8, 112, 176);
g.fillStyle = '#f2ead8'; g.font = 'bold 15px Georgia'; g.textAlign = 'center';
title.split(' ').reduce((lines, w) => { const l = lines[lines.length - 1]; if ((l + ' ' + w).length > 11) lines.push(w); else lines[lines.length - 1] = (l + ' ' + w).trim(); return lines; }, [''])
.forEach((line, i) => g.fillText(line, 64, 90 + i * 20));
const t = gl.createTexture(); gl.bindTexture(gl.TEXTURE_2D, t);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA8, gl.RGBA, gl.UNSIGNED_BYTE, c); gl.generateMipmap(gl.TEXTURE_2D);
return t;
});
const items = books.map(({ color }, i) => ({ color: rgb(color), model: m4.model(-1.3 + i * 0.52, 0.5, 0, 0.42, 0.98, 0.12, (30 - i * 12) * Math.PI / 180) }));
items.push({ color: rgb('#8a6240'), model: m4.model(0, -0.03, 0, 3.4, 0.06, 0.7) }); // the plank: object 7
// The visible pass, with the selection as uniforms: brighten the book, frame the face in gold.
const lit = program(`#version 300 es
layout(location = 0) in vec3 aPosition; layout(location = 1) in vec3 aNormal; layout(location = 2) in vec2 aUV;
uniform mat4 uViewProjection, uModel; uniform mat3 uNormalMatrix;
out vec3 vPosition, vNormal; out vec2 vUV; flat out int vFace; // integers must be flat
void main() { vec4 w = uModel * vec4(aPosition, 1.0); vPosition = w.xyz; vNormal = uNormalMatrix * aNormal;
vUV = aUV; vFace = gl_VertexID / 6; gl_Position = uViewProjection * w; }`, `#version 300 es
precision highp float;
in vec3 vPosition, vNormal; in vec2 vUV; flat in int vFace; out vec4 outColor;
uniform sampler2D uCover; uniform vec3 uColor, uLamp;
uniform int uPart; uniform float uTextured, uSelected; // selected face; 1 if this book is selected
void main() {
float light = 0.4 + 0.75 * max(dot(normalize(vNormal), normalize(uLamp - vPosition)), 0.0);
vec3 base = vFace == 4 && uTextured > 0.5 ? texture(uCover, vUV).rgb : uColor;
vec2 edge = min(vUV, 1.0 - vUV);
bool frame = uSelected > 0.5 && vFace == uPart && min(edge.x, edge.y) < 0.04;
outColor = vec4(frame ? vec3(1.0, 0.75, 0.15) : base * light * (1.0 + 0.3 * uSelected), 1.0);
}`);
// The ID pass: an R32UI target the size of the canvas; each fragment writes object * 16 + face.
const idProgram = program(`#version 300 es
layout(location = 0) in vec3 aPosition; uniform mat4 uViewProjection, uModel; flat out uint vFace;
void main() { vFace = uint(gl_VertexID / 6); gl_Position = uViewProjection * uModel * vec4(aPosition, 1.0); }`,
`#version 300 es
flat in uint vFace; uniform uint uObject; out uint id;
void main() { id = uObject * 16u + vFace; }`);
const idTexture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, idTexture);
gl.texStorage2D(gl.TEXTURE_2D, 1, gl.R32UI, canvas.width, canvas.height);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
const idDepth = gl.createRenderbuffer();
gl.bindRenderbuffer(gl.RENDERBUFFER, idDepth);
gl.renderbufferStorage(gl.RENDERBUFFER, gl.DEPTH_COMPONENT24, canvas.width, canvas.height);
const idFBO = gl.createFramebuffer();
gl.bindFramebuffer(gl.FRAMEBUFFER, idFBO);
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, idTexture, 0);
gl.framebufferRenderbuffer(gl.FRAMEBUFFER, gl.DEPTH_ATTACHMENT, gl.RENDERBUFFER, idDepth);
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
function renderIDs(viewProjection, w = canvas.width, h = canvas.height) {
gl.bindFramebuffer(gl.FRAMEBUFFER, idFBO);
gl.viewport(0, 0, w, h);
gl.clearBufferuiv(gl.COLOR, 0, [0, 0, 0, 0]); // clear() is invalid on integer targets
gl.clearBufferfv(gl.DEPTH, 0, [1]);
gl.enable(gl.DEPTH_TEST); gl.enable(gl.CULL_FACE);
gl.useProgram(idProgram);
gl.bindVertexArray(sceneVAO);
gl.uniformMatrix4fv(u(idProgram, 'uViewProjection'), false, viewProjection);
items.forEach(({ model }, i) => {
gl.uniform1ui(u(idProgram, 'uObject'), i + 1); // 0 means nothing
gl.uniformMatrix4fv(u(idProgram, 'uModel'), false, model);
gl.drawArrays(gl.TRIANGLES, 0, 36);
});
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
}
function readID([x, y]) {
const id = new Uint32Array(4); // RGBA_INTEGER always returns four
gl.bindFramebuffer(gl.FRAMEBUFFER, idFBO);
gl.readPixels(x, y, 1, 1, gl.RGBA_INTEGER, gl.UNSIGNED_INT, id);
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
return id[0];
}
function drawScene(viewProjection, selected = 0, part = -1) {
gl.enable(gl.DEPTH_TEST); gl.enable(gl.CULL_FACE);
gl.useProgram(lit);
gl.bindVertexArray(sceneVAO);
gl.uniformMatrix4fv(u(lit, 'uViewProjection'), false, viewProjection);
gl.uniform3f(u(lit, 'uLamp'), -0.6, 1.8, 2.0);
gl.uniform1i(u(lit, 'uPart'), part);
items.forEach(({ color, model }, i) => {
gl.bindTexture(gl.TEXTURE_2D, covers[i] || null);
gl.uniform1f(u(lit, 'uTextured'), i < 6 ? 1 : 0);
gl.uniform1f(u(lit, 'uSelected'), i + 1 === selected ? 1 : 0);
gl.uniformMatrix4fv(u(lit, 'uModel'), false, model);
gl.uniformMatrix3fv(u(lit, 'uNormalMatrix'), false, m4.normal(model));
gl.uniform3fv(u(lit, 'uColor'), color);
gl.drawArrays(gl.TRIANGLES, 0, 36);
});
}
// Pointer input: drags turn an orbit camera (with inertia), a press and release that barely moved is a click.
function locate(e) { // CSS pixels -> buffer pixels (y up) and clip space
const r = canvas.getBoundingClientRect();
const x = (e.clientX - r.left) * canvas.width / r.width, y = (e.clientY - r.top) * canvas.height / r.height;
return { pixel: [Math.floor(x), canvas.height - 1 - Math.floor(y)], clip: [2 * x / canvas.width - 1, 1 - 2 * y / canvas.height] };
}
function makeOrbit({ target = [0, 0.4, 0], distance = 3.4, yaw = 15, pitch = 20, click = () => {} } = {}) {
const orbit = { yaw, pitch, vx: 0, vy: 0, dx: 0, dy: 0, dragging: false, moved: 0, x: 0, y: 0 };
canvas.style.touchAction = 'none'; // no scrolling on touch drags
canvas.addEventListener('pointerdown', (e) => {
if (e.isTrusted) canvas.setPointerCapture(e.pointerId); // keep the moves that leave the canvas
Object.assign(orbit, { dragging: true, moved: 0, x: e.clientX, y: e.clientY });
});
canvas.addEventListener('pointermove', (e) => {
if (!orbit.dragging) return;
const dx = e.clientX - orbit.x, dy = e.clientY - orbit.y;
orbit.moved += Math.hypot(dx, dy); orbit.x = e.clientX; orbit.y = e.clientY;
orbit.dx += dx; orbit.dy += dy;
});
canvas.addEventListener('pointerup', (e) => {
orbit.dragging = false;
if (orbit.moved < 4) click(locate(e)); // a drag is not a click
});
orbit.step = () => { // once per frame
if (orbit.dragging) [orbit.vx, orbit.vy] = [orbit.dx * 0.4, orbit.dy * 0.4];
else [orbit.vx, orbit.vy] = [orbit.vx * 0.92, orbit.vy * 0.92]; // inertia: 8% less per frame
orbit.yaw -= orbit.vx;
orbit.pitch = Math.min(80, Math.max(-80, orbit.pitch + orbit.vy)); // never flip over
orbit.dx = orbit.dy = 0;
};
orbit.eye = () => {
const y = orbit.yaw * Math.PI / 180, p = orbit.pitch * Math.PI / 180;
return [target[0] + distance * Math.cos(p) * Math.sin(y), target[1] + distance * Math.sin(p), target[2] + distance * Math.cos(p) * Math.cos(y)];
};
orbit.camera = (aspect) => m4.multiply(m4.perspective(40, aspect, 0.1, 20), m4.lookAt(orbit.eye(), target));
return orbit;
}
// Scripted input for the preview: synthetic PointerEvents play the user.
const fire = (type, x, y) => {
const r = canvas.getBoundingClientRect();
canvas.dispatchEvent(new PointerEvent(type, { clientX: r.left + x * r.width, clientY: r.top + y * r.height, pointerId: 1, bubbles: true }));
};
const W = 600, H = 480;
const camera = m4.multiply(m4.perspective(40, W / H, 0.1, 20), m4.lookAt([1.2, 1.6, 3.6], [0, 0.4, 0]));
renderIDs(camera, W, H); // one ID pass, into the target's first 600 x 480
// Probes: known points on four objects, projected to buffer pixels, then read back.
function pixelOf(item, [x, y, z]) {
const m = item.model, w = [0, 1, 2].map(r => m[r] * x + m[4 + r] * y + m[8 + r] * z + m[12 + r]);
const c = [0, 1, 3].map(r => camera[r] * w[0] + camera[4 + r] * w[1] + camera[8 + r] * w[2] + camera[12 + r]);
return [Math.floor((c[0] / c[2] + 1) * W / 2), Math.floor((c[1] / c[2] + 1) * H / 2)];
}
const probes = { 'book 2 front cover': [1, [0, 0.2, 0.5]], 'book 5 fore-edge': [4, [0.5, 0, 0]],
'book 1 top': [0, [0, 0.5, 0]], 'plank top': [6, [0.3, 0.5, 0.3]] };
const lines = Object.entries(probes).map(([name, [item, point]]) => {
const pixel = pixelOf(items[item], point), id = readID(pixel);
return `${name.padEnd(19)} at ${String(pixel).padEnd(8)} -> id ${String(id).padStart(3)} = object ${id >> 4}, ${FACES[id & 15]}`;
});
lines.push(`empty corner at 0,${H - 1} -> id ${readID([0, H - 1])} (nothing)`);
document.getElementById('log').textContent = lines.join('\n');
// A viewer for the integer texture: usampler2D, texelFetch (integer textures cannot be filtered).
const viewer = program(`#version 300 es
void main() { gl_Position = vec4(gl_VertexID == 1 ? 3.0 : -1.0, gl_VertexID == 2 ? 3.0 : -1.0, 0.0, 1.0); }`,
`#version 300 es
precision highp float; precision highp usampler2D;
uniform usampler2D uIDs; uniform vec2 uOrigin; out vec4 c;
void main() {
uint id = texelFetch(uIDs, ivec2(gl_FragCoord.xy - uOrigin), 0).r;
if (id == 0u) { c = vec4(0.15, 0.15, 0.17, 1.0); return; }
float object = float(id >> 4), face = float(id & 15u);
vec3 hue = 0.55 + 0.45 * cos(6.2832 * (object / 7.0 + vec3(0.0, 0.33, 0.67)));
c = vec4(hue * (0.45 + 0.1 * face), 1.0);
}`);
gl.viewport(0, 0, canvas.width, canvas.height);
gl.clearColor(0.93, 0.91, 0.87, 1); gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
gl.viewport(0, 0, W, H);
drawScene(camera);
gl.disable(gl.DEPTH_TEST);
gl.viewport(W, 0, W, H);
gl.useProgram(viewer);
gl.bindTexture(gl.TEXTURE_2D, idTexture);
gl.uniform2f(u(viewer, 'uOrigin'), W, 0);
gl.drawArrays(gl.TRIANGLES, 0, 3);
</script>