How can I use a matrix from CSS's matrix3d in a WebGL2 shader?

rebane2001
July 2, 2024
172 views
0 votes
2 Answers

I currently have a CSS matrix3d transform I like, let’s say this one for example:

transform: matrix3d(1.3453,0.1357,0.0,0.0003,0.2096,1.3453,0.0,0.0003,0.0,0.0,1.0,0.0,-100.0,-100.0,0.0,1.0);
transform-origin: 0 0;

I am using a WebGL2 canvas and I’d like an image I’ve drawn on it to have the same transformation.

To achieve this, I believe I need a vertex shader to take in the matrix and multiply it:

#version 300 es

in vec2 a_position;
in vec2 a_texCoord;
uniform vec2 u_resolution;
out vec2 v_texCoord;

uniform mat4 u_matrix;

void main() {
  vec2 zeroToOne = a_position / u_resolution;
  vec2 zeroToTwo = zeroToOne * 2.0;
  vec2 clipSpace = zeroToTwo - 1.0;

  gl_Position = u_matrix*vec4(clipSpace * vec2(1, -1), 0, 1);
  v_texCoord = a_texCoord;
}

I can then pass the matrix to the shader in my JavaScript code:

const matrixLocation = gl.getUniformLocation(program, "u_matrix");
gl.uniformMatrix4fv(matrixLocation, false, new Float32Array(matrix));

What I’m currently stuck at is figuring out what the matrix value in my code should be. I’ve tried reading up on how the matrix3d transform is made up, and rearranging the matrix based on that, but with no luck.

How can I use a matrix3d transform in the WebGL2 shader here?

Edit: A full working example added as per request in the comments

// Set up matrices

// Based on: matrix3d(1.3453,0.1357,0.0,0.0003,0.2096,1.3453,0.0,0.0003,0.0,0.0,1.0,0.0,-100.0,-100.0,0.0,1.0)
const cssMatrix = new Float32Array([1.3453, 0.1357, 0.0, 0.0003, 0.2096, 1.3453, 0.0, 0.0003, 0.0, 0.0, 1.0, 0.0, -100.0, -100.0, 0.0, 1.0]);
const identityMatrix = new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]);

// identityMatrix works correctly while the cssMatrix does not
const matrixArray = cssMatrix;
// const matrixArray = identityMatrix;

// Set up shaders

const vertexShaderSource = `#version 300 es

in vec2 a_position;
in vec2 a_texCoord;

uniform vec2 u_resolution;
uniform mat4 u_matrix;
out vec2 v_texCoord;

void main() {
  vec2 zeroToOne = a_position / u_resolution;
  vec2 zeroToTwo = zeroToOne * 2.0;
  vec2 clipSpace = zeroToTwo - 1.0;

  gl_Position = u_matrix*vec4(clipSpace * vec2(1, -1), 0,1);

  v_texCoord = a_texCoord;
}`;

const fragmentShaderSource = `#version 300 es

precision highp float;
uniform sampler2D u_image;
in vec2 v_texCoord;
out vec4 outColor;

void main() {
  outColor = texture(u_image, v_texCoord);
}`;

// Load the test image

const img = document.getElementById("image");
img.src = "https://i.imgur.com/NIt86ft.png";
img.onload = () => renderImg(img);

// Rest of this code is boilerplate based off of
// https://webgl2fundamentals.org/webgl/lessons/webgl-image-processing.html

function createProgram(
  gl, shaders, opt_attribs, opt_locations, opt_errorCallback) {
  const errFn = opt_errorCallback || console.error;
  const program = gl.createProgram();
  shaders.forEach(function(shader) {
    gl.attachShader(program, shader);
  });
  if (opt_attribs) {
    opt_attribs.forEach(function(attrib, ndx) {
      gl.bindAttribLocation(
        program,
        opt_locations ? opt_locations[ndx] : ndx,
        attrib);
    });
  }
  gl.linkProgram(program);

  // Check the link status
  const linked = gl.getProgramParameter(program, gl.LINK_STATUS);
  if (!linked) {
    // something went wrong with the link
    const lastError = gl.getProgramInfoLog(program);
    errFn('Error in program linking:' + lastError);

    gl.deleteProgram(program);
    return null;
  }
  return program;
}

function loadShader(gl, shaderSource, shaderType, opt_errorCallback) {
  const errFn = opt_errorCallback || console.error;
  // Create the shader object
  const shader = gl.createShader(shaderType);
  // Load the shader source
  gl.shaderSource(shader, shaderSource);
  // Compile the shader
  gl.compileShader(shader);
  // Check the compile status
  const compiled = gl.getShaderParameter(shader, gl.COMPILE_STATUS);
  if (!compiled) {
    // Something went wrong during compilation; get the error
    const lastError = gl.getShaderInfoLog(shader);
    console.error('*** Error compiling shader '' + shader + '':' + lastError + `n` + shaderSource.split('n').map((l, i) => `${i + 1}: ${l}`).join('n'));
    gl.deleteShader(shader);
    return null;
  }
  return shader;
}

function createProgramFromSources(
  gl, shaderSources, opt_attribs, opt_locations, opt_errorCallback) {
  const shaders = [];
  const defaultShaderType = [
    'VERTEX_SHADER',
    'FRAGMENT_SHADER',
  ];
  for (let ii = 0; ii < shaderSources.length; ++ii) {
    shaders.push(loadShader(
      gl, shaderSources[ii], gl[defaultShaderType[ii]], opt_errorCallback));
  }
  return createProgram(gl, shaders, opt_attribs, opt_locations, opt_errorCallback);
}

function resizeCanvasToDisplaySize(canvas, multiplier) {
  multiplier = multiplier || 1;
  const width = canvas.clientWidth * multiplier | 0;
  const height = canvas.clientHeight * multiplier | 0;
  if (canvas.width !== width || canvas.height !== height) {
    canvas.width = width;
    canvas.height = height;
    return true;
  }
  return false;
}

function renderImg(image) {
  const canvas = document.getElementById("canvas");
  const gl = canvas.getContext("webgl2");
  if (!gl)
    return;

  var program = createProgramFromSources(gl, [vertexShaderSource, fragmentShaderSource]);

  var positionAttributeLocation = gl.getAttribLocation(program, "a_position");
  var texCoordAttributeLocation = gl.getAttribLocation(program, "a_texCoord");
  var resolutionLocation = gl.getUniformLocation(program, "u_resolution");
  var imageLocation = gl.getUniformLocation(program, "u_image");
  var matrixLocation = gl.getUniformLocation(program, "u_matrix");

  // Create a vertex array object (attribute state)
  var vao = gl.createVertexArray();

  // and make it the one we're currently working with
  gl.bindVertexArray(vao);

  // Create a buffer and put a single pixel space rectangle in
  // it (2 triangles)
  var positionBuffer = gl.createBuffer();

  // Turn on the attribute
  gl.enableVertexAttribArray(positionAttributeLocation);

  // Bind it to ARRAY_BUFFER (think of it as ARRAY_BUFFER = positionBuffer)
  gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);

  // Tell the attribute how to get data out of positionBuffer (ARRAY_BUFFER)
  var size = 2; // 2 components per iteration
  var type = gl.FLOAT; // the data is 32bit floats
  var normalize = false; // don't normalize the data
  var stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
  var offset = 0; // start at the beginning of the buffer
  gl.vertexAttribPointer(
    positionAttributeLocation, size, type, normalize, stride, offset);

  // provide texture coordinates for the rectangle.
  var texCoordBuffer = gl.createBuffer();
  gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
  gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
    0.0, 0.0,
    1.0, 0.0,
    0.0, 1.0,
    0.0, 1.0,
    1.0, 0.0,
    1.0, 1.0,
  ]), gl.STATIC_DRAW);

  // Turn on the attribute
  gl.enableVertexAttribArray(texCoordAttributeLocation);

  // Tell the attribute how to get data out of texCoordBuffer (ARRAY_BUFFER)
  var size = 2; // 2 components per iteration
  var type = gl.FLOAT; // the data is 32bit floats
  var normalize = false; // don't normalize the data
  var stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
  var offset = 0; // start at the beginning of the buffer
  gl.vertexAttribPointer(
    texCoordAttributeLocation, size, type, normalize, stride, offset);

  // Create a texture.
  var texture = gl.createTexture();

  // make unit 0 the active texture uint
  // (ie, the unit all other texture commands will affect
  gl.activeTexture(gl.TEXTURE0 + 0);

  // Bind it to texture unit 0's 2D bind point
  gl.bindTexture(gl.TEXTURE_2D, texture);

  // Set the parameters so we don't need mips and so we're not filtering
  // and we don't repeat at the edges.
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);

  // Upload the image into the texture.
  var mipLevel = 0; // the largest mip
  var internalFormat = gl.RGBA; // format we want in the texture
  var srcFormat = gl.RGBA; // format of data we are supplying
  var srcType = gl.UNSIGNED_BYTE; // type of data we are supplying
  gl.texImage2D(gl.TEXTURE_2D,
    mipLevel,
    internalFormat,
    srcFormat,
    srcType,
    image);

  gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
  setRectangle(gl, 0, 0, image.width, image.height);

  resizeCanvasToDisplaySize(gl.canvas);
  gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
  gl.clearColor(0, 0, 0, 0);
  gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
  gl.useProgram(program);
  gl.bindVertexArray(vao);
  gl.uniform2f(resolutionLocation, gl.canvas.width, gl.canvas.height);
  gl.uniform1i(imageLocation, 0);

  gl.uniformMatrix4fv(matrixLocation, false, matrixArray);

  gl.drawArrays(gl.TRIANGLES, 0, 6);
}

function setRectangle(gl, x, y, width, height) {
  var x1 = x;
  var x2 = x + width;
  var y1 = y;
  var y2 = y + height;
  gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
    x1, y1,
    x2, y1,
    x1, y2,
    x1, y2,
    x2, y1,
    x2, y2,
  ]), gl.STATIC_DRAW);
}

#image {
  transform: matrix3d(1.3453, 0.1357, 0.0, 0.0003, 0.2096, 1.3453, 0.0, 0.0003, 0.0, 0.0, 1.0, 0.0, -100.0, -100.0, 0.0, 1.0);
  transform-origin: 0 0;
}

img,
canvas {
  border: 1px solid #000;
}

<img src="" crossOrigin="" id="image"><br>
<canvas id="canvas" width=320 height=240></canvas>

Answers

- Mike39Pomax39Kamermans
- June 27, 2024 at 9:01 pm
- 0 votes
0
A CSS matrix3d is a normal 4×4 homogeneous matrix, with the elements ordered by row, so in your case row 1 is 1.3453, 0.1357, 0.0, 0.0003, row 2 is 0.2096, 1.3453, 0.0, 0.0003, row 3 is 0.0, 0.0, 1.0, 0.0, and row 4 is -100.0, -100.0, 0.0, 1.0.

Converting that to a float32 array is pretty easy, you can either work with the CSS string and chop that up to convert the numbers into real numbers and "cast" that to a float32 array, or you can take advantage of the DOMMatrix object and make that do the conversion for you.

E.g.
```
const matrixFromCSS = `matrix3d(.....)`;
const matrixAsFloat32Array = new Float32Array(
  matrixFromCSS 
    .replace(`matrix3d(`,``)
    .split(`,`)
    .map(v => parseFloat(v)
  )
);
```
or
```
const matrixFromCSS = `matrix3d(.....)`;
const matrixAsDOMMatrix = new DOMMatrix(matrixFromCSS);
const matrixAsFloat32Array = matrixAsDOMMatrix.toFloat32Array();
```
Login or Signup to reply.

I just guested by experimenting rather than look up the docs for the correct math so this might not actually work but it’s working for my test. 😅

There’s a bunch of assumptions in this code, like assuming the vertex data is in pixels (because that’s what you had) and so that needs to change if the image size changes.

Another issue you’ll have is 3d css pops the the element out of the normal rectangle the element would fill but WebGL can’t draw outside of the canvas.

I also removed the transform-origin: because that would add to the math. So would perspective: or any other transforms to the css.

One difference from the code in your question. That code was multiplying the projected vertex positions by a matrix (after dividing by the resolution, and multiplying by 2 and subtracting 1).

This code simplified the shader by just multiplying by a single matrix, no other math in the shader, as covered in this article on matrix math

// Set up matrices


// Based on: matrix3d(1.3453,0.1357,0.0,0.0003,0.2096,1.3453,0.0,0.0003,0.0,0.0,1.0,0.0,-100.0,-100.0,0.0,1.0)
const cssMatrix = new Float32Array([
  1.3453, 0.1357, 0.0, 0.0003, 
  0.2096, 1.3453, 0.0, 0.0003,
  0.0, 0.0, 1.0, 0.0,
  -100.0, -100.0, 0.0, 1.0,
]);
const identityMatrix = new Float32Array([
  1, 0, 0, 0,
  0, 1, 0, 0,
  0, 0, 1, 0,
  0, 0, 0, 1,
]);

// identityMatrix works correctly while the cssMatrix does not
const matrixArray = cssMatrix;
//const matrixArray = identityMatrix;

// Set up shaders

const vertexShaderSource = `#version 300 es

in vec4 a_position;
in vec2 a_texCoord;

uniform mat4 u_matrix;
out vec2 v_texCoord;

void main() {
  gl_Position = u_matrix * a_position;

  v_texCoord = a_texCoord;
}`;

const fragmentShaderSource = `#version 300 es

precision highp float;
uniform sampler2D u_image;
in vec2 v_texCoord;
out vec4 outColor;

void main() {
  outColor = texture(u_image, v_texCoord);
}`;

// Load the test image

const img = document.getElementById("image");
img.src = "https://i.imgur.com/NIt86ft.png";
img.onload = () => renderImg(img);

// Rest of this code is boilerplate based off of
// https://webgl2fundamentals.org/webgl/lessons/webgl-image-processing.html

function createProgram(
  gl, shaders, opt_attribs, opt_locations, opt_errorCallback) {
  const errFn = opt_errorCallback || console.error;
  const program = gl.createProgram();
  shaders.forEach(function(shader) {
    gl.attachShader(program, shader);
  });
  if (opt_attribs) {
    opt_attribs.forEach(function(attrib, ndx) {
      gl.bindAttribLocation(
        program,
        opt_locations ? opt_locations[ndx] : ndx,
        attrib);
    });
  }
  gl.linkProgram(program);

  // Check the link status
  const linked = gl.getProgramParameter(program, gl.LINK_STATUS);
  if (!linked) {
    // something went wrong with the link
    const lastError = gl.getProgramInfoLog(program);
    errFn('Error in program linking:' + lastError);

    gl.deleteProgram(program);
    return null;
  }
  return program;
}

function loadShader(gl, shaderSource, shaderType, opt_errorCallback) {
  const errFn = opt_errorCallback || console.error;
  // Create the shader object
  const shader = gl.createShader(shaderType);
  // Load the shader source
  gl.shaderSource(shader, shaderSource);
  // Compile the shader
  gl.compileShader(shader);
  // Check the compile status
  const compiled = gl.getShaderParameter(shader, gl.COMPILE_STATUS);
  if (!compiled) {
    // Something went wrong during compilation; get the error
    const lastError = gl.getShaderInfoLog(shader);
    console.error('*** Error compiling shader '' + shader + '':' + lastError + `n` + shaderSource.split('n').map((l, i) => `${i + 1}: ${l}`).join('n'));
    gl.deleteShader(shader);
    return null;
  }
  return shader;
}

function createProgramFromSources(
  gl, shaderSources, opt_attribs, opt_locations, opt_errorCallback) {
  const shaders = [];
  const defaultShaderType = [
    'VERTEX_SHADER',
    'FRAGMENT_SHADER',
  ];
  for (let ii = 0; ii < shaderSources.length; ++ii) {
    shaders.push(loadShader(
      gl, shaderSources[ii], gl[defaultShaderType[ii]], opt_errorCallback));
  }
  return createProgram(gl, shaders, opt_attribs, opt_locations, opt_errorCallback);
}

function resizeCanvasToDisplaySize(canvas, multiplier) {
  multiplier = multiplier || 1;
  const width = canvas.clientWidth * multiplier | 0;
  const height = canvas.clientHeight * multiplier | 0;
  if (canvas.width !== width || canvas.height !== height) {
    canvas.width = width;
    canvas.height = height;
    return true;
  }
  return false;
}

function renderImg(image) {
  const canvas = document.getElementById("canvas");
  const gl = canvas.getContext("webgl2");
  if (!gl)
    return;

  var program = createProgramFromSources(gl, [vertexShaderSource, fragmentShaderSource]);

  var positionAttributeLocation = gl.getAttribLocation(program, "a_position");
  var texCoordAttributeLocation = gl.getAttribLocation(program, "a_texCoord");
  var resolutionLocation = gl.getUniformLocation(program, "u_resolution");
  var imageLocation = gl.getUniformLocation(program, "u_image");
  var matrixLocation = gl.getUniformLocation(program, "u_matrix");

  // Create a vertex array object (attribute state)
  var vao = gl.createVertexArray();

  // and make it the one we're currently working with
  gl.bindVertexArray(vao);

  // Create a buffer and put a single pixel space rectangle in
  // it (2 triangles)
  var positionBuffer = gl.createBuffer();

  // Turn on the attribute
  gl.enableVertexAttribArray(positionAttributeLocation);

  // Bind it to ARRAY_BUFFER (think of it as ARRAY_BUFFER = positionBuffer)
  gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);

  // Tell the attribute how to get data out of positionBuffer (ARRAY_BUFFER)
  var size = 2; // 2 components per iteration
  var type = gl.FLOAT; // the data is 32bit floats
  var normalize = false; // don't normalize the data
  var stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
  var offset = 0; // start at the beginning of the buffer
  gl.vertexAttribPointer(
    positionAttributeLocation, size, type, normalize, stride, offset);

  // provide texture coordinates for the rectangle.
  var texCoordBuffer = gl.createBuffer();
  gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
  gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
    0.0, 0.0,
    1.0, 0.0,
    0.0, 1.0,
    0.0, 1.0,
    1.0, 0.0,
    1.0, 1.0,
  ]), gl.STATIC_DRAW);

  // Turn on the attribute
  gl.enableVertexAttribArray(texCoordAttributeLocation);

  // Tell the attribute how to get data out of texCoordBuffer (ARRAY_BUFFER)
  var size = 2; // 2 components per iteration
  var type = gl.FLOAT; // the data is 32bit floats
  var normalize = false; // don't normalize the data
  var stride = 0; // 0 = move forward size * sizeof(type) each iteration to get the next position
  var offset = 0; // start at the beginning of the buffer
  gl.vertexAttribPointer(
    texCoordAttributeLocation, size, type, normalize, stride, offset);

  // Create a texture.
  var texture = gl.createTexture();

  // make unit 0 the active texture uint
  // (ie, the unit all other texture commands will affect
  gl.activeTexture(gl.TEXTURE0 + 0);

  // Bind it to texture unit 0's 2D bind point
  gl.bindTexture(gl.TEXTURE_2D, texture);

  // Set the parameters so we don't need mips and so we're not filtering
  // and we don't repeat at the edges.
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
  gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);

  // Upload the image into the texture.
  var mipLevel = 0; // the largest mip
  var internalFormat = gl.RGBA; // format we want in the texture
  var srcFormat = gl.RGBA; // format of data we are supplying
  var srcType = gl.UNSIGNED_BYTE; // type of data we are supplying
  gl.texImage2D(gl.TEXTURE_2D,
    mipLevel,
    internalFormat,
    srcFormat,
    srcType,
    image);

  gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
  setRectangle(gl, 0, 0, image.width, image.height);

  function render(time) {
    time *= 0.001; // convert to seconds
    resizeCanvasToDisplaySize(gl.canvas);
    gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
    gl.clearColor(0, 0, 0, 0);
    gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
    gl.useProgram(program);
    gl.bindVertexArray(vao);
    gl.uniform2f(resolutionLocation, gl.canvas.width, gl.canvas.height);
    gl.uniform1i(imageLocation, 0);

    // compute some CSSMatrix
    const m = m4.perspective(
      10 * Math.PI / 180,
      canvas.clientWidth / canvas.clientHeight,
      10, 500);
    const cm = m4.lookAt(
      [0, 0, 50],  // camera
      [0, 0, 0],   // target
      [0, 1, 0],   // up
    );
    const view = m4.inverse(cm);
    m4.multiply(m, view, m);
    m4.zRotate(m, time, m);
    m4.yRotate(m, Math.sin(time) * 0.25, m);
    
    // Convert the CSSMatrix to WebGL?
    // Note: There's an assumption here that the vertex data
    // is in pixels which is a poor assumption IMO. It would be better,
    // at least for the simple case, to use a unit square around the origin
    const mat = m4.identity();
    m4.orthographic(0, canvas.width, canvas.height, 0, -100, 100, mat);
    m4.translate(mat, img.width / 2, img.height / 2, 0, mat); 
    m4.multiply(mat, m, mat);
    m4.translate(mat, -img.width / 2, -img.height / 2, 0, mat); 
    gl.uniformMatrix4fv(matrixLocation, false, mat);

    img.style.transform = `matrix3d(${[...m].join(',')})`;

    gl.drawArrays(gl.TRIANGLES, 0, 6);
    requestAnimationFrame(render);
  }
  requestAnimationFrame(render);
}

function setRectangle(gl, x, y, width, height) {
  var x1 = x;
  var x2 = x + width;
  var y1 = y;
  var y2 = y + height;
  gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([
    x1, y1,
    x2, y1,
    x1, y2,
    x1, y2,
    x2, y1,
    x2, y2,
  ]), gl.STATIC_DRAW);
}

#image {
}

img,
canvas {
  border: 1px solid #000;
}

<img src="" crossOrigin="" id="image"><br>
<canvas id="canvas" width=320 height=240></canvas>
<script src="https://webgl2fundamentals.org/webgl/resources/m4.js"></script>

The code above is not using your matrix but if you comment out all the code that manipulates m and just change it to const m = matrixArray then you should see they match.

Please signup or login to give your own answer.

Click here to cancel reply.