WASM video filters

source code

Now let's see if we can use the same filters on video. Our demo will be based on the MDN tutorial from Manipulating video using canvas.

Input

Output


Code

Get video information

const video = document.getElementById('video');
const width = 320;
const height = 192;

Set up canvas

const hex_output = document.getElementById('hex');

const input_canvas = document.getElementById('input_canvas');
input_canvas.width = width;
input_canvas.height = height;
const input_ctx = input_canvas.getContext('2d');
input_ctx.fillStyle = 'green';
input_ctx.fillRect(0, 0, width, height);
const output_canvas = document.getElementById('output_canvas');
output_canvas.width = width;
output_canvas.height = height;
output_canvas.style.background = 'url(avatar.png)';
const output_ctx = output_canvas.getContext('2d');
output_ctx.fillStyle = 'blue';
output_ctx.fillRect(0, 0, width, height);

Global variables and fucntions

let input_imageData = input_ctx.getImageData(0, 0, width, height);
let output_imageData = output_ctx.getImageData(0, 0, width, height);
const length = input_imageData.data.length;
let x = 0;
let y = 0;
let offset = 0;

Heap and imageData

const mem = new WebAssembly.Memory({ initial: 4, maximum: 4 });
let heap = new Uint8ClampedArray(mem.buffer);

for (let index = 0; index < length; index++) {
    heap[index] = input_imageData.data[index];
};

Canvas UI

const check_boundary = () => {
    if (x < 0) { x = 0 }
    if (x > width - 1) { x = 0 }
    if (y < 0) { y = 0 }
    if (y > height - 1) { y = 0 }
    offset = 4 * (y * width + x);
}

const move = (e) => {
    e.preventDefault();
    const rect = output_canvas.getBoundingClientRect();

    const { changedTouches } = e;
    if (changedTouches && changedTouches[0]) {
        x = Math.floor(changedTouches[0].pageX - output_canvas.offsetLeft);
        y = Math.floor(changedTouches[0].pageY - output_canvas.offsetTop);
    }
    else {
        x = Math.floor(e.clientX - rect.left);
        y = Math.floor(e.clientY - rect.top);
    }
    check_boundary();
    hexdump();
};

output_canvas.addEventListener('mousemove', move, false);
output_canvas.addEventListener('touchmove', move, false);

const hexdump = () => {
    const { data } = output_imageData;
    let output = '';
    for (let i = offset; i < offset + 0x100; i += 4) {
        if (i < data.length - 4) {
            for (let j = 3; j > -1; j--) {
                if (data[i + j] < 0x10) {
                    output += `0${data[i + j].toString(16)}`;
                } else {
                    output += `${data[i + j].toString(16)}`;
                }
            }
            if (((i - offset) % 0x20) === 0x1c) {
                output += '\n';
            }
            else {
                output += ' ';
            }
        }
    }
    hex_output.innerHTML = `
x: 0x${parseInt(x).toString(16)}
y: 0x${parseInt(y).toString(16)}
offset: 0x${offset.toString(16)} 
(mouse point = upper left corner of hexdump)        

(8 x 8) i32 = 256 bytes (1/256 of the total memory of a 64K page)
(0x${0x10000.toString(16)} = 0x${0x100.toString(16)} * 0x${0x100.toString(16)} = ${0x100 * 0x100} )

${output}
`;
};

const canvas_render = () => {
    for (let i = 0; i < length; i++) {
        output_imageData.data[i] = heap[i];
    }
    output_ctx.putImageData(output_imageData, 0, 0);
}

WASM

RTFM

const magic = [0x00, 0x61, 0x73, 0x6d];
const version = [0x01, 0x00, 0x00, 0x00];

main module

const main = {
    section_01: [
        0x01, // type section
        0x15, // 21 bytes
        0x04, // number of functions
        0x60, // import alpha filter
        0x01, // takes 1 param ()
        0x7f, // param i32
        0x01, 
        0x7f, // result i32
        0x60, // import grayscale filter
        0x01, // takes 1 param ()
        0x7f, // param i32
        0x01, 
        0x7f, // result i32
        0x60, // export alpha
        0x02, // takes two params (range)
        0x7f, // param i32
        0x7f, // param i32
        0x00, // no output
        0x60, // export grayscale
        0x02, // takes two params (range)
        0x7f, // param i32
        0x7f, // param i32
        0x00, // no output
    ],
    section_02: [
        0x02, // import section
        0x20, // 32 bytes
        0x03, // 3 imports
        0x01, // 1 byte
        0x6a, // j
        0x01, // 1 byte
        0x6d, // m
        0x02, // mem import
        0x00, // min pages
        0x01, // max pages
        0x01, // 1 byte
        0x6a, // j
        0x05, // 5 byte name
        0x61, // a
        0x6c, // l
        0x70, // p
        0x68, // h
        0x61, // a
        0x00, // func
        0x00, // alpha filter
        0x01, // 1 byte
        0x6a, // j
        0x09, // 9 byte name
        0x67, // g
        0x72, // r
        0x61, // a
        0x79, // y
        0x73, // s
        0x63, // c
        0x61, // a
        0x6c, // l
        0x65, // e
        0x00, // func
        0x01, // grayscale filter
    ],
    section_03: [
        0x03, // func section
        0x03, // 2 bytes
        0x02, // number of functions
        0x02, // alpha
        0x03, // grayscale 
    ],
    section_07: [
        0x07, // export section
        0x15, // 21 bytes
        0x02, // number of exports
        0x05, // 5 byte name
        0x61, // a
        0x6c, // l
        0x70, // p
        0x68, // h
        0x61, // a
        0x00, // function
        0x02,  // alpha
        0x09, // 9 byte name
        0x67, // g
        0x72, // r
        0x61, // a
        0x79, // y
        0x73, // s
        0x63, // c
        0x61, // a
        0x6c, // l
        0x65, // e
        0x00, // function
        0x03  // grayscale
    ],
    section_0a: [
        0x0a, // code section 
        0x45, // 69 bytes
        0x02, // 2 function body
        0x21, // 33 bytes
        0x00, // no more local 
        0x03, // loop
        0x40, // block
        0x20, 0x00, // get address
        0x20, 0x00, // get address
        0x28, 0x02, 0x00, // i32.load the value
        0x10, 0x00, // call alpha
        0x36, 0x02, 0x00, // store 
        0x20, 0x00, // get address
        0x20, 0x01, // upper limit
        0x4b, // i32.gt_u compare
        0x0d, 0x01, // br_if
        0x20, 0x00, // get address
        0x41, 0x04, // i32.const = 4
        0x6a, // i32.add 
        0x21, 0x00, // set new address
        0x0c, 0x00, // br 0 
        0x0b, // end block
        0x0b,  // end code
        0x21, // 33 bytes
        0x00, // no more local 
        0x03, // loop
        0x40, // block
        0x20, 0x00, // get address
        0x20, 0x00, // get address
        0x28, 0x02, 0x00, // i32.load the value
        0x10, 0x01, // call grayscale
        0x36, 0x02, 0x00, // store 
        0x20, 0x00, // get address
        0x20, 0x01, // upper limit
        0x4b, // i32.gt_u compare
        0x0d, 0x01, // br_if
        0x20, 0x00, // get address
        0x41, 0x04, // i32.const = 4
        0x6a, // i32.add 
        0x21, 0x00, // set new address
        0x0c, 0x00, // br 0 
        0x0b, // end block
        0x0b  // end code
    ]
};

main.wasm = new Uint8Array(
    magic.concat(version)
        .concat(main.section_01)
        .concat(main.section_02)
        .concat(main.section_03)
        .concat(main.section_07)
        .concat(main.section_0a)
    );

alpha_filter

const alpha_filter = {
    section_01: [
        0x01, // type section
        0x06, // 6 bytes
        0x01, // number of functions
        0x60, // filter
        0x01, // takes 1 param ()
        0x7f, // param i32
        0x01, 
        0x7f // result i32
    ],
    section_03: [
        0x03, // func section
        0x02, // 2 bytes
        0x01, // number of functions
        0x00  // filter
    ],
    section_07: [
        0x07, // export section
        0x0a, // 10 bytes
        0x01, // number of exports
        0x06, // 6 byte name
        0x66, // f
        0x69, // i
        0x6c, // l
        0x74, // t
        0x65, // e
        0x72, // r
        0x00, // function
        0x00  // filter
    ],
    section_0a: [
        0x0a, // code section
        0x39, // 57 bytes
        0x01, // number of function
        0x37, // 55 bytes filter
        0x00, // no extra local var
        0x20, 0x00, // local.get
        0x41, 0xff, 0x01, // mask for r i32.const 0x000000ff
        0x71, // i32.and
        0x41, 0xe4, 0x00, //
        0x4b, // i32.gt_u 
        0x04, // r > 100
        0x40, // block
        0x20, 0x00, // local.get
        0x41, 0x80, 0xfe, 0x03, // mask for g i32.const 0x0000ff00
        0x71, // i32.and
        0x41, 0x08, 0x76, // right shift two bytes
        0x41, 0xe4, 0x00, //
        0x4b, // i32.gt_u 
        0x04, // g > 100
        0x40, // block
        0x20, 0x00, // local.get
        0x41, 0x80, 0x80, 0xfc, 0x07, // mask for b i32.const 0x00ff0000
        0x71, // i32.and
        0x41, 0x10, 0x76, // right shift four bytes
        0x41, 0x20, //
        0x49, // i32.lt_u 
        0x04, // b < 43
        0x40, // block
        0x41, 0x00,
        0x21, 0x00, // set transparent pixel
        0x0b,
        0x0b,
        0x0b,
        0x20, 0x00, // local.get
        0x0b // opcode for end
    ]
};

alpha_filter.wasm = new Uint8Array(
    magic.concat(version)
        .concat(alpha_filter.section_01)
        .concat(alpha_filter.section_03)
        .concat(alpha_filter.section_07)
        .concat(alpha_filter.section_0a)
    );

grayscale_filter

const grayscale_filter = {
    section_01: [
        0x01, // type section
        0x06, // 6 bytes
        0x01, // number of functions
        0x60, // filter
        0x01, // takes 1 param ()
        0x7f, // param i32
        0x01, 
        0x7f // result i32
    ],
    section_03: [
        0x03, // func section
        0x02, // 2 bytes
        0x01, // number of functions
        0x00  // filter
    ],
    section_07: [
        0x07, // export section
        0x0a, // 10 bytes
        0x01, // number of exports
        0x06, // 6 byte name
        0x66, // f
        0x69, // i
        0x6c, // l
        0x74, // t
        0x65, // e
        0x72, // r
        0x00, // function
        0x00  // filter
    ],
    section_0a: [
        0x0a, // code section
        0x4b, // 75 bytes
        0x01, // number of function
        0x49, // 73 bytes filter
        0x00, // no extra local var
        0x20, 0x00, // local.get
        0x41, 0xff, 0x01, // mask for r i32.const 0x000000ff
        0x71, // i32.and
        0xb3, // f32.convert_i32_u
        0x43, 0x6c, 0x09, 0x99, 0x3e, // f32.const 0.2989
        0x94, // f32.mul
        0x20, 0x00, // local.get
        0x41, 0x80, 0xfe, 0x03, // mask for g i32.const 0x0000ff00
        0x71, // i32.and
        0x41, 0x08, 0x76, // right shift two bytes
        0xb3, // f32.convert_i32_u
        0x43, 0xa2, 0x45, 0x16, 0x3f, // f32.const 0.5870
        0x94, // f32.mul
        0x92, // f32.add
        0x20, 0x00, // local.get
        0x41, 0x80, 0x80, 0xfc, 0x07, // mask for b i32.const 0x00ff0000
        0x71, // i32.and
        0x41, 0x10, 0x76, // right shift four bytes
        0xb3, // f32.convert_i32_u
        0x43, 0xd5, 0x78, 0xe9, 0x3d, // f32.const 0.1140
        0x94, // f32.mul
        0x92, // f32.add
        0xa9, // i32.trunc_f32_u
        0x22, 0x00, // local.tee (add r channel)
        0x20, 0x00, // local.get
        0x41, 0x08, 0x74, 0x6a, // add g channel
        0x20, 0x00, // local.get
        0x41, 0x10, 0x74, 0x6a, // add b channel
        0x41, 0x80, 0x80, 0x80, 0x78, // a i32.const 0xff000000
        0x6a, // add a channel 
        0x0b // opcode for end
    ]
};

grayscale_filter.wasm = new Uint8Array(
    magic.concat(version)
        .concat(grayscale_filter.section_01)
        .concat(grayscale_filter.section_03)
        .concat(grayscale_filter.section_07)
        .concat(grayscale_filter.section_0a)
    );

Instantiate WASM modules


alpha_filter.module = await WebAssembly.compile(alpha_filter.wasm.buffer);
alpha_filter.importObject = { };
alpha_filter.instance = await WebAssembly.instantiate(
    alpha_filter.module, 
    alpha_filter.importObject);

grayscale_filter.module = await WebAssembly.compile(grayscale_filter.wasm.buffer);
grayscale_filter.importObject = { };
grayscale_filter.instance = await WebAssembly.instantiate(
    grayscale_filter.module, 
    grayscale_filter.importObject);

main.module = await WebAssembly.compile(main.wasm.buffer);
main.importObject = { 
    j: {
         m: mem, 
         alpha: alpha_filter.instance.exports.filter,
         grayscale: grayscale_filter.instance.exports.filter 
    }
};

main.instance = await WebAssembly.instantiate(
    main.module, 
    main.importObject);



const play = () =>{
    if (video.paused || video.ended) {
      return;
    }

    input_ctx.drawImage(video, 0, 0, width, height);
    input_imageData = input_ctx.getImageData(0, 0, width, height);
    
    for (let index = 0; index < length; index++) {
       heap[index] = input_imageData.data[index];
    };

    main.instance.exports.alpha(
        4 * Math.floor(length / 12),
        4 * Math.floor(length / 6));
    
    main.instance.exports.grayscale(
        4 * Math.floor(length / 6), 
        4 * Math.floor(length / 4 -  2));

    canvas_render();
    setTimeout(() => { play()}, 0); // loop
};

video.addEventListener('play', play, false);

hexdump();