Remove sequentially duplicate frames in p5.js

Question

In the code below I want to get the first 120 frames.
Then I want to remove the sequentially duplicate ones

ffmpeg has a mpdecimate filter that is supposed to drop frames that don't vary greatly from each other.

But I don't know if there is an ffmpeg Js library (or equivalent) to do something like that within a p5.js context.

function setup() {
  frameRate(24)
  video = createCapture(VIDEO)
  video.size(videoWidth, videoHeight)
  video.hide()

  createCanvas(canvasWidth, canvasWidth)
}

let frames       = []
let uniqueFrames = []
let STOP         = false

function draw() {

  if (video && video.loadedmetadata && frameCount <= 100) {
    frames.push(image(video.get()))
  } else if (!STOP) {
    STOP = true
    uniqueFrames = removeDuplicates(frames)
  }
}

user_89766 · Answer

One thing you could do is take absolute difference between two frames.

When you do that the result will be an image that is completely black if all the pixels match otherwise you'll have brighter pixels where the frames differ. It's a basic technique for a very useful one for a bunch of traditional computer vision tasks.

What you could do is write a function that takes in two frames, computes the absolute difference, counts the non zero pixels and returns that. This value would be useful in an if condition against a threshold value: maybe the frames don't need to be 100% identical, but if they're let's say 85% percent identical then you can mark that as a duplicate.

Kyle McDonald's Frame Difference p5.js example is great !

For reference I'm simply listing Kyle's example here:

// https://kylemcdonald.github.io/cv-examples/

var capture;
var previousPixels;
var w = 640;
var h = 480;

function setup() {
    capture = createCapture({
        audio: false,
        video: {
            width: w,
            height: h
        }
    }, function() {
        console.log('capture ready.')
    });
    capture.elt.setAttribute('playsinline', '');
    capture.size(w, h);
    createCanvas(w, h);
    capture.hide();
}

function copyImage(src, dst) {
    var n = src.length;
    if (!dst || dst.length != n) dst = new src.constructor(n);
    while (n--) dst[n] = src[n];
    return dst;
}

function draw() {
    capture.loadPixels();
    var total = 0;
    if (capture.pixels.length > 0) { // don't forget this!
        if (!previousPixels) {
            previousPixels = copyImage(capture.pixels, previousPixels);
        } else {
            var w = capture.width,
                h = capture.height;
            var i = 0;
            var pixels = capture.pixels;
            var thresholdAmount = select('#thresholdAmount').value() * 255. / 100.;
            thresholdAmount *= 3; // 3 for r, g, b
            for (var y = 0; y < h; y++) {
                for (var x = 0; x < w; x++) {
                    // calculate the differences
                    var rdiff = Math.abs(pixels[i + 0] - previousPixels[i + 0]);
                    var gdiff = Math.abs(pixels[i + 1] - previousPixels[i + 1]);
                    var bdiff = Math.abs(pixels[i + 2] - previousPixels[i + 2]);
                    // copy the current pixels to previousPixels
                    previousPixels[i + 0] = pixels[i + 0];
                    previousPixels[i + 1] = pixels[i + 1];
                    previousPixels[i + 2] = pixels[i + 2];
                    var diffs = rdiff + gdiff + bdiff;
                    var output = 0;
                    if (diffs > thresholdAmount) {
                        output = 255;
                        total += diffs;
                    }
                    pixels[i++] = output;
                    pixels[i++] = output;
                    pixels[i++] = output;
                    // also try this:
                    // pixels[i++] = rdiff;
                    // pixels[i++] = gdiff;
                    // pixels[i++] = bdiff;
                    i++; // skip alpha
                }
            }
        }
    }
    // need this because sometimes the frames are repeated
    if (total > 0) {
        select('#motion').elt.innerText = total;
        capture.updatePixels();
        image(capture, 0, 0, 640, 480);
    }
}




    
    DifferenceImage
    
    
    



    
        Threshold: 
    
    
        Motion: 0

See it in action here.

Good luck encapsulating the frame difference code into a reusable function which you can call in removeDiplicates().

Remove sequentially duplicate frames in p5.js

1 Answers