Stream video to Flask app using flask_socketio

Question

I have a video element whos src is set as the feed from my webcam. This is done using OpenCV (opencvjs).

Using opencv, I read a frame and render it into a canvas.

I then read the data from the canvas and using socketio, I emit to my server app, which is a Flask app, as base64...

Server-side, I catch the incoming video frame, decode from base64, do some processing, reencode to base64, and send it back to the client.

This whole base64 -> image -> base64 is nonsense and horribly inefficient. However, this is my first Flask app and I've not really touched web dev for a number of years, so I am bit confused here.

How can I send the video from my browser to the server, do processing, and send it back to the client without all the base64 conversion? I'm looking for a more efficient way to do this.

Here is my code:

index.html




    
    Title
    
    
    



    
        
            Live Streaming
            
        
    
    
        
            Edited video
            
        
    
    
        
            SOCKETIO IMAGES

index.js

let video = null; 

let width = 320;
let height = 0;

let src = null;
let dst = null;

let streaming = false;
let stream = null;
let vc = null;

let socket = null;

function processVideo() {

    vc.read(src);
    cv.imshow("canvasOutput", src);

    var type = "image/png"
    var data = document.getElementById("canvasOutput").toDataURL(type);
    data = data.replace('data:' + type + ';base64,', ''); 

    socket.emit('image', data);

    requestAnimationFrame(processVideo);
}

function stopVideoProcessing() {
    if (src != null && !src.isDeleted()) src.delete();
}

function stopCamera() {
    if (!streaming) return;
    stopVideoProcessing();
    document.getElementById("canvasOutput").getContext("2d").clearRect(0, 0, width, height);
    video.pause();
    video.srcObject=null;
    stream.getVideoTracks()[0].stop();
    streaming = false;
}

function startVideoProcessing() {
    if (!streaming) { console.warn("Please startup your webcam"); return; }
    stopVideoProcessing();
    src = new cv.Mat(height, width, cv.CV_8UC4);
    requestAnimationFrame(processVideo);
}

function startCamera() {
    if (streaming) return;
    navigator.mediaDevices.getUserMedia({video: true, audio: false})
        .then(function(s) {
            stream = s;
            video.srcObject = s;
            video.play();
        })
        .catch(function(err) {
            console.log("An error occured! " + err);
        });

    video.addEventListener("canplay", function(ev){
        if (!streaming) {
            height = video.videoHeight / (video.videoWidth/width);
            video.setAttribute("width", width);
            video.setAttribute("height", height);
            streaming = true;
            vc = new cv.VideoCapture(video);
        }
        startVideoProcessing();
    }, false);
}

function onOpenCvReady() {
   console.log('OpenCV.js is readyyyyy.');
}

$( document ).ready(function() {
    console.log('document ready');

    video = document.getElementById("videoInput"); 
    video.width = 640;
    video.height = 480;
    startCamera();

    socket = io('http://localhost:8080');
    socket.on('connect', function(){
        console.log("Connected...!", socket.connected)
    });
    socket.on('response_back', function(image){
        console.log('got response back!');
        const image_id = document.getElementById('imagebox');
        image_id.src = image;
    });
});

server.py

#Import necessary libraries
from flask import Flask, render_template, Response, request , jsonify
import cv2
import random
import json
import numpy as np
import base64
import os , io , sys
from PIL import Image
from flask_socketio import SocketIO, emit
import imutils


#Initialize the Flask app
app = Flask(__name__)
socketio = SocketIO(app=app,logger=True)


@socketio.on('image')
def image(image):

    sbuf = io.StringIO()
    sbuf.write(image)

    # decode and convert into image
    b = io.BytesIO(base64.b64decode(image))
    pimg = Image.open(b)

    ## converting RGB to BGR, as opencv standards
    frame = cv2.cvtColor(np.array(pimg), cv2.COLOR_RGB2BGR)

    # Process the image frame
    # frame = imutils.resize(frame, width=700)
    frame = cv2.flip(frame, 1)
    imgencode = cv2.imencode('.jpg', frame)[1]

    # base64 encode
    stringData = base64.b64encode(imgencode).decode('utf-8')
    b64_src = 'data:image/jpg;base64,'
    stringData = b64_src + stringData

    # emit the frame back
    emit('response_back', stringData)


@app.route('/')
def index():
    return render_template('index.html')

if __name__ == "__main__":
    app.run(host='0.0.0.0', port=8080, debug=False)

Stream video to Flask app using flask_socketio

0 Answers