Run processes in parallel while capturing video with opencv

Viewed 1246

I'm using opencv to capture a video from my webcam. Every 5 seconds, I'm processing a single frame / an image which can take some seconds. So far everything works. But whenever a frame is processed the entire video is freezing for a couple of seconds (Until the process is finished). I'm trying to get rid of it by using Threading. Here is what I did so far:

Inside the while loop which is capturing the video:

    while True:
        ret, image = cap.read()

        if next_time <= datetime.now():

            content_type = 'image/jpeg'
            headers = {'content-type': content_type}
            _, img_encoded = cv2.imencode('.jpg', image)

            loop = asyncio.get_event_loop()
            future = asyncio.ensure_future(self.async_faces(img_encoded, headers))
            loop.run_until_complete(future)

            next_time += period
            ...

        cv2.imshow('img', image)

Here are the methods:

async def async_faces(self, img, headers):
    with ThreadPoolExecutor(max_workers=10) as executor:

        loop = asyncio.get_event_loop()

        tasks = [
            loop.run_in_executor(
                executor,
                self.face_detection,
                *(img, headers)  # Allows us to pass in multiple arguments to `fetch`
            )
        ]

        for response in await asyncio.gather(*tasks):
            pass

def face_detection(self, img, headers):
    try:
        response = requests.post(self.url, data=img.tostring(), headers=headers)
        ...
    except Exception as e:
        ...

    ...

But unfortunately it's not working.

EDIT 1

In the following I add what the whole thing is supposed to do.

Originally, the function looked like:

import requests
import cv2
from datetime import datetime, timedelta

def face_recognition(self):

    # Start camera
    cap = cv2.VideoCapture(0)

    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1920)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 1080)

    emotional_states = []
    font = cv2.FONT_HERSHEY_SIMPLEX

    period = timedelta(seconds=self.time_period)
    next_time = datetime.now() + period

    cv2.namedWindow('img', cv2.WND_PROP_FULLSCREEN)
    cv2.setWindowProperty('img', cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)

    while True:
        ret, image = cap.read()

        if next_time <= datetime.now():

            # Prepare headers for http request
            content_type = 'image/jpeg'
            headers = {'content-type': content_type}
            _, img_encoded = cv2.imencode('.jpg', image)

            try:
                # Send http request with image and receive response
                response = requests.post(self.url, data=img_encoded.tostring(), headers=headers)
                emotional_states = response.json().get("emotions")
                face_locations = response.json().get("locations")
            except Exception as e:
                emotional_states = []
                face_locations = []
                print(e)

            next_time += period

        for i in range(0, len(emotional_states)):
            emotion = emotional_states[i]
            face_location = face_locations[i]
            cv2.putText(image, emotion, (int(face_location[0]), int(face_location[1])),
                        font, 0.8, (0, 255, 0), 2, cv2.LINE_AA)

        cv2.imshow('img', image)
        k = cv2.waitKey(1) & 0xff
        if k == 27:
            cv2.destroyAllWindows()
            cap.release()
            break
        if k == ord('a'):
            cv2.resizeWindow('img', 700,700)

I use the above method to film myself. This film will be shown live on my screen. Further, every 5 seconds one frame is send to an API where the image is processed in such a way that the emotion of the person in the image is returned. This emotion is displayed on my screen, next to myself. The problem is, that the live video is freezing for a couple of seconds until the emotion is returned from the API.

My OS is Ubuntu.

EDIT 2

The API is running locally. I created a Flask App and the following method is receiving the request:

from flask import Flask, request, Response
import numpy as np
import cv2
import json

@app.route('/api', methods=['POST'])
def facial_emotion_recognition():

    # Convert string of image data to uint8
    nparr = np.fromstring(request.data, np.uint8)
    # Decode image
    img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)

    # Analyse the image
    emotional_state, face_locations = emotionDetection.analyze_facial_emotions(img)

    json_dump = json.dumps({'emotions': emotional_state, 'locations': face_locations}, cls=NumpyEncoder)

    return Response(json_dump, mimetype='application/json')
0 Answers
Related