How can I read a camera and display the images at the cameras frame rate?
I want to continuously read images from my webcam, (do some fast preprocessing) and then display the image in a window. This should run at the frame rate, that my webcam provides (29 fps). It seems like the OpenCV GUI and Tkinter GUI is too slow, to display images at such a frame rate. These are clearly the bottlenecks in my experiments. Even without the preprocessing, the images are not displayed fast enough. I am on a MacBook Pro 2018.
Here is what I tried. The webcam is always read with OpenCV:
- Everything happens in the main thread, the images are displayed with OpenCV: 12 fps
- Read camera and do preprocessing in separate threads, show image with OpenCV in the main thread: 20 fps
- multithreaded like above, but do not show the image: 29 fps
- multithreaded like above, but show the images with Tkinter: don't know the exact fps but it feels like <10 fps.
Here is the code:
Single loop, OpenCV GUI:
import cv2
import time
def main():
cap = cv2.VideoCapture(0)
window_name = "FPS Single Loop"
cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
start_time = time.time()
frames = 0
seconds_to_measure = 10
while start_time + seconds_to_measure > time.time():
success, img = cap.read()
img = img[:, ::-1] # mirror
time.sleep(0.01) # simulate some processing time
cv2.imshow(window_name, img)
cv2.waitKey(1)
frames = frames + 1
cv2.destroyAllWindows()
print(
f"Captured {frames} in {seconds_to_measure} seconds. FPS: {frames/seconds_to_measure}"
)
if __name__ == "__main__":
main()
Captured 121 in 10 seconds. FPS: 12.1
Multithreaded, opencv gui:
import logging
import time
from queue import Full, Queue
from threading import Thread, Event
import cv2
logger = logging.getLogger("VideoStream")
def setup_webcam_stream(src=0):
cap = cv2.VideoCapture(src)
width, height = (
cap.get(cv2.CAP_PROP_FRAME_WIDTH),
cap.get(cv2.CAP_PROP_FRAME_HEIGHT),
)
logger.info(f"Camera dimensions: {width, height}")
logger.info(f"Camera FPS: {cap.get(cv2.CAP_PROP_FPS)}")
grabbed, frame = cap.read() # Read once to init
if not grabbed:
raise IOError("Cannot read video stream.")
return cap
def video_stream_loop(video_stream: cv2.VideoCapture, queue: Queue, stop_event: Event):
while not stop_event.is_set():
try:
success, img = video_stream.read()
# We need a timeout here to not get stuck when no images are retrieved from the queue
queue.put(img, timeout=1)
except Full:
pass # try again with a newer frame
def processing_loop(input_queue: Queue, output_queue: Queue, stop_event: Event):
while not stop_event.is_set():
try:
img = input_queue.get()
img = img[:, ::-1] # mirror
time.sleep(0.01) # simulate some processing time
# We need a timeout here to not get stuck when no images are retrieved from the queue
output_queue.put(img, timeout=1)
except Full:
pass # try again with a newer frame
def main():
stream = setup_webcam_stream(0)
webcam_queue = Queue()
processed_queue = Queue()
stop_event = Event()
window_name = "FPS Multi Threading"
cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
start_time = time.time()
frames = 0
seconds_to_measure = 10
try:
Thread(
target=video_stream_loop, args=[stream, webcam_queue, stop_event]
).start()
Thread(
target=processing_loop, args=[webcam_queue, processed_queue, stop_event]
).start()
while start_time + seconds_to_measure > time.time():
img = processed_queue.get()
cv2.imshow(window_name, img)
cv2.waitKey(1)
frames = frames + 1
finally:
stop_event.set()
cv2.destroyAllWindows()
print(
f"Captured {frames} frames in {seconds_to_measure} seconds. FPS: {frames/seconds_to_measure}"
)
print(f"Webcam queue: {webcam_queue.qsize()}")
print(f"Processed queue: {processed_queue.qsize()}")
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
main()
INFO:VideoStream:Camera dimensions: (1280.0, 720.0)
INFO:VideoStream:Camera FPS: 29.000049
Captured 209 frames in 10 seconds. FPS: 20.9
Webcam queue: 0
Processed queue: 82
Here you can see that there are images remaining in the second queue where the images get fetched for displaying them.
When I uncomment these two lines:
cv2.imshow(window_name, img)
cv2.waitKey(1)
then the output is:
INFO:VideoStream:Camera dimensions: (1280.0, 720.0)
INFO:VideoStream:Camera FPS: 29.000049
Captured 291 frames in 10 seconds. FPS: 29.1
Webcam queue: 0
Processed queue: 0
So it is able to process all frames at the webcams speed without a GUI displaying them.
Multithreaded, Tkinter gui:
import logging
import time
import tkinter
from queue import Full, Queue, Empty
from threading import Thread, Event
import PIL
from PIL import ImageTk
import cv2
logger = logging.getLogger("VideoStream")
def setup_webcam_stream(src=0):
cap = cv2.VideoCapture(src)
width, height = cap.get(cv2.CAP_PROP_FRAME_WIDTH), cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
logger.info(f"Camera dimensions: {width, height}")
logger.info(f"Camera FPS: {cap.get(cv2.CAP_PROP_FPS)}")
grabbed, frame = cap.read() # Read once to init
if not grabbed:
raise IOError("Cannot read video stream.")
return cap, width, height
def video_stream_loop(video_stream: cv2.VideoCapture, queue: Queue, stop_event: Event):
while not stop_event.is_set():
try:
success, img = video_stream.read()
# We need a timeout here to not get stuck when no images are retrieved from the queue
queue.put(img, timeout=1)
except Full:
pass # try again with a newer frame
def processing_loop(input_queue: Queue, output_queue: Queue, stop_event: Event):
while not stop_event.is_set():
try:
img = input_queue.get()
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = img[:, ::-1] # mirror
time.sleep(0.01) # simulate some processing time
# We need a timeout here to not get stuck when no images are retrieved from the queue
output_queue.put(img, timeout=1)
except Full:
pass # try again with a newer frame
class App:
def __init__(self, window, window_title, image_queue: Queue, image_dimensions: tuple):
self.window = window
self.window.title(window_title)
self.image_queue = image_queue
# Create a canvas that can fit the above video source size
self.canvas = tkinter.Canvas(window, width=image_dimensions[0], height=image_dimensions[1])
self.canvas.pack()
# After it is called once, the update method will be automatically called every delay milliseconds
self.delay = 1
self.update()
self.window.mainloop()
def update(self):
try:
frame = self.image_queue.get(timeout=0.1) # Timeout to not block this method forever
self.photo = ImageTk.PhotoImage(image=PIL.Image.fromarray(frame))
self.canvas.create_image(0, 0, image=self.photo, anchor=tkinter.NW)
self.window.after(self.delay, self.update)
except Empty:
pass # try again next time
def main():
stream, width, height = setup_webcam_stream(0)
webcam_queue = Queue()
processed_queue = Queue()
stop_event = Event()
window_name = "FPS Multi Threading"
try:
Thread(target=video_stream_loop, args=[stream, webcam_queue, stop_event]).start()
Thread(target=processing_loop, args=[webcam_queue, processed_queue, stop_event]).start()
App(tkinter.Tk(), window_name, processed_queue, (width, height))
finally:
stop_event.set()
print(f"Webcam queue: {webcam_queue.qsize()}")
print(f"Processed queue: {processed_queue.qsize()}")
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
main()
INFO:VideoStream:Camera dimensions: (1280.0, 720.0)
INFO:VideoStream:Camera FPS: 29.000049
Webcam queue: 0
Processed queue: 968
