I have an an app, where I get all the frames from camera using AVFoundation, and process using the code below. I was wondering if there is a way to make this part multi threaded, so it can run faster. Maybe putting each frame in a queue in one thread, another thread to process the queue, and one queue to show the output of each frame as the output? I don't know if this can be done, but this is because the processing of each frame might take more time to process, and the image freeze in the output as a result.
This is the code for CaptureManager class:
class CaptureManager: NSObject {
internal static let shared = CaptureManager()
weak var delegate: CaptureManagerDelegate?
var session: AVCaptureSession?
var isBackCamera = true
override init() {
super.init()
session = AVCaptureSession()
session?.sessionPreset = .high
//setup input
var device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back)
let defaults = UserDefaults.standard
if let stringOne = defaults.string(forKey: defaultsKeys.rememberCamera) {
if(stringOne != "back"){
device = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .front)
}
}else{
defaults.set("back", forKey: defaultsKeys.rememberCamera)
}
if(device != nil){
device?.set(frameRate: 30)
let input = try! AVCaptureDeviceInput(device: device!)
session?.addInput(input)
//setup output
let output = AVCaptureVideoDataOutput()
output.alwaysDiscardsLateVideoFrames = true
output.videoSettings = [kCVPixelBufferPixelFormatTypeKey as AnyHashable as! String: kCVPixelFormatType_32BGRA]
output.setSampleBufferDelegate(self, queue: DispatchQueue.main)
session?.addOutput(output)
}else{
print("no camera")
}
}
func startSession() {
session?.startRunning()
}
func stopSession() {
session?.stopRunning()
}
func switchCamera(){
//Remove existing input
guard let currentCameraInput: AVCaptureInput = session?.inputs.first else {
return
}
//Indicate that some changes will be made to the session
session?.beginConfiguration()
session?.removeInput(currentCameraInput)
let defaults = UserDefaults.standard
if let stringOne = defaults.string(forKey: defaultsKeys.rememberCamera) {
if(stringOne == "back"){
defaults.set("front", forKey: defaultsKeys.rememberCamera)
}else{
defaults.set("back", forKey: defaultsKeys.rememberCamera)
}
}
//Get new input
var newCamera: AVCaptureDevice! = nil
if let input = currentCameraInput as? AVCaptureDeviceInput {
if (input.device.position == .back) {
newCamera = cameraWithPosition(position: .front)
} else {
newCamera = cameraWithPosition(position: .back)
}
}
newCamera.set(frameRate: 30)
//Add input to session
var err: NSError?
var newVideoInput: AVCaptureDeviceInput!
do {
newVideoInput = try AVCaptureDeviceInput(device: newCamera)
} catch let err1 as NSError {
err = err1
newVideoInput = nil
}
if newVideoInput == nil || err != nil {
print("Error creating capture device input: \(err?.localizedDescription)")
} else {
session?.addInput(newVideoInput)
}
isBackCamera.toggle()
//Commit all the configuration changes at once
session?.commitConfiguration()
}
func cameraWithPosition(position: AVCaptureDevice.Position) -> AVCaptureDevice? {
let discoverySession = AVCaptureDevice.DiscoverySession(deviceTypes: [.builtInWideAngleCamera], mediaType: AVMediaType.video, position: .unspecified)
for device in discoverySession.devices {
if device.position == position {
return device
}
}
return nil
}
func getImageFromSampleBuffer(sampleBuffer: CMSampleBuffer) ->UIImage? {
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
return nil
}
CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
let baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer)
let width = CVPixelBufferGetWidth(pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer)
let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
let colorSpace = CGColorSpaceCreateDeviceRGB()
let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue)
guard let context = CGContext(data: baseAddress, width: width, height: height, bitsPerComponent: 8, bytesPerRow: bytesPerRow, space: colorSpace, bitmapInfo: bitmapInfo.rawValue) else {
return nil
}
guard let cgImage = context.makeImage() else {
return nil
}
var image: UIImage
let defaults = UserDefaults.standard
if let stringOne = defaults.string(forKey: defaultsKeys.rememberCamera) {
if(stringOne == "back"){
image = UIImage(cgImage: cgImage, scale: 1, orientation:.right)
}else{
image = UIImage(cgImage: cgImage, scale: 1, orientation:.leftMirrored)
}
}else{
image = UIImage(cgImage: cgImage, scale: 1, orientation:.right)
}
CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
return image
}
}
This is the extention to process the each frame:
extension CaptureManager: AVCaptureVideoDataOutputSampleBufferDelegate {
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard let outputImage = getImageFromSampleBuffer(sampleBuffer: sampleBuffer) else {
return
}
delegate?.processCapturedImage(image: outputImage)
}
}
Process function:
extension ViewController: CaptureManagerDelegate {
func processCapturedImage(image: UIImage) {
self.imageView.image = ...
//process image
}
}
And this is how its called in the ViewController:
CaptureManager.shared.startSession()