I have an avassetwriter to record a video with an applied filter to then play back via avqueueplayer.
My issue is the audio output appends to the audio input, but no sound plays in the play back. Have not come across any existing solutions, and would appreciate any guidance available..
Secondarily, my .AVPlayerItemDidPlayToEndTime notification observer, which I have to loop the playback, does not fire as well..
AVCaptureSession Setup
func setupSession() {
let session = AVCaptureSession()
session.sessionPreset = .medium
guard
let camera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .front),
let mic = AVCaptureDevice.default(.builtInMicrophone, for: .audio, position: .unspecified),
let videoInput = try? AVCaptureDeviceInput(device: camera),
let audioInput = try? AVCaptureDeviceInput(device: mic),
session.canAddInput(videoInput), session.canAddInput(audioInput) else { return }
let videoOutput = AVCaptureVideoDataOutput()
let audioOutput = AVCaptureAudioDataOutput()
guard session.canAddOutput(videoOutput), session.canAddOutput(audioOutput) else { return }
let queue = DispatchQueue(label: "recordingQueue", qos: .userInteractive)
videoOutput.setSampleBufferDelegate(self, queue: queue)
audioOutput.setSampleBufferDelegate(self, queue: queue)
session.beginConfiguration()
session.addInput(videoInput)
session.addInput(audioInput)
session.addOutput(videoOutput)
session.addOutput(audioOutput)
session.commitConfiguration()
if let connection = videoOutput.connection(with: AVMediaType.video) {
if connection.isVideoStabilizationSupported { connection.preferredVideoStabilizationMode = .auto }
connection.isVideoMirrored = true
connection.videoOrientation = .portrait
}
_videoOutput = videoOutput
_audioOutput = audioOutput
_captureSession = session
DispatchQueue.global(qos: .default).async { session.startRunning() }
}
AVAssetWriter Setup + didOutput Delegate
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
let timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer).seconds
if output == _videoOutput {
if connection.isVideoOrientationSupported { connection.videoOrientation = .portrait }
guard let cvImageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
let ciImage = CIImage(cvImageBuffer: cvImageBuffer)
guard let filteredCIImage = applyFilters(inputImage: ciImage) else { return }
self.ciImage = filteredCIImage
guard let cvPixelBuffer = getCVPixelBuffer(from: filteredCIImage) else { return }
self.cvPixelBuffer = cvPixelBuffer
self.ciContext.render(filteredCIImage, to: cvPixelBuffer, bounds: filteredCIImage.extent, colorSpace: CGColorSpaceCreateDeviceRGB())
metalView.draw()
}
switch _captureState {
case .start:
guard let outputUrl = tempURL else { return }
let writer = try! AVAssetWriter(outputURL: outputUrl, fileType: .mp4)
let videoSettings = _videoOutput!.recommendedVideoSettingsForAssetWriter(writingTo: .mp4)
let videoInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings)
videoInput.mediaTimeScale = CMTimeScale(bitPattern: 600)
videoInput.expectsMediaDataInRealTime = true
let pixelBufferAttributes = [
kCVPixelBufferCGImageCompatibilityKey: NSNumber(value: true),
kCVPixelBufferCGBitmapContextCompatibilityKey: NSNumber(value: true),
kCVPixelBufferPixelFormatTypeKey: NSNumber(value: Int32(kCVPixelFormatType_32ARGB))
] as [String:Any]
let adapter = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: videoInput, sourcePixelBufferAttributes: pixelBufferAttributes)
if writer.canAdd(videoInput) { writer.add(videoInput) }
let audioSettings = _audioOutput!.recommendedAudioSettingsForAssetWriter(writingTo: .mp4) as? [String:Any]
let audioInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings)
audioInput.expectsMediaDataInRealTime = true
if writer.canAdd(audioInput) { writer.add(audioInput) }
_filename = outputUrl.absoluteString
_assetWriter = writer
_assetWriterVideoInput = videoInput
_assetWriterAudioInput = audioInput
_adapter = adapter
_captureState = .capturing
_time = timestamp
writer.startWriting()
writer.startSession(atSourceTime: .zero)
case .capturing:
if output == _videoOutput {
if _assetWriterVideoInput?.isReadyForMoreMediaData == true {
let time = CMTime(seconds: timestamp - _time, preferredTimescale: CMTimeScale(600))
_adapter?.append(self.cvPixelBuffer, withPresentationTime: time)
}
} else if output == _audioOutput {
if _assetWriterAudioInput?.isReadyForMoreMediaData == true {
_assetWriterAudioInput?.append(sampleBuffer)
}
}
break
case .end:
guard _assetWriterVideoInput?.isReadyForMoreMediaData == true, _assetWriter!.status != .failed else { break }
_assetWriterVideoInput?.markAsFinished()
_assetWriterAudioInput?.markAsFinished()
_assetWriter?.finishWriting { [weak self] in
guard let output = self?._assetWriter?.outputURL else { return }
self?._captureState = .idle
self?._assetWriter = nil
self?._assetWriterVideoInput = nil
self?._assetWriterAudioInput = nil
self?.previewRecordedVideo(with: output)
}
default:
break
}
}