I have a MacOS Swift app that processes audio data recorded from a microphone. The microphone has stereo capabilities, but I'm only able to record mono data.
In the code below, if I let alwaysMono = true, the func setup() reports that the active format is stereo, but overrides it to mono. Everything works with a mono stream of input.
If I let alwaysMono = false, setup() sets nChannels to 2. But captureOutput doesn't get any data. The AudioBuffer returned from UnsafeMutableAudioBufferListPointer always has a nil mData. If I didn't check for nil mData, the program would crash.
How can I get the full stereo input?
EDIT: In captureOutput, CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer is returning error code -12737, which corresponds to kCMSampleBufferError_ArrayTooSmall. I've examined the sampleBuffer arg passed in to captureOutput, and I can't see anything obviously wrong with it. But I don't know what to look for.
ANOTHER EDIT: I tested my code with my code with the built-in mono mike, and to my surprise it thinks it's also stereo, which indicates there's clearly something wrong with how I'm obtaining and using AVCaptureDevice.activeFormat. I don't know where to go from here.
class Recorder: NSObject, AVCaptureAudioDataOutputSampleBufferDelegate {
let alwaysMono = false
var nChannels:UInt32 = 1
let session : AVCaptureSession!
static let realTimeQueue = DispatchQueue(label: "com.myapp.realtime",
qos: DispatchQoS( qosClass:DispatchQoS.QoSClass.userInitiated, relativePriority: 0 ))
override init() {
session = AVCaptureSession()
super.init()
}
static var recorder:Recorder?
static func record() ->Bool {
if recorder == nil {
recorder = Recorder()
if !recorder!.setup(callback:record) {
recorder = nil
return false
}
}
realTimeQueue.async {
if !recorder!.session.isRunning {
recorder!.session.startRunning()
}
}
return true
}
static func pause() {
recorder!.session.stopRunning()
}
func setup( callback:@escaping (()->Bool)) -> Bool {
let device = AVCaptureDevice.default( for: AVMediaType.audio )
if device == nil { return false }
if let format = Recorder.getActiveFormat() {
nChannels = format.mChannelLayoutTag == kAudioChannelLayoutTag_Stereo ? 2 : 1
print("active format is \((nChannels==2) ? "Stereo" : "Mono")")
if alwaysMono {
print( "Overriding to mono" )
nChannels = 1
}
}
if #available(OSX 10.14, *) {
let status = AVCaptureDevice.authorizationStatus( for:AVMediaType.audio )
if status == .notDetermined {
AVCaptureDevice.requestAccess(for: AVMediaType.audio ){ granted in
_ = callback()
}
return false
} else if status != .authorized {
return false
}
}
var input : AVCaptureDeviceInput
do {
try device!.lockForConfiguration()
try input = AVCaptureDeviceInput( device: device! )
device!.unlockForConfiguration()
} catch {
device!.unlockForConfiguration()
return false
}
let output = AVCaptureAudioDataOutput()
output.setSampleBufferDelegate(self, queue: Intonia.realTimeQueue)
let settings = [
AVFormatIDKey: kAudioFormatLinearPCM,
AVNumberOfChannelsKey : nChannels,
AVSampleRateKey : 44100,
AVLinearPCMBitDepthKey : 16,
AVLinearPCMIsFloatKey : false
] as [String : Any]
output.audioSettings = settings
session.beginConfiguration()
if !session.canAddInput( input ) {
return false
}
session.addInput( input )
if !session.canAddOutput( output ) {
return false
}
session.addOutput( output )
session.commitConfiguration()
return true
}
func getActiveFormat() -> AudioFormatListItem? {
if #available(OSX 10.15, *) {
let device = AVCaptureDevice.default( for: AVMediaType.audio )
if device == nil { return nil }
let list = device!.activeFormat.formatDescription.audioFormatList
if list.count < 1 { return nil }
return list[0]
}
return nil
}
func captureOutput(_ captureOutput: AVCaptureOutput,
didOutput sampleBuffer: CMSampleBuffer,
from connection: AVCaptureConnection){
var buffer: CMBlockBuffer? = nil
var audioBufferList = AudioBufferList(
mNumberBuffers: 1,
mBuffers: AudioBuffer(mNumberChannels: nChannels, mDataByteSize: 0, mData: nil)
)
CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
sampleBuffer,
bufferListSizeNeededOut: nil,
bufferListOut: &audioBufferList,
bufferListSize: MemoryLayout<AudioBufferList>.size,
blockBufferAllocator: nil,
blockBufferMemoryAllocator: nil,
flags: UInt32(kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment),
blockBufferOut: &buffer
)
let abl = UnsafeMutableAudioBufferListPointer(&audioBufferList)
for buff in abl {
if buff.mData != nil {
let count = Int(buff.mDataByteSize)/MemoryLayout<Int16>.size
let samples = UnsafeMutablePointer<Int16>(OpaquePointer(buff.mData))
process(samples:samples!, count:count)
} else {
print("No data!")
}
}
}
func process( samples: UnsafeMutablePointer<Int16>, count: Int ) {
let firstValue = samples[0]
print( "\(count) values received, first is \(firstValue)" )
}
}