I am working on the communication between angular socketio (frontend) to Flask-SocketIO(Backend). Connection is being established successfully and I am able to exchange messages.In the application At the frontend, there is a microphone, that a user turns on and starts recording audio. this audio is sent in chunks of 5s over the socket to the backend. However at the backend I am not getting all the audio bytes. They are being lost in the communication. and I am not getting all the audio data.
Here are the versions at the front end: socketio 2.2
Here is the front end code:
recordingOptions = {
volume: 1,
channels: 1,
bufferSize: 9048,
sampleRate: 16000
}
startRecord()
{
this.socket = io(this.url);
this.socket.emit("connection", "Connection Successful");
this.socket.on("connection", (socket) => {
console.log(socket);
});
this.socket.io.on("reconnect_attempt", () => {
console.log("recon");
});
this.socket;
navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => {
this.recorder = new Recorder(stream);
this.recorder.start().then(()=>{
this.isRecording = true;
setTimeout(this.exportStreamData.bind(this), this.exportPeriod);
});
});
}
async exportStreamData() {
if(this.isRecording){
setTimeout(this.exportStreamData.bind(this), this.exportPeriod);
}
const audioContext = new AudioContext(this.recordingOptions);
const fileReader = new FileReader();
// Set up file reader on loaded end event
fileReader.onloadend = () => {
const arrayBuffer = fileReader.result as ArrayBuffer;
// Convert array buffer into audio buffer
audioContext.decodeAudioData(arrayBuffer, (audioBuffer) => {
resampler(audioBuffer, this.recordingOptions.sampleRate, (resamped) =>{
var newBuffer = resamped.getAudioBuffer();
if(this.buffer){
this.buffer = this.appendBuffer(this.buffer, newBuffer);
} else {
this.buffer = newBuffer;
}
if(this.socket.connected){
var blb = new Blob([newBuffer.getChannelData(0)], { 'type' : "audio/wav" });
-
this.socket.emit("audiodata", blb);
}
});
})
}
this.recorder.export()
.then((rblob: Blob) => {
//Load blob
fileReader.readAsArrayBuffer(rblob)
});
}
appendBuffer(buffer1, buffer2) {
///Using AudioBuffer
var numberOfChannels = Math.min(buffer1.numberOfChannels, buffer2.numberOfChannels);
var tmp = new AudioContext().createBuffer(numberOfChannels, (buffer1.length + buffer2.length), buffer1.sampleRate);
for (var i = 0; i < numberOfChannels; i++) {
var channel = tmp.getChannelData(i);
channel.set(buffer1.getChannelData(i), 0);
channel.set(buffer2.getChannelData(i), buffer1.length);
}
return tmp;
}
downsampleBuffer(b: AudioBuffer, sampleRate, outSampleRate) {
var buffer;
b.copyFromChannel(buffer, 0);
if (outSampleRate == sampleRate) {
return buffer;
}
if (outSampleRate > sampleRate) {
throw "downsampling rate show be smaller than original sample rate";
}
var sampleRateRatio = sampleRate / outSampleRate;
var newLength = Math.round(buffer.length / sampleRateRatio);
var result = new Int16Array(newLength);
var offsetResult = 0;
var offsetBuffer = 0;
while (offsetResult < result.length) {
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
var accum = 0, count = 0;
for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
accum += buffer[i];
count++;
}
result[offsetResult] = Math.min(1, accum / count)*0x7FFF;
offsetResult++;
offsetBuffer = nextOffsetBuffer;
}
var rb = new AudioBuffer({length: result.buffer.byteLength, sampleRate: outSampleRate, numberOfChannels: 1});
return result.buffer;
}
**Here are the socket versions at the back end: ** python-engineio 3.13.2 python-socketio 4.6.0
Here is the backend code:
sio = socketio.AsyncServer(async_mode='aiohttp',async_handlers=True, logger=True, engineio_logger=True)
sockapp = web.Application()
sio.attach(sockapp)
@sio.on('connection')
async def handle_message(sid, data):
try:
print("Got message {} from session {}".format(data, sid))
await sio.emit('connection', "Connected Successfully")
except Exception as e:
print(e)
@sio.on('audiodata')
async def handle_data(sid,data):
print("inside audiodata")
print((data))
print("the length of data:")
print(len(data))
if __name__ == "__main__":
print("starting socket")
web.run_app(sockapp, port=8080)