Not receiving complete audio in flask socketio from Angular socketio

Viewed 16

I am working on the communication between angular socketio (frontend) to Flask-SocketIO(Backend). Connection is being established successfully and I am able to exchange messages.In the application At the frontend, there is a microphone, that a user turns on and starts recording audio. this audio is sent in chunks of 5s over the socket to the backend. However at the backend I am not getting all the audio bytes. They are being lost in the communication. and I am not getting all the audio data.

Here are the versions at the front end: socketio 2.2

Here is the front end code:

  recordingOptions = {
    volume: 1,
    channels: 1,
    bufferSize: 9048,
    sampleRate: 16000
  }
startRecord()
  {
    this.socket = io(this.url);
    this.socket.emit("connection", "Connection Successful");
    this.socket.on("connection", (socket) => {
      console.log(socket);  
    });
    this.socket.io.on("reconnect_attempt", () => {      
      console.log("recon");
    });
    this.socket;
    navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => {
      this.recorder = new Recorder(stream);
      this.recorder.start().then(()=>{
        this.isRecording = true;
        setTimeout(this.exportStreamData.bind(this), this.exportPeriod);
      });   
    });
  }
 async exportStreamData() {
    if(this.isRecording){
     setTimeout(this.exportStreamData.bind(this), this.exportPeriod);
    }
    const audioContext = new AudioContext(this.recordingOptions);
    const fileReader = new FileReader();

    // Set up file reader on loaded end event
    fileReader.onloadend = () => {
      const arrayBuffer = fileReader.result as ArrayBuffer;
      // Convert array buffer into audio buffer
      audioContext.decodeAudioData(arrayBuffer, (audioBuffer) => {
        resampler(audioBuffer, this.recordingOptions.sampleRate, (resamped) =>{
          var newBuffer = resamped.getAudioBuffer();
          if(this.buffer){
            this.buffer = this.appendBuffer(this.buffer, newBuffer);
          } else {
            this.buffer = newBuffer;
          }
          if(this.socket.connected){

            var blb = new Blob([newBuffer.getChannelData(0)], { 'type' : "audio/wav" });
            - 
            this.socket.emit("audiodata", blb);
             
          }
        });
      })
    }
    this.recorder.export()
      .then((rblob: Blob) => {
      //Load blob
      fileReader.readAsArrayBuffer(rblob)
  });
      
    
  }
  appendBuffer(buffer1, buffer2) {
    ///Using AudioBuffer
    var numberOfChannels = Math.min(buffer1.numberOfChannels, buffer2.numberOfChannels);
    var tmp = new AudioContext().createBuffer(numberOfChannels, (buffer1.length + buffer2.length), buffer1.sampleRate);
    for (var i = 0; i < numberOfChannels; i++) {
        var channel = tmp.getChannelData(i);
        channel.set(buffer1.getChannelData(i), 0);
        channel.set(buffer2.getChannelData(i), buffer1.length);
    }
    return tmp;
  }

   downsampleBuffer(b: AudioBuffer, sampleRate, outSampleRate) {
    var buffer;
    b.copyFromChannel(buffer, 0);
    if (outSampleRate == sampleRate) {
        return buffer;
    }
    if (outSampleRate > sampleRate) {
        throw "downsampling rate show be smaller than original sample rate";
    }
    var sampleRateRatio = sampleRate / outSampleRate;
    var newLength = Math.round(buffer.length / sampleRateRatio);
    var result = new Int16Array(newLength);
    var offsetResult = 0;
    var offsetBuffer = 0;
    while (offsetResult < result.length) {
        var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
        var accum = 0, count = 0;
        for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
            accum += buffer[i];
            count++;
        }

        result[offsetResult] = Math.min(1, accum / count)*0x7FFF;
        offsetResult++;
        offsetBuffer = nextOffsetBuffer;
    }
    var rb = new AudioBuffer({length: result.buffer.byteLength, sampleRate: outSampleRate, numberOfChannels: 1});

    return result.buffer;
}

**Here are the socket versions at the back end: ** python-engineio 3.13.2 python-socketio 4.6.0

Here is the backend code:

sio = socketio.AsyncServer(async_mode='aiohttp',async_handlers=True, logger=True, engineio_logger=True)
sockapp = web.Application()
sio.attach(sockapp)

@sio.on('connection')
async def handle_message(sid, data):
    try:
        print("Got message {} from session {}".format(data, sid))
        await sio.emit('connection', "Connected Successfully")
    except Exception as e:
        print(e)
        
@sio.on('audiodata')
async def handle_data(sid,data):
    print("inside audiodata")
    print((data))
    print("the length of data:")
    print(len(data))
    
if __name__ == "__main__":
    print("starting socket")
    web.run_app(sockapp, port=8080)
0 Answers
Related