I am trying to implement google speech API with my Django + react project to let users conversate with the website. The process I am following is user speak with react front-end using "mediaDevices" then the audio stream is converted into a buffer and sent to Django backend over WebSocket. Django is supposed to transcribe the buffer to text and send back the response to the client (react) but google API is not transcribing the request nither giving any error. All I receive from google transcribe is
<google.api_core.grpc_helpers._StreamingResponseIterator object at 0x000001CD1D2C5E88>
according to google speech docs and example, the response is supposed to be Iterable but I am unable to iterate over the object. After trying for days I am unable to understand What I am doing wrong.
here is the code sample that I am using.
React
import React, { Component } from 'react'
import * as audioUtils from "./audioUtils";
import mic from "microphone-stream";
import "./style.scss";
class VttMic extends Component {
state = {}
transcription = []
SAMPLE_RATE = 16000;
SAMPLE_SIZE = 16
startTranscribe = () => {
window.navigator.mediaDevices.getUserMedia({
video: false,
audio: {
echoCancellation: true,
channelCount: 1,
sampleRate: {
ideal: this.SAMPLE_RATE
},
sampleSize: this.SAMPLE_SIZE
}
})
.then(this.streamAudioToWebSocket)
.catch( (error) => {
this.handleError("There was an error streaming your audio to Amazon Transcribe. Please try again.")
});
}
streamAudioToWebSocket = async (userMediaStream) => {
try{
this.micStream = new mic();
this.micStream.on("format", (data) => {
this.inputSampleRate = data.sampleRate;
});
this.micStream.setStream(userMediaStream);
this.socket = new WebSocket('ws://localhost:8000/ws/stt/');
this.socket.binaryType = "arraybuffer";
this.socket.onopen = () => {
this.micStream.on('data', (rawAudioChunk) => {
let binary = this.convertAudioToBinaryMessage(rawAudioChunk);
if (this.socket.readyState === this.socket.OPEN)
this.socket.send(binary);
}
)};
this.wireSocketEvents();
}catch(err){
this.handleError("An error occured at streamAudioToWebSocket", err);
}
}
wireSocketEvents() {
// handle inbound messages from Transcribe
let transcribeException = false;
let socketError = false;
this.socket.onmessage = (message) => {
//handle socket transcribe response here
};
this.socket.onerror = () => {
socketError = true;
this.handleError("WebSocket connection error. Try again.");
};
this.socket.onclose = (closeEvent) => {
this.micStream.stop();
if (!socketError && !transcribeException) {
if (closeEvent.code != 1000) {
this.handleError('Streaming Exception: ' + closeEvent.reason);
}
}
};
}
convertAudioToBinaryMessage(audioChunk) {
//converting audiochunks to pcm buffer
let raw = mic.toRaw(audioChunk);
if (raw == null)
return;
let downsampledBuffer = audioUtils.downsampleBuffer(raw, this.inputSampleRate, this.sampleRate);
let pcmEncodedBuffer = audioUtils.pcmEncode(downsampledBuffer);
return pcmEncodedBuffer;
}
closeSocket = () => {
if (this.socket && this.socket.readyState === this.socket.OPEN) {
this.micStream.stop();
this.socket.close();
}
}
handleError = (err) => {
this.closeTranscription();
this.props.onError && this.props.onError(err);
}
handleMicClick = () => {
if(this.state.started){
this.closeTranscription();
}else{
this.startTranscription();
}
}
startTranscription = () => {
this.setState({started: true});
this.startTranscribe();
}
closeTranscription = () => {
this.setState({started: false});
this.transcription = [];
this.closeSocket();
this.props.onClose && this.props.onClose()
}
startCloseWaitTimer = () => {
if(this.t)
clearTimeout(this.t);
this.t = setTimeout(() => {
this.closeTranscription();
}, 10000);
}
render() {
let { started } = this.state;
return (
<div className="stt-mic-container" onClick={this.handleMicClick}>
<i
className={`sttMic fa fa-microphone text-primary`}
/>
<div class={`circle-ani ${started ? "active" : ""}`} />
</div>
)
}
}
export default VttMic;
React audioUtils.js looks like
export function pcmEncode(input) {
var offset = 0;
var buffer = new ArrayBuffer(input.length * 2);
var view = new DataView(buffer);
for (var i = 0; i < input.length; i++, offset += 2) {
var s = Math.max(-1, Math.min(1, input[i]));
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
}
return buffer;
}
export function downsampleBuffer(buffer, inputSampleRate = 44100, outputSampleRate = 16000) {
if (outputSampleRate === inputSampleRate) {
return buffer;
}
var sampleRateRatio = inputSampleRate / outputSampleRate;
var newLength = Math.round(buffer.length / sampleRateRatio);
var result = new Float32Array(newLength);
var offsetResult = 0;
var offsetBuffer = 0;
while (offsetResult < result.length) {
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
var accum = 0,
count = 0;
for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++ ) {
accum += buffer[i];
count++;
}
result[offsetResult] = accum / count;
offsetResult++;
offsetBuffer = nextOffsetBuffer;
}
return result;
}
and for django channels code goes something like this.
from channels.generic.websocket import WebsocketConsumer
from google.cloud import speech
from google.oauth2 import service_account
credentials = service_account.Credentials.from_service_account_file("319006-603d31ac86ca.json")
client = speech.SpeechClient(credentials=credentials)
client = speech.SpeechClient(credentials=credentials)
streaming_config = None
class SpeechToTextConsumer(WebsocketConsumer):
def connect(self):
self.accept()
def disconnect(self, close_code):
print("disconnect")
def process(self, responses):
print(responses) #here i am receiving response as <google.api_core.grpc_helpers._StreamingResponseIterator object at 0x000002161C7BBA88>
#noting happens after this
for response in responses:
print("response", response)
if not response.results:
print("continue")
continue
result = response.results[0]
print("result", result)
self.send(text_data=json.dumps(result))
def receive(self, text_data=None, bytes_data=None):
rate = 16000
config = speech.RecognitionConfig(
encoding=speech.RecognitionConfig.AudioEncoding.LINEAR16,
sample_rate_hertz=rate,
language_code=language_code,
)
streaming_config = speech.StreamingRecognitionConfig(
config=config, interim_results=True
)
speech.StreamingRecognizeRequest(streaming_config=streaming_config)
stream = [bytes_data]
print(stream)
requests = (
speech.StreamingRecognizeRequest(audio_content=chunk) for chunk in stream
)
responses = client.streaming_recognize(streaming_config, requests)
self.process(responses)
I went through the official documentation and but found nothing useful. Also looked on the internet but I am clueless about what am I doing wrong.
any help is appreciable.
Thanks