A problem with sound producing: How to make sound with Fourier coefficients

user_17888300 0
Viewed 143

I'm trying to create a sound using Fourier coefficients.

First of all please let me show how I got Fourier coefficients.

(1) I took a snapshot of a waveform from a microphone sound.

  • Getting microphone: getUserMedia()
  • Getting microphone sound: MediaStreamAudioSourceNode
  • Getting waveform data: AnalyserNode.getByteTimeDomainData()

The data looks like the below: (I stringified Uint8Array, which is the return value of getByteTimeDomainData(), and added length property in order to change this object to Array later)

const raw = '{"length": 512,"0":126,"1":121,"2":121,"3":124,"4":129,"5":135,"6":140,"7":147,"8":153,"9":156,"10":152,"11":141,"12":125,"13":112,"14":106,"15":108,"16":113,"17":120,"18":127,"19":132,"20":138,"21":142,"22":141,"23":136,"24":126,"25":115,"26":106,"27":103,"28":105,"29":111,"30":117,"31":121,"32":123,"33":124,"34":124,"35":120,"36":112,"37":103,"38":97,"39":95,"40":96,"41":98,"42":101,"43":106,"44":112,"45":117,"46":117,"47":113,"48":105,"49":98,"50":93,"51":91,"52":91,"53":92,"54":93,"55":95,"56":97,"57":101,"58":105,"59":108,"60":106,"61":101,"62":96,"63":95,"64":97,"65":100,"66":100,"67":97,"68":94,"69":94,"70":99,"71":104,"72":106,"73":105,"74":104,"75":105,"76":108,"77":111,"78":112,"79":110,"80":108,"81":105,"82":105,"83":107,"84":110,"85":113,"86":114,"87":115,"88":116,"89":120,"90":123,"91":125,"92":124,"93":121,"94":120,"95":121,"96":123,"97":124,"98":124,"99":126,"100":128,"101":131,"102":133,"103":134,"104":134,"105":134,"106":134,"107":134,"108":134,"109":133,"110":132,"111":131,"112":131,"113":134,"114":137,"115":139,"116":141,"117":142,"118":143,"119":142,"120":142,"121":139,"122":136,"123":131,"124":128,"125":128,"126":131,"127":134,"128":137,"129":139,"130":140,"131":141,"132":142,"133":141,"134":137,"135":132,"136":126,"137":122,"138":123,"139":127,"140":132,"141":135,"142":135,"143":134,"144":134,"145":135,"146":134,"147":130,"148":125,"149":121,"150":120,"151":121,"152":124,"153":129,"154":132,"155":134,"156":134,"157":133,"158":131,"159":129,"160":128,"161":127,"162":125,"163":124,"164":123,"165":124,"166":125,"167":128,"168":130,"169":131,"170":132,"171":132,"172":131,"173":129,"174":129,"175":129,"176":130,"177":129,"178":129,"179":128,"180":129,"181":132,"182":134,"183":135,"184":134,"185":133,"186":131,"187":131,"188":131,"189":132,"190":134,"191":134,"192":134,"193":134,"194":137,"195":140,"196":142,"197":142,"198":141,"199":138,"200":136,"201":135,"202":137,"203":138,"204":137,"205":135,"206":134,"207":137,"208":142,"209":147,"210":148,"211":147,"212":146,"213":144,"214":144,"215":144,"216":144,"217":142,"218":138,"219":136,"220":137,"221":141,"222":145,"223":149,"224":150,"225":150,"226":150,"227":150,"228":150,"229":148,"230":145,"231":142,"232":142,"233":144,"234":146,"235":146,"236":146,"237":147,"238":150,"239":153,"240":153,"241":149,"242":145,"243":143,"244":141,"245":141,"246":142,"247":143,"248":143,"249":142,"250":144,"251":148,"252":153,"253":152,"254":142,"255":130,"256":123,"257":123,"258":127,"259":130,"260":132,"261":134,"262":139,"263":147,"264":154,"265":155,"266":148,"267":134,"268":119,"269":108,"270":106,"271":110,"272":115,"273":119,"274":124,"275":129,"276":136,"277":141,"278":141,"279":135,"280":125,"281":115,"282":108,"283":105,"284":105,"285":108,"286":111,"287":115,"288":119,"289":122,"290":121,"291":116,"292":110,"293":106,"294":104,"295":101,"296":98,"297":96,"298":98,"299":103,"300":110,"301":115,"302":116,"303":112,"304":104,"305":98,"306":95,"307":95,"308":94,"309":91,"310":88,"311":88,"312":94,"313":101,"314":107,"315":110,"316":107,"317":103,"318":100,"319":99,"320":99,"321":98,"322":95,"323":89,"324":87,"325":89,"326":96,"327":103,"328":107,"329":109,"330":110,"331":111,"332":113,"333":113,"334":110,"335":105,"336":102,"337":102,"338":104,"339":105,"340":107,"341":110,"342":115,"343":120,"344":123,"345":123,"346":122,"347":120,"348":120,"349":121,"350":123,"351":124,"352":123,"353":122,"354":122,"355":126,"356":133,"357":137,"358":136,"359":132,"360":128,"361":129,"362":134,"363":139,"364":139,"365":135,"366":131,"367":131,"368":135,"369":141,"370":144,"371":143,"372":140,"373":138,"374":138,"375":140,"376":142,"377":140,"378":136,"379":131,"380":130,"381":133,"382":138,"383":141,"384":141,"385":140,"386":140,"387":140,"388":139,"389":136,"390":132,"391":129,"392":128,"393":128,"394":129,"395":131,"396":133,"397":135,"398":136,"399":136,"400":135,"401":132,"402":129,"403":125,"404":123,"405":123,"406":125,"407":126,"408":126,"409":126,"410":128,"411":131,"412":133,"413":133,"414":130,"415":127,"416":125,"417":125,"418":125,"419":125,"420":125,"421":125,"422":125,"423":126,"424":129,"425":131,"426":132,"427":131,"428":128,"429":126,"430":126,"431":128,"432":129,"433":130,"434":130,"435":130,"436":132,"437":134,"438":136,"439":135,"440":133,"441":131,"442":129,"443":128,"444":129,"445":130,"446":132,"447":134,"448":136,"449":138,"450":140,"451":142,"452":143,"453":142,"454":140,"455":137,"456":135,"457":134,"458":134,"459":134,"460":134,"461":135,"462":137,"463":139,"464":143,"465":147,"466":148,"467":147,"468":146,"469":145,"470":144,"471":141,"472":139,"473":137,"474":136,"475":137,"476":139,"477":142,"478":145,"479":149,"480":150,"481":151,"482":152,"483":152,"484":151,"485":146,"486":141,"487":138,"488":140,"489":145,"490":147,"491":146,"492":145,"493":147,"494":152,"495":157,"496":156,"497":151,"498":145,"499":140,"500":137,"501":139,"502":143,"503":147,"504":147,"505":144,"506":143,"507":146,"508":152,"509":152,"510":143,"511":129}';

※ If we draw this data to canvas, we can see the below: (This is a sound of vowel 'i' (it sounds like 'ee'))

waveform of a vowel sound of 'i'

It seems 2 period of the wave is captured. Since length is 512, we can guess that data of one period is located at index 0 ~ 255.

(2) I processed the data.

const parsed = JSON.parse(raw);
const arrayfied = Array.from(parsed);
const sliced = arrayfied.slice(0, 256);
const refined = [];

// According to the Web Audio API specification,
// "The values stored in the unsigned byte array are computed in the following way.
// Let x[k] be the time-domain data. Then the byte value, b[k], is
// b[k]=⌊128(1+x[k])⌋." So, I manipulate the array like the following:

for (let i = 0; i < sliced.length; i++) {
  refined[i] = (sliced[i] / 128) - 1;
}

(3) I calculated Fourier coefficients.

  • Fourier coefficient formula for a0
  • Fourier coefficient formula for an
  • Fourier coefficient formual for bn
// This function calculates Riemann sum (area approximation using rectangles)
// fn: function to be calculated
// initial: calculation start point
// final: calculation end point
// division: number of rectangles to use
// nth: used for an, bn (please see below)
function numerical_integration(fn, initial, final, division, nth = null) {
  let accumulation = 0;
  const STEP = (final - initial) / division;

  for (let i = initial; i <= final; i++) {
    // calculate an area of a rectangle and add
    accumulation += fn(i, initial, final, nth) * STEP;
  }

  return accumulation;
}

// This is f(t)
function f0(t) {
  const result = refined[t];

  return result;
}

// This is f(t) * cos(nwt)
// ※ w = 2 * Math.PI / period
function fc(t, i, f, n) {
  const result = f0(t) * Math.cos(n * 2 * Math.PI * t / (f - i));

  return result;
}

// This is f(t) * sin(nwt)
function fs(t, i, f, n) {
  const result = f0(t) * Math.sin(n * 2 * Math.PI * t / (f - i));

  return result;
}

// This function returns a0 value
// period is 256 (0 ~ 255) and the last element of array refined is at index 255,
// so I subtract one.
function getA0(period) {
  const result = numerical_integration(f0, 0, period - 1, 100) / period;

  return result;
}

// This function returns an values
function getAn(period) {
  const result = [];

  for (let i = 1; i <= 49; i++) {
    result.push(numerical_integration(fc, 0, period - 1, 100, i) * 2 / period);
  }

  return result;
}

// This function returns bn values
function getBn(period) {
  const result = [];

  for (let i = 1; i <= 49; i++) {
    result.push(numerical_integration(fs, 0, period - 1, 100, i) * 2 / period);
  }

  return result;
}

So far so good! Now we can check whether our Fourier coefficients are well calculated by making a wave function using the coefficients and drawing it to canvas!

const a0 = getA0(refined.length);
const an = getAn(refined.length);
const bn = getBn(refined.length);

// returns y coordinate
function getY(t) {
  let anSum = 0;
  let bnSum = 0;

  for (let i = 0; i <= 48; i++) {
    anSum += an[i] * Math.cos((i + 1) * 2 * Math.PI * t / refined.length);
    bnSum += bn[i] * Math.sin((i + 1) * 2 * Math.PI * t / refined.length);
  }

  const result = a0 + anSum + bnSum;

  return result;
}

// draw
canvasContext.lineTo(x, getY(t));

waveform approximated by using Fourier coefficients

Wow! Nicely done! It is nearly as same as the original wave!

Then you might ask "So, what is your question?" Therefore, I'm going to ask my question: How to reproduce sound by using Fourier coefficients? (I do not know much about Web Audio API and digital sound)

What I've thought is three things:

  • Maybe AudioBuffer?
  • AudioWorklet?
  • PeriodicWave and OscillatorNode?

I tried AudioWorklet but it sounded like saturated(?) A4(perhaps) with crackle 'tick tick' sound. The AudioWorklet code is as follows:

class IWaveProducer extends AudioWorkletProcessor {
  constructor() {
    super();

    this.t = 0;
  }

  process(inputs, outputs, parameters) {
    const output = outputs[0];

    output.forEach(channel => {
      for (let i = 0; i < channel.length; i++) {
        channel[i] = getY(this.t);
      }
    });

    this.t++;

    return true;
  }
}

registerProcessor('i-wave-producer', IWaveProducer);

And this is the graph:

waveform by AudioWorklet

So this time I tried PeriodicWave and OscillatorNode but it also failed. The code is as follows:

const real = new Float32Array(50);
const imag = new Float32Array(50);

real[0] = a0;
imag[0] = 0;

for (let i = 1; i <= 48; i++) {
  real[i + 1] = an[i];
  imag[i + 1] = bn[i];
}

const wave = new PeriodicWave(audioCtx, { real, imag, disableNormalization: false });
const osc = new OscillatorNode(audioCtx, { periodicWave: wave });

osc.connect(analyser)
   .connect(audioCtx.destination);
osc.start();

And this is the graph:

waveform by Periodicwave and OscillatorNode

It sounded like a A4 sawtooth wave(perhaps). Also, interestingly, it seems all the data is inserted correctly, since the form of the wave is quite similar to the above picture (please see 'waveform approximated' picture). (Its pattern: one high mountain and one small mountain)

...But this is totally not I want! What I want is to reproduce a vowel 'i' sound! How can I achieve my goal? Please let me know if you know something. It would be greatly appreciated. I'm curious to death. Please help me ㅠㅠ. Thank you very much for reading this long question.

Or is it impossible to make 'voice' with Web Audio API? But I've seen a library making voice using JavaScript before. For example:

Hi again! I think just found the answer! The answer is... 「AudioBuffer」. I'm literally crying...with delight... Anyway, here is the code!!!

// Since the length of wave is 256
// and I guess (maybe wrong) that it means
// this wave lasts for 256 / 44100 seconds (= 0.0058).
// Thus, in order to make it longer,
// multiply 1000. So this sound will exist for 5.8 seconds.
// (Since sampling rate is 44100 per sec,
// the formula results in the length of this buffer--256000.)
const audioBuffer = new AudioBuffer({ numberOfChannels: 1, length: 1000 * audioCtx.sampleRate * 256 / 44100, sampleRate: audioCtx.sampleRate });

const buffering = audioBuffer.getChannelData(0);
let count = 0;

for (let i = 0; i < audioBuffer.length; i++) {
  buffering[i] = refined[count];

  if (count === 255) {
    count = 0;
  } else {
    count++;
  }
}

const source = new AudioBufferSourceNode(audioCtx, { buffer: audioBuffer });
  
source.connect(analyser)
      .connect(audioCtx.destination);

source.start();

The resulting sound is a bit hilarious! But I think it surely sounds like 'i (ee)'. It also sounds like 'fa' note. Why is that? Let's think about it together. First I think we need to calculate the wave's frequency. Since the wave's one period is 0.0058 seconds, thus the frequency is 1 / 0.0058, which is 172.4138 Hz.

Next, A4 is 440 Hz. Therefore A3 is 220Hz. Four notes below is F3 (A3, G#4, G4, F#3, F3). Then the frequency of F3 is 220 * 2^(-4/12) = 174.6141 Hz.

172 is nearly same with 174. !!!!! Which so absolutely makes sense! The secret is now solved. That's why that sound sounds like fa.

Thank you for reading my tough, but at the same time beautiful fighting story against Web Audio API. Bye!

Hi again again! I've just found that PeriodicWave and OscillatorNode can also be the answer!

const osc = new OscillatorNode(audioCtx, { periodicWave: wave, frequency: 174 });

Setting the frequency parameter is the key! Bye again!

Then the only one left is AudioWorklet. Can it be also the answer? It makes me curious.

web-audio-api signal-processing audio javascript
1 Answers

In golang I have taken an array ARR1 which represents a time series ( could be audio or in my case an image ) where each element of this time domain array is a floating point value which represents the height of the raw audio curve as it wobbles ... I then fed this floating point array into a FFT call which returned a new array ARR2 by definition in the frequency domain where each element of this array is a single complex number where both the real and the imaginary parts are floating points ... when I then fed this array into an inverse FFT call ( IFFT ) it gave back a floating point array ARR3 in the time domain ... to a first approximation ARR3 matched ARR1 ... needless to say if I then took ARR3 and fed it into a FFT call its output ARR4 would match ARR2 ... essentially you have this time_domain_array --> FFT call -> frequency_domain_array --> InverseFFT call -> time_domain_array ... rinse N repeat

I know Web Audio API has a FFT call ... do not know whether it has an IFFT api call however if no IFFT ( inverse FFT ) you can write your own such function here is how ... iterate across ARR2 and for each element calculate the magnitude of this frequency ( each element of ARR2 represents one frequency and in the literature you will see ARR2 referred to as the frequency bins which simply means each element of the array holds one complex number and as you iterate across the array each successive element represents a distinct frequency starting from element 0 to store frequency 0 and each subsequent array element will represent a frequency defined by adding incr_freq to the frequency of the prior array element )

Each index of ARR2 represents a frequency where element 0 is the DC bias which is the zero offset bias of your input ARR1 curve if its centered about the zero crossing point this value is zero normally element 0 can be ignored ... the difference in frequency between each element of ARR2 is a constant frequency increment which can be calculated using

incr_freq := sample_rate / number_of_samples  //  with sample_rate of 44100 samples per second, and one second worth of samples ( 44100 )
                                              //  this gives you a frequency increment resolution of 1 Hertz ... IE each freq bin is 1 Hertz apart

For a given element of ARR2 you need to calculate the magnitude using

curr_real = real(curr_complex) // pluck out real portion of imaginary number
curr_imag = imag(curr_complex) // ditto for im

curr_mag = 2.0 * math.Sqrt(curr_real*curr_real+curr_imag*curr_imag) / number_of_samples  // magnitude of this freq

curr_theta = math.Atan2(curr_imag, curr_real)  //  phase of this frequency

At this point you are iterating across ARR2 and inside this loop you are doing above magnitude and phase calculation for each element of ARR2 ... now simply synthesize a sine curve for each frequency now that you know both the magnitude and phase ... remember as you iterate across ARR2 you are incrementing by the above mentioned incr_freq ... then simply combine each of these freshly synthesized sine curves into a single output curve and stow into a new ARR3 which will match your original source ARR1

Have fun and good luck !!!

UPDATE The notion of window refers to a set of audio samples ... say you have an input WAV format file from which you open up and iterate across its raw audio curve ... if this WAV file holds a 10 minute song if you feed the entire audio into one FFT call the result will represent the entire file however if you instead chop the entire set of audio samples ( these are just points on the raw audio curve ) into several windows where each window is only say 1024 such audio samples and feed each of these windows of samples into a FFT call each output from FFT call will be specific to only that portion of the song ... when you use a tool like Audacity to play a song while you view its real time FFT spectrum that is showing you a refreshed view of the current window as Audacity chops the input audio into a new window very 1024 or so audio samples

user_1471750
Related