i am continuously reading mp3 files and processing them, but the memory keeps getting build up even though i freed it.
At the bottom read_audio_mp3(), they are already freeing some variable.
why do i still face a memory build up and how do i deal with it ?
following this code : https://rodic.fr/blog/libavcodec-tutorial-decode-audio-file/, i read mp3 using this function
int read_audio_mp3(string filePath_str, const int sample_rate,
double** output_buffer, int &AUDIO_DURATION){
const char* path = filePath_str.c_str();
/* Reads the file header and stores information about the file format. */
AVFormatContext* format = avformat_alloc_context();
if (avformat_open_input(&format, path, NULL, NULL) != 0) {
fprintf(stderr, "Could not open file '%s'\n", path);
return -1;
}
/* Check out the stream information in the file. */
if (avformat_find_stream_info(format, NULL) < 0) {
fprintf(stderr, "Could not retrieve stream info from file '%s'\n", path);
return -1;
}
/* find an audio stream. */
int stream_index =- 1;
for (unsigned i=0; i<format->nb_streams; i++) {
if (format->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO) {
stream_index = i;
break;
}
}
if (stream_index == -1) {
fprintf(stderr, "Could not retrieve audio stream from file '%s'\n", path);
return -1;
}
AVStream* stream = format->streams[stream_index];
// find & open codec
AVCodecContext* codec = stream->codec;
if (avcodec_open2(codec, avcodec_find_decoder(codec->codec_id), NULL) < 0) {
fprintf(stderr, "Failed to open decoder for stream #%u in file '%s'\n", stream_index, path);
return -1;
}
// prepare resampler
struct SwrContext* swr = swr_alloc();
av_opt_set_int(swr, "in_channel_count", codec->channels, 0);
av_opt_set_int(swr, "out_channel_count", 1, 0);
av_opt_set_int(swr, "in_channel_layout", codec->channel_layout, 0);
av_opt_set_int(swr, "out_channel_layout", AV_CH_LAYOUT_MONO, 0);
av_opt_set_int(swr, "in_sample_rate", codec->sample_rate, 0);
av_opt_set_int(swr, "out_sample_rate", sample_rate, 0);
av_opt_set_sample_fmt(swr, "in_sample_fmt", codec->sample_fmt, 0);
av_opt_set_sample_fmt(swr, "out_sample_fmt", AV_SAMPLE_FMT_DBL, 0);
swr_init(swr);
if (!swr_is_initialized(swr)) {
fprintf(stderr, "Resampler has not been properly initialized\n");
return -1;
}
/* Allocate an audio frame. */
AVPacket packet;
av_init_packet(&packet);
AVFrame* frame = av_frame_alloc();
if (!frame) {
fprintf(stderr, "Error allocating the frame\n");
return -1;
}
// iterate through frames
*output_buffer = NULL;
AUDIO_DURATION = 0;
while (av_read_frame(format, &packet) >= 0) {
// decode one frame
int gotFrame;
if (avcodec_decode_audio4(codec, frame, &gotFrame, &packet) < 0) {
// free packet
av_free_packet(&packet);
break;
}
if (!gotFrame) {
// free packet
av_free_packet(&packet);
continue;
}
// resample frames
double* buffer;
av_samples_alloc((uint8_t**) &buffer, NULL, 1, frame->nb_samples, AV_SAMPLE_FMT_DBL, 0);
int frame_count = swr_convert(swr, (uint8_t**) &buffer, frame->nb_samples, (const uint8_t**) frame->data, frame->nb_samples);
// append resampled frames to output_buffer
*output_buffer = (double*) realloc(*output_buffer,
(AUDIO_DURATION + frame->nb_samples) * sizeof(double));
memcpy(*output_buffer + AUDIO_DURATION, buffer, frame_count * sizeof(double));
AUDIO_DURATION += frame_count;
// free buffer & packet
av_free_packet(&packet);
av_free( buffer );
}
// clean up
av_frame_free(&frame);
swr_free(&swr);
avcodec_close(codec);
avformat_free_context(format);
return 0;
}
Main Script : MemoryLeak.cpp
// imports
#include <fstream>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <iostream>
#include <sstream>
#include <vector>
#include <sys/time.h>
extern "C"
{
#include <libavutil/opt.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libswresample/swresample.h>
}
using namespace std;
int main (int argc, char ** argv) {
string wavpath = argv[1];
printf("wavpath=%s\n", wavpath.c_str());
printf("\n==== Params =====\n");
// Init
int AUDIO_DURATION;
int sample_rate = 8000;
av_register_all();
printf("\n==== Reading MP3 =====\n");
while (true) {
// Read mp3
double* buffer;
if (read_audio_mp3(wavpath, sample_rate, &buffer, AUDIO_DURATION) != 0) {
printf("Cannot read %s\n", wavpath.c_str());
continue;
}
/*
Process the buffer for down stream tasks.
*/
// Freeing the buffer
free(buffer);
}
return 0 ;
}
Compiling
g++ -o ./MemoryLeak.out -Ofast -Wall -Wextra \
-std=c++11 "./MemoryLeak.cpp" \
-lavformat -lavcodec -lavutil -lswresample
Running, by right my input an argument wav.scp that reads text file of all the mp3s.
But for easy to replicate purpose, i only read 1 file song.mp3 in and i keep re-reading it
./MemoryLeak.out song.mp3
Why do i know i have memory leaks?
- I was running up 32 jobs in parallel for 14 million files, and when i wake up in the morning, they were abruptly killed.
- I run
htopand i monitor the progress when i re-run it, and i saw that theVIRT&RES&Memare continuously increasing.
Edit 1: My setup :
ffmpeg version 2.8.15-0ubuntu0.16.04.1
built with gcc 5.4.0