I've made a program that takes a video as in input, edits it in opengl and gives an output by encoding it. So far the video section of the program worked just fine. Then I added audio, normally it finishes without any errors and the output video has an audio, but the audio is wrong, I can hear the intended audio in the background but its mostly a distorted noise. I just want to decode an audio and use that decoded audio and put it on another video. I used the Muxing.c example of ffmpeg to create my encoder, normally there was a dummy audio before I removed it. Here is what it looks like now (I'm trimming some parts, mainly video encoding parts):
video_encoder.cpp:
short video_encoder::encode_one_frame()
{
if (enc_inf.encode_video || enc_inf.encode_audio) {
/* select the stream to encode */
if (enc_inf.encode_video &&
(!enc_inf.encode_audio || av_compare_ts(enc_inf.video_st.next_pts, enc_inf.video_st.enc->time_base,
enc_inf.audio_st.next_pts, enc_inf.audio_st.enc->time_base) <= 0)) {
//std::cout << "Allocing video..." << std::endl;
enc_inf.encode_video = !write_video_frame(enc_inf.oc, &enc_inf.video_st);
return 1;
} else {
//std::cout << "Allocing audio..." << std::endl;
enc_inf.encode_audio = !write_audio_frame(enc_inf.oc, &enc_inf.audio_st);
return 2;
}
}
return 0;
}
void video_encoder::set_audio_frame(AVFrame* audio, AVSampleFormat* format)
{
audio_data = *audio;
input_sample_fmt = *format;
}
int video_encoder::write_audio_frame(AVFormatContext *oc, OutputStream *ost)
{
AVCodecContext *c;
AVFrame *frame;
int ret;
int dst_nb_samples;
c = ost->enc;
#if __AUDIO_ENABLED
c->sample_fmt = input_sample_fmt;
#endif
frame = get_audio_frame(ost);
if (frame) {
/* convert samples from native format to destination codec format, using the resampler */
/* compute destination number of samples */
dst_nb_samples = av_rescale_rnd(swr_get_delay(ost->swr_ctx, c->sample_rate) + frame->nb_samples,
c->sample_rate, c->sample_rate, AV_ROUND_UP);
//av_assert0(dst_nb_samples == frame->nb_samples);
/* when we pass a frame to the encoder, it may keep a reference to it
* internally;
* make sure we do not overwrite it here
*/
ret = av_frame_make_writable(ost->frame);
if (ret < 0)
exit(1);
/* convert to destination format */
ret = swr_convert(ost->swr_ctx,
ost->frame->data, dst_nb_samples,
(const uint8_t **)frame->data, frame->nb_samples);
if (ret < 0) {
fprintf(stderr, "Error while converting\n");
exit(1);
}
frame = ost->frame;
frame->pts = av_rescale_q(ost->samples_count, (AVRational){1, c->sample_rate}, c->time_base);
ost->samples_count += dst_nb_samples;
}
return write_frame(oc, c, ost->st, frame, ost->tmp_pkt);
}
AVFrame* video_encoder::get_audio_frame(OutputStream *ost)
{
AVFrame *frame = ost->tmp_frame;
int j, i, v;
int16_t *q = (int16_t*)frame->data[0];
#if __AUDIO_ENABLED
*ost->tmp_frame = audio_data;
#endif
//(int16_t)*audio_frame->data[0];
/* check if we want to generate more frames */
if (av_compare_ts(ost->next_pts, ost->enc->time_base,
STREAM_DURATION, (AVRational){ 1, 1 }) > 0)
return NULL;
for (j = 0; j <frame->nb_samples; j++) {
#if !__AUDIO_ENABLED
v = (int)(sin(ost->t) * 10000);
#endif
for (i = 0; i < ost->enc->channels; i++)
#if !__AUDIO_ENABLED
*q++ = v;
#endif
ost->t += ost->tincr;
ost->tincr += ost->tincr2;
}
frame->pts = ost->next_pts;
ost->next_pts += frame->nb_samples;
#if __AUDIO_ENABLED
return &audio_data;
#else
return frame;
#endif
}
void video_encoder::open_audio(AVFormatContext *oc, const AVCodec *codec,
OutputStream *ost, AVDictionary *opt_arg)
{
AVCodecContext *c;
int nb_samples;
int ret;
AVDictionary *opt = NULL;
c = ost->enc;
/* open it */
av_dict_copy(&opt, opt_arg, 0);
ret = avcodec_open2(c, codec, &opt);
av_dict_free(&opt);
if (ret < 0) {
fprintf(stderr, "Could not open audio codec: %s\n", ret);
exit(1);
}
/* init signal generator */
ost->t = 0;
ost->tincr = 2 * M_PI * 110.0 / c->sample_rate;
/* increment frequency by 110 Hz per second */
ost->tincr2 = 2 * M_PI * 110.0 / c->sample_rate / c->sample_rate;
if (c->codec->capabilities & AV_CODEC_CAP_VARIABLE_FRAME_SIZE)
nb_samples = 10000;
else
nb_samples = c->frame_size;
ost->frame = alloc_audio_frame(c->sample_fmt, c->channel_layout,
c->sample_rate, nb_samples);
ost->tmp_frame = alloc_audio_frame(AV_SAMPLE_FMT_S16, c->channel_layout,
c->sample_rate, nb_samples);
/* copy the stream parameters to the muxer */
ret = avcodec_parameters_from_context(ost->st->codecpar, c);
if (ret < 0) {
fprintf(stderr, "Could not copy the stream parameters\n");
exit(1);
}
/* create resampler context */
ost->swr_ctx = swr_alloc();
if (!ost->swr_ctx) {
fprintf(stderr, "Could not allocate resampler context\n");
exit(1);
}
/* set options */
av_opt_set_int (ost->swr_ctx, "in_channel_count", c->channels, 0);
av_opt_set_int (ost->swr_ctx, "in_sample_rate", c->sample_rate, 0);
av_opt_set_sample_fmt(ost->swr_ctx, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
av_opt_set_int (ost->swr_ctx, "out_channel_count", c->channels, 0);
av_opt_set_int (ost->swr_ctx, "out_sample_rate", c->sample_rate, 0);
av_opt_set_sample_fmt(ost->swr_ctx, "out_sample_fmt", c->sample_fmt, 0);
/* initialize the resampling context */
if ((ret = swr_init(ost->swr_ctx)) < 0) {
fprintf(stderr, "Failed to initialize the resampling context\n");
exit(1);
}
}
int video_encoder::write_frame(AVFormatContext *fmt_ctx, AVCodecContext *c,
AVStream *st, AVFrame *frame, AVPacket *pkt)
{
int ret;
// Conditional jump or move depends on uninitialised value
// Use of uninitialised value of size 8
// send the frame to the encoder
// Error is about c.
ret = avcodec_send_frame(c, frame);
if (ret < 0) {
fprintf(stderr, "Error sending a frame to the encoder: %s\n",
ret);
exit(1);
}
while (ret >= 0) {
ret = avcodec_receive_packet(c, pkt);
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
break;
else if (ret < 0) {
fprintf(stderr, "Error encoding a frame: %s\n", ret);
exit(1);
}
/* rescale output packet timestamp values from codec to stream timebase */
av_packet_rescale_ts(pkt, c->time_base, st->time_base);
pkt->stream_index = st->index;
/* Write the compressed frame to the media file. */
//log_packet(fmt_ctx, pkt);
//std::cout << "Packet: " << pkt << std::endl;
ret = av_interleaved_write_frame(fmt_ctx, pkt);
/* pkt is now blank (av_interleaved_write_frame() takes ownership of
* its contents and resets pkt), so that no unreferencing is necessary.
* This would be different if one used av_write_frame(). */
if (ret < 0) {
fprintf(stderr, "Error while writing output packet: %s\n", ret);
exit(1);
}
}
return ret == AVERROR_EOF ? 1 : 0;
}
video_encoder::video_encoder()
{
...
enc_inf.video_st, enc_inf.audio_st = (struct OutputStream) { 0 };
enc_inf.video_st.next_pts = 1;
enc_inf.audio_st.next_pts = 1;
enc_inf.encode_audio, enc_inf.encode_video = 0;
...
if (enc_inf.fmt->audio_codec != AV_CODEC_ID_NONE) {
add_stream(&enc_inf.audio_st, enc_inf.oc, &audio_codec, enc_inf.fmt->audio_codec);
enc_inf.have_audio = 1;
enc_inf.encode_audio = 1;
}
/* Now that all the parameters are set, we can open the audio and
* video codecs and allocate the necessary encode buffers. */
if (enc_inf.have_video)
open_video(enc_inf.oc, video_codec, &enc_inf.video_st, opt);
if (enc_inf.have_audio)
open_audio(enc_inf.oc, audio_codec, &enc_inf.audio_st, opt);
}
And here is how I read the audio:
video_decoder.cpp:
int video_decode::video_reader_read_frame(){
while (av_read_frame(av_format_ctx, av_packet) >= 0) {
// Audio decode
if (av_packet->stream_index == video_stream_index){
response = avcodec_send_packet(av_codec_ctx, av_packet);
if (response < 0) {
printf("Failed to decode packet: %s\n", av_make_error(response));
return false;
}
response = avcodec_receive_frame(av_codec_ctx, av_frame);
if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
av_packet_unref(av_packet);
continue;
} else if (response < 0) {
printf("Failed to decode packet: %s\n", av_make_error(response));
return false;
}
*pts = av_frame->pts;
...
#if __AUDIO_ENABLED
else if (av_packet->stream_index == audio_stream_index){
decode_audio(audio_codec_ctx, av_packet);
av_packet_unref(av_packet);
return 2;
}
#endif
}
}
uint8_t** video_decode::decode_audio(AVCodecContext *dec, const AVPacket *pkt)
{
auto& frame= state.av_frame;
uint8_t ret = 0;
// submit the packet to the decoder
ret = avcodec_send_packet(dec, pkt);
if (ret < 0) {
std::cout << "Error submitting a packet for decoding" << std::endl;
}
// get all the available frames from the decoder
while (ret >= 0) {
ret = avcodec_receive_frame(dec, frame);
if (ret < 0) {
// those two return values are special and mean there is no output
// frame available, but there were no errors during decoding
if (ret == AVERROR_EOF || ret == AVERROR(EAGAIN)){
return nullptr;
}
std::cout << "Decode err" << std::endl;
}
return frame->data;
}
return nullptr;
}
bool video_decode::video_reader_open(const char* filename)
{
...
if (open_codec_context(&audio_stream_index, &audio_dec_ctx, av_format_ctx, AVMEDIA_TYPE_AUDIO) >= 0) {
audio_stream = av_format_ctx->streams[audio_stream_index];
}
...
}
And here is how everything comes together inside main:
int main()
{
...
video_encoder encoder = new video_encoder();
video_decode decoder = new video_decode("file_name");
decoder.video_reader_open();
int frame_Type = 0;
...
while(true)
{
...
decoder->video_reader_read_frame();
...
if(encoder->enc_inf.encode_video || encoder->enc_inf.encode_audio)
{
if(!time_line.audio_only)
{
set_video_frame(fbo);
encoder->set_encode_framebuffer(fbo);
}else{
uint8_t* fbo = nullptr;
encoder->set_encode_framebuffer(fbo);
}
#if __AUDIO_ENABLED
if(frame_type == 2)
frame_type = encoder->set_audio_frame(audio_test->get_state().av_frame, &audio_test->get_state().audio_codec_ctx->sample_fmt);
#endif
}else{
break;
}
}
}
I know I'm doing something wrong when connecting the decoder and the encoder but not quite sure where I went wrong.
Using ubuntu.