I'm trying to add a deinterlacing filter to the decoding loop of ffmpeg in Chromium by using a custom avFilter.
What I have now:
I made alll the changes required for ffmpeg to work with the yadif filter.
I created an avFilter graph:
void FFmpegDecodingLoop::InitFilterGraph(AVFrame *frame) {
if (media_log_) MEDIA_LOG(DEBUG, media_log_) << "entering InitFilterGraph : " << filter_initialised;
if (filter_initialised) return;
int result;
const AVFilter *buffer_src = avfilter_get_by_name("buffer");
const AVFilter *buffer_sink = avfilter_get_by_name("buffersink");
AVFilterInOut *inputs = avfilter_inout_alloc();
AVFilterInOut *outputs = avfilter_inout_alloc();
AVCodecContext *ctx = context_;
char args[512];
int frame_fix = 0; // fix bad width on some streams
if (frame->width < 704) frame_fix = 2;
else if (frame->width > 704) frame_fix = -16;
snprintf(args, sizeof(args),
"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",
frame->width + frame_fix,
frame->height,
frame->format,// ctx->pix_fmt,
ctx->time_base.num,
ctx->time_base.den,
ctx->sample_aspect_ratio.num,
ctx->sample_aspect_ratio.den);
const char *description = "yadif";
filter_graph = avfilter_graph_alloc();
if (media_log_) MEDIA_LOG(DEBUG, media_log_) << "Filter graph - args : " << args;
result = avfilter_graph_create_filter(&buffersrc_ctx_, buffer_src, "in", args, NULL, filter_graph);
if (result < 0) {
if (media_log_) MEDIA_LOG(ERROR, media_log_) << "Filter graph - Unable to create buffer source : " << result;
return;
}
AVBufferSinkParams *params = av_buffersink_params_alloc();
enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV420P9, AV_PIX_FMT_NONE };
params->pixel_fmts = pix_fmts;
result = avfilter_graph_create_filter(&buffersink_ctx_, buffer_sink, "out", NULL, params, filter_graph);
av_free(params);
if (result < 0) {
if (media_log_) MEDIA_LOG(ERROR, media_log_) << "Filter graph - Unable to create buffer sink";
return;
}
inputs->name = av_strdup("out");
inputs->filter_ctx = buffersink_ctx_;
inputs->pad_idx = 0;
inputs->next = NULL;
outputs->name = av_strdup("in");
outputs->filter_ctx = buffersrc_ctx_;
outputs->pad_idx = 0;
outputs->next = NULL;
result = avfilter_graph_parse_ptr(filter_graph, description, &inputs, &outputs, NULL);
if (result < 0 && media_log_) MEDIA_LOG(ERROR, media_log_) << "Filter graph - avfilter_graph_parse_ptr ERROR : " << result;
result = avfilter_graph_config(filter_graph, NULL);
if (result < 0 && media_log_) MEDIA_LOG(ERROR, media_log_) << "Filter graph - avfilter_graph_config error";
filter_initialised = true;
}
And when I receive the frame, depending on the fact it's interlaced or not, I send it to the avFilter graph or straight to the frame_ready_cb.
...
bool frame_processing_success = false;
if (!frame_.get()->interlaced_frame) { // not interlaced
if (media_log_) MEDIA_LOG(DEBUG, media_log_) << "Detected not interlaced video frame";
frame_processing_success = frame_ready_cb.Run(frame_.get());
} else {
if (media_log_) MEDIA_LOG(DEBUG, media_log_) << "Detected interlaced video frame" << frame_.get()->metadata;
if (!filter_initialised) {
this->InitFilterGraph(frame_.get());
if (media_log_) MEDIA_LOG(DEBUG, media_log_) << "Media filter ok";
}
if (av_buffersrc_add_frame_flags(buffersrc_ctx_, frame_.get(), AV_BUFFERSRC_FLAG_KEEP_REF) < 0) {
if (!continue_on_decoding_errors_)
return DecodeStatus::kDecodeFrameFailed;
decoder_error = true;
continue;
}
while (true) {
const int ret = av_buffersink_get_frame(buffersink_ctx_, filter_frame_.get());
if (media_log_) MEDIA_LOG(DEBUG, media_log_) << "ret = " << ret;
if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF){
if (media_log_) MEDIA_LOG(DEBUG, media_log_) << "ret error but waiting for more frames" << ret;
frame_processing_success = true;
if (media_log_) MEDIA_LOG(DEBUG, media_log_) << "ret error but waitning for more frames ; filter description = " << buffersink_ctx_->filter->description;
break;
}
if (ret < 0) {
if (!continue_on_decoding_errors_)
return DecodeStatus::kDecodeFrameFailed;
decoder_error = true;
frame_processing_success = true;
break;
}
frame_processing_success = frame_ready_cb.Run(filter_frame_.get());
if (media_log_) MEDIA_LOG(DEBUG, media_log_) << "producing deinterlaced frame : " << frame_processing_success;
av_frame_unref(filter_frame_.get());
}
}
av_frame_unref(frame_.get());
if (!frame_processing_success)
return DecodeStatus::kFrameProcessingFailed;
}
This seems to work, I can send frames to the avFilter and get them back but I'm having an issue with the metadas (it seems).
The issue:
When I get a frame from the avFilter (av_buffersink_get_frame(buffersink_ctx_, filter_frame_.get())) and send it to the frame ready callback I have this kind of errors :
[36723:38755:0914/105148.322600:FATAL:values.cc(516)] Check failed: is_dict().
0 libbase.dylib 0x00000001052febff base::debug::CollectStackTrace(void**, unsigned long) + 31
1 libbase.dylib 0x0000000104f9717b base::debug::StackTrace::StackTrace(unsigned long) + 75
2 libbase.dylib 0x0000000104f971fd base::debug::StackTrace::StackTrace(unsigned long) + 29
3 libbase.dylib 0x0000000104f971d8 base::debug::StackTrace::StackTrace() + 40
4 libbase.dylib 0x0000000104fe4897 logging::LogMessage::~LogMessage() + 183
5 libbase.dylib 0x0000000104fe3515 logging::LogMessage::~LogMessage() + 21
6 libbase.dylib 0x00000001052df4dc base::Value::SetKey(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&&, base::Value&&) + 188
7 libmedia.dylib 0x000000010eda0dea media::VideoFrameMetadata::SetBoolean(media::VideoFrameMetadata::Key, bool) + 90
8 libmedia.dylib 0x000000010efb6943 media::FFmpegVideoDecoder::OnNewFrame(AVFrame*) + 403
9 libmedia.dylib 0x000000010efb716b bool base::internal::FunctorTraits<bool (media::FFmpegVideoDecoder::*)(AVFrame*), void>::Invoke<bool (media::FFmpegVideoDecoder::*)(AVFrame*), media::FFmpegVideoDecoder*, AVFrame*>(bool (media::FFmpegVideoDecoder::*)(AVFrame*), media::FFmpegVideoDecoder*&&, AVFrame*&&) + 155
10 libmedia.dylib 0x000000010efb7046 bool base::internal::InvokeHelper<false, bool>::MakeItSo<bool (media::FFmpegVideoDecoder::* const&)(AVFrame*), media::FFmpegVideoDecoder*, AVFrame*>(bool (media::FFmpegVideoDecoder::* const&)(AVFrame*), media::FFmpegVideoDecoder*&&, AVFrame*&&) + 102
11 libmedia.dylib 0x000000010efb6fa5 bool base::internal::Invoker<base::internal::BindState<bool (media::FFmpegVideoDecoder::*)(AVFrame*), base::internal::UnretainedWrapper<media::FFmpegVideoDecoder> >, bool (AVFrame*)>::RunImpl<bool (media::FFmpegVideoDecoder::* const&)(AVFrame*), std::__1::tuple<base::internal::UnretainedWrapper<media::FFmpegVideoDecoder> > const&, 0ul>(bool (media::FFmpegVideoDecoder::* const&)(AVFrame*), std::__1::tuple<base::internal::UnretainedWrapper<media::FFmpegVideoDecoder> > const&, std::__1::integer_sequence<unsigned long, 0ul>, AVFrame*&&) + 101
12 libmedia.dylib 0x000000010efb6f0a base::internal::Invoker<base::internal::BindState<bool (media::FFmpegVideoDecoder::*)(AVFrame*), base::internal::UnretainedWrapper<media::FFmpegVideoDecoder> >, bool (AVFrame*)>::Run(base::internal::BindStateBase*, AVFrame*) + 106
13 libmedia.dylib 0x000000010f127101 base::RepeatingCallback<bool (AVFrame*)>::Run(AVFrame*) const & + 113
14 libmedia.dylib 0x000000010f126e70 media::FFmpegDecodingLoop::DecodePacket(AVPacket const*, base::RepeatingCallback<bool (AVFrame*)>) + 2528
So the error is in FFmpegVideoDecoder::OnNewFrame when it calls media::VideoFrameMetadata::SetBoolean.
I get the same kind of errors when void VideoFrameMetadata::SetTimeTicks gets called.
So obviously I have an issue with the metadatas of the video frame I get back from the avFilter which has just a yadif filter.
Is there something I missed with the metadas or otherwise ?