How to use ws_scale to scale slice by slice of yuv image

Viewed 78

I have a YUV420 image. I want to split image into two slices (height/2) and scale(yuv->rgb) them separately using ws_scale (may be in different threads). For example

#include <string>

extern "C" {
#include <libavcodec/avcodec.h>
#include <libavutil/imgutils.h>
#include <libswscale/swscale.h>
#include <libavformat/avformat.h>
}
#include "utils.h"

AVFormatContext * initContext(const std::string& file_name) {
    auto * ctx = avformat_alloc_context();
    const char *src_filename = file_name.c_str();
    auto res = avformat_open_input(&ctx, src_filename, nullptr, nullptr);

    if (res < 0) return nullptr;

    return ctx;
}

AVCodecContext * initDecoder(AVFormatContext * ctx) {

    auto type = AVMediaType::AVMEDIA_TYPE_VIDEO;
    auto stream_index = av_find_best_stream(ctx, type, -1, -1, NULL, 0);

    if (stream_index < 0) return nullptr;

    AVStream * st = ctx->streams[stream_index];
    AVCodec * codec = avcodec_find_decoder(st->codecpar->codec_id);

    if (!codec) return nullptr;

    auto * codec_ctx = avcodec_alloc_context3(codec);
    if (!codec_ctx) return nullptr;

    auto res = avcodec_parameters_to_context(codec_ctx, st->codecpar);
    if (res < 0)  return nullptr;

    AVDictionary *opts = NULL;
    res = avcodec_open2(codec_ctx, codec, &opts);
    if (res < 0) return nullptr;

    return codec_ctx;
}

int readFrame(AVCodecContext * decoder, AVFormatContext * ctx, AVFrame * out) {
    AVPacket pkt;
    av_init_packet(&pkt);

    auto type = AVMediaType::AVMEDIA_TYPE_VIDEO;
    auto stream_index = av_find_best_stream(ctx, type, -1, -1, NULL, 0);

    for(;;) {
        int res = av_read_frame(ctx, &pkt);
        if (res < 0) return res;

        if (pkt.stream_index != stream_index) continue;

        res = avcodec_send_packet(decoder, &pkt);
        if (res < 0) return res;

        res = avcodec_receive_frame(decoder, out);

        if (res == AVERROR(EAGAIN)) continue;

        if (res < 0) return res;

        break;
    }

    return 0;
}

SwsContext * initSws(int width, int height, AVPixelFormat in, AVPixelFormat out) {
    return sws_getContext(width, height, in, width, height, out,
                          SWS_BICUBIC | SWS_FULL_CHR_H_INT | SWS_ACCURATE_RND, NULL, NULL, NULL);
}


int saveFrame(AVFrame *frame, const std::string& fileName) {
    AVCodec *codec = avcodec_find_encoder(AV_CODEC_ID_PNG);
    if (!codec)  return -1;

    AVCodecContext *out_ctx = avcodec_alloc_context3(codec);
    if (!out_ctx)  return -1;

    out_ctx->codec_type = AVMEDIA_TYPE_VIDEO;
    out_ctx->pix_fmt = AVPixelFormat::AV_PIX_FMT_RGB24;
    out_ctx->height = frame->height;
    out_ctx->width = frame->width;
    out_ctx->time_base = {1, 1000};

    if (avcodec_open2(out_ctx, codec, NULL) < 0) return -1;

    FILE *out_file;
    AVPacket packet;
    packet.data = NULL;
    packet.size = 0;

    av_init_packet(&packet);
    int gotFrame;

    if (avcodec_encode_video2(out_ctx, &packet, frame, &gotFrame) < 0) return -1;

    out_file = fopen(fileName.c_str(), "wb");
    fwrite(packet.data, 1, packet.size, out_file);
    fclose(out_file);

    av_packet_unref(&packet);
    avcodec_close(out_ctx);
    return 0;
}


int main() {

    avcodec_register_all();
    avformat_network_init();
    av_log_set_level(AV_LOG_INFO);

    AVFormatContext * fmt_ctx = initContext("res/fhd.mp4");
    if (!fmt_ctx) return -1;

    AVCodecContext * decoder = initDecoder(fmt_ctx);
    if (!decoder) return -1;

    AVFrame * frame = av_frame_alloc();
    auto res = readFrame(decoder, fmt_ctx, frame);
    if (res < 0) return -1;


    AVPixelFormat out_format = AVPixelFormat::AV_PIX_FMT_RGB24;
    SwsContext * sws = initSws(frame->width, frame->height, (AVPixelFormat) frame->format, out_format);
    if (!sws) return -1;

    AVFrame * scaled_frame = av_frame_alloc();
    scaled_frame->width = frame->width;
    scaled_frame->height = frame->height;
    scaled_frame->format = out_format;
    av_frame_get_buffer(scaled_frame, 32);

    auto h1 = frame->height/2;
    auto h2 = frame->height - h1;

    auto y1 = 0;
    auto y2 = h1;

    uint8_t * slice2[3] = {
            (uint8_t*) frame->data[0] + frame->linesize[0] * y2,
            (uint8_t*) frame->data[1] + frame->linesize[1] * y2/2,
            (uint8_t*) frame->data[2] + frame->linesize[2] * y2/2
    };

    // for image 1920x1080 res1 = 537
    auto res1 = sws_scale(sws, frame->data, frame->linesize, y1, h1, scaled_frame->data, scaled_frame->linesize);

    // res2 = 543
    auto res2 = sws_scale(sws, slice2, frame->linesize, y2, h2, scaled_frame->data, scaled_frame->linesize);

    saveFrame(scaled_frame, "img/out.png");

    return 0;
}

Is it possible with sws_scale? And if possible how to specify sw_scale paraments in this case?

srcSlice the array containing the pointers to the planes of the source slice

srcStride the array containing the strides for each plane of the source image

srcSliceY the position in the source image of the slice to process, that is the number (counted starting from zero) in the image of the first row of the slice

srcSliceH the height of the source slice, that is the number of rows in the slice

PS

I found some solution, but it works if I use same SwsContext for both slices. If I use individual context for each slice the result is transparent rectangle (height 6 px) in the middle of the image. Need some overlap of slices in this case? Or am I doing something wrong? Can I use same context in different threads?

1 Answers

It looks like the slicing feature works only when the same SwsContext is used for both slices.

In case we want to use individual context for each slice, we have to configure each SwsContext as if there are two individual images.

The fist context may "scale" the top half, and the second context may "scale" the bottom half.

  • The image size for both contexts applies height/2.
  • The fist context gets pointers to source and destination data of the top half.
  • The second context gets pointers to source and destination data of the bottom half (both source and destination pointers are advanced).

For YUV420p input, the solution is going to work only if height is a multiple of 4.


Here is the relevant part of the code:

auto h1 = frame->height / 2;
auto h2 = frame->height - h1;

auto y1 = 0;
auto y2 = h1;

//sws1 applies top half
SwsContext* sws1 = initSws(frame->width, h1, (AVPixelFormat)frame->format, out_format);  //Set the height to frame->height/2
//SwsContext* sws1 = initSws(frame->width, frame->height, (AVPixelFormat)frame->format, out_format);
if (!sws1) return -1;

//sws2 applies bottom half
SwsContext* sws2 = initSws(frame->width, h2, (AVPixelFormat)frame->format, out_format);  //Set the height to frame->height/2
if (!sws2) return -1;


AVFrame* scaled_frame = av_frame_alloc();
scaled_frame->width = frame->width;
scaled_frame->height = frame->height;
scaled_frame->format = out_format;
av_frame_get_buffer(scaled_frame, 32);


uint8_t* src_slice2[3] = {
        (uint8_t*)frame->data[0] + frame->linesize[0] * y2,
        (uint8_t*)frame->data[1] + frame->linesize[1] * y2 / 2,
        (uint8_t*)frame->data[2] + frame->linesize[2] * y2 / 2
};

//Bottom half of the destination
uint8_t* dst_slice2[1] = {
        (uint8_t*)scaled_frame->data[0] + scaled_frame->linesize[0] * y2,
};


// for image 1920x1080 res1 = 540
auto res1 = sws_scale(sws1, frame->data, frame->linesize, 0, h1, scaled_frame->data, scaled_frame->linesize);   //Set srcSliceY = 0

// res2 = 540
auto res2 = sws_scale(sws2, src_slice2, frame->linesize, 0, h2, dst_slice2, scaled_frame->linesize);    //Set srcSliceY = 0
//auto res2 = sws_scale(sws1, src_slice2, frame->linesize, y2, h2, scaled_frame->data, scaled_frame->linesize);
Related