Ffmpeg C++ Could not find a valid device and can't configure encoder

I'm trying to change my encoders format to "h264" from "mpeg4" but I'm getting these two errors back-to-back:

[h264_v4l2m2m @ 0x55d6ba5b0780] Could not find a valid device
[h264_v4l2m2m @ 0x55d6ba5b0780] can't configure encoder

I've seen this question and tried to run ./configure --enable-shared --enable-libx264 --enable-encoder=libx264 --enable-gpl It succeeded and when I run ffmpeg -codecs | grep h264 I get this as output:

 DEV.LS h264    H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10 (decoders: h264 h264_v4l2m2m ) (encoders: h264_v4l2m2m )

But after all these I'm still getting the error. Why is this? I just want my encoder to encode in h264 format. Here is my code so far (this is mostly the muxing.c example of ffpmeg):

video_encoder::video_encoder(int w, int h, float fps, unsigned int duration) 
 :width(w), height(h), STREAM_FRAME_RATE(fps), STREAM_DURATION(duration)
{
    std::string as_str = "./output/video.mp4";

    char* filename = const_cast<char*>(as_str.c_str());
    enc_inf.video_st, enc_inf.audio_st = (struct OutputStream) { 0 };
    enc_inf.video_st.next_pts = 1; 
    enc_inf.encode_video = 0;
    int ret;
    int i;
    
    avformat_alloc_output_context2(&enc_inf.oc, NULL, "h264", filename);

    //video_codec = avcodec_find_encoder(AV_CODEC_ID_H264);

    if (!enc_inf.oc) {
        std::cout << "FAILED" << std::endl;
        avformat_alloc_output_context2(&enc_inf.oc, NULL, "mpeg", filename);
    }

    enc_inf.fmt = enc_inf.oc->oformat;

    //enc_inf.fmt->video_codec = AV_CODEC_ID_H264;

    if (enc_inf.fmt->video_codec != AV_CODEC_ID_NONE) {
        add_stream(&enc_inf.video_st, enc_inf.oc, &video_codec, enc_inf.fmt->video_codec);
        enc_inf.have_video = 1;
        enc_inf.encode_video = 1;
    }

    if (enc_inf.have_video)
        open_video(enc_inf.oc, video_codec, &enc_inf.video_st, opt);

    av_dump_format(enc_inf.oc, 0, filename, 1);

    if (!(enc_inf.fmt->flags & AVFMT_NOFILE)) {
        ret = avio_open(&enc_inf.oc->pb, filename, AVIO_FLAG_WRITE);
        if (ret < 0) {
            exit(0);
        }
    }
    ret = avformat_write_header(enc_inf.oc, &opt);
    if (ret < 0) {
        exit(0);
    }
}

void video_encoder::add_stream(OutputStream *ost, AVFormatContext *oc,
                       const AVCodec **codec,
                       enum AVCodecID codec_id)
{
    AVCodecContext *c;
    int i;

    /* find the encoder */
    *codec = avcodec_find_encoder(codec_id);
    
    if (!(*codec)) {
        fprintf(stderr, "Could not find encoder for '%s'\n",
                avcodec_get_name(codec_id));
        exit(1);
    }

    ost->tmp_pkt = av_packet_alloc();

    if (!ost->tmp_pkt) {
        fprintf(stderr, "Could not allocate AVPacket\n");
        exit(1);
    }

    ost->st = avformat_new_stream(oc, NULL);
    if (!ost->st) {
        fprintf(stderr, "Could not allocate stream\n");
        exit(1);
    }
    ost->st->id = oc->nb_streams-1;
    c = avcodec_alloc_context3(*codec);
    if (!c) {
        fprintf(stderr, "Could not alloc an encoding context\n");
        exit(1);
    }
    ost->enc = c;


    switch ((*codec)->type) {
    case AVMEDIA_TYPE_VIDEO:
        c->codec_id = codec_id;

        c->bit_rate = 10000;
        c->width    = width;
        c->height   = height;
        ost->st->time_base = (AVRational){ 1, STREAM_FRAME_RATE }; // *frame_rate
        c->time_base       = ost->st->time_base;

        c->gop_size      = 7;
        c->pix_fmt       = STREAM_PIX_FMT;
        if (c->codec_id == AV_CODEC_ID_MPEG1VIDEO) {
            c->mb_decision = 2;
        }

        break;
    }
     

    /* Some formats want stream headers to be separate. */
    if (oc->oformat->flags & AVFMT_GLOBALHEADER)
        c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
}
void video_encoder::open_video(AVFormatContext *oc, const AVCodec *codec,
                       OutputStream *ost, AVDictionary *opt_arg)
{
    int ret;
    AVCodecContext *c = ost->enc;
    AVDictionary *opt = NULL;
    av_dict_copy(&opt, opt_arg, 0);
    /* open the codec */
    ret = avcodec_open2(c, codec, &opt);
    av_dict_free(&opt);
    
    if (ret < 0) {
        fprintf(stderr, "Could not open video codec: %s\n", ret);
        exit(1);
    }

    /* allocate and init a re-usable frame */
    ost->frame = alloc_picture(c->pix_fmt, c->width, c->height);
    if (!ost->frame) {
        fprintf(stderr, "Could not allocate video frame\n");
        exit(1);
    }

    /* If the output format is not YUV420P, then a temporary YUV420P
     * picture is needed too. It is then converted to the required
     * output format. */
    ost->tmp_frame = NULL;


    /* copy the stream parameters to the muxer */
    ret = avcodec_parameters_from_context(ost->st->codecpar, c);
    if (ret < 0) {
        fprintf(stderr, "Could not copy the stream parameters\n");
        exit(1);
    }
}

Error happens at ret = avcodec_open2(c, codec, &opt); inside video_encoder::open_video.

Am I setting the format wrong or is my configuration faulty? I'm using ubuntu.