How to convert VP8 interframe into image with Pion/Webrtc?

user_16711370 0
Viewed 553

I have tried snapshot example to convert VP8 video stream to jpeg in Pion/Webrtc examples.

It works well, but interframes are not included. An error message displayed when I was fixing it.

vp8: Golden / AltRef frames are not implemented.

This error comes from golang's vp8 package.

The modified code is shown below:

if !videoKeyframe {
    decoder.Init(bytes.NewReader(sample.Data), len(sample.Data))

    _, err := decoder.DecodeFrameHeader()
    if err != nil {
        fmt.Printf("Header Error: %s\n", err)
        continue
    }

    _, err2 := decoder.DecodeFrame()
    if err2 != nil {
        fmt.Printf("DecodeFrame Error: %s\n", err2)
        continue
    }
}

I also tried to modify gstreamer-receive example. But it's hard for me to do in C.

Is there anyway to convert interframe to jpeg easily ?

webrtc go
2 Answers

Finally, I solved with libvpx-go

I modified code as shown below.

package main

import (
    "bytes"
    "fmt"
    "image/jpeg"

    //  "log"
    "os"

    "github.com/pion/rtp"
    "github.com/pion/rtp/codecs"
    "github.com/pion/webrtc/v3/pkg/media/samplebuilder"
    "github.com/xlab/libvpx-go/vpx"
)

// type Frame struct {
//  *image.RGBA
//  Timecode   time.Duration
//  IsKeyframe bool
// }

type VDecoder struct {
    enabled bool

    src   <-chan *rtp.Packet
    ctx   *vpx.CodecCtx
    iface *vpx.CodecIface
}

type VCodec string

const (
    CodecVP8  VCodec = "V_VP8"
    CodecVP9  VCodec = "V_VP9"
    CodecVP10 VCodec = "V_VP10"
)

func NewVDecoder(codec VCodec, src <-chan *rtp.Packet) *VDecoder {
    dec := &VDecoder{
        src: src,
        ctx: vpx.NewCodecCtx(),
    }
    switch codec {
    case CodecVP8:
        dec.iface = vpx.DecoderIfaceVP8()
    case CodecVP9:
        dec.iface = vpx.DecoderIfaceVP9()
    default: // others are currently disabled
        log.Println("[WARN] unsupported VPX codec:", codec)
        return dec
    }
    err := vpx.Error(vpx.CodecDecInitVer(dec.ctx, dec.iface, nil, 0, vpx.DecoderABIVersion))
    if err != nil {
        log.Println("[WARN]", err)
        return dec
    }
    dec.enabled = true
    return dec
}

func (v *VDecoder) Save(savePath string) { //, out chan<- Frame
    //  defer close(out)
    i := 0

    sampleBuilder := samplebuilder.New(20000, &codecs.VP8Packet{}, 90000)

    for pkt := range v.src {

        sampleBuilder.Push(pkt)

        // Use SampleBuilder to generate full picture from many RTP Packets
        sample := sampleBuilder.Pop()
        if sample == nil {
            continue
        }

        if !v.enabled {
            continue
        }

        dataSize := uint32(len(sample.Data))

        err := vpx.Error(vpx.CodecDecode(v.ctx, string(sample.Data), dataSize, nil, 0))
        if err != nil {
            log.Println("[WARN]", err)
            continue
        }

        var iter vpx.CodecIter
        img := vpx.CodecGetFrame(v.ctx, &iter)
        if img != nil {
            img.Deref()

            // out <- Frame{
            //  RGBA:     img.ImageRGBA(),
            //  Timecode: time.Duration(pkt.Timestamp),
            // }

            i++

            buffer := new(bytes.Buffer)
            if err = jpeg.Encode(buffer, img.ImageYCbCr(), nil); err != nil {
                //  panic(err)
                fmt.Printf("jpeg Encode Error: %s\r\n", err)
            }

            fo, err := os.Create(fmt.Sprintf("%s%d%s", savePath, i, ".jpg"))

            if err != nil {
                fmt.Printf("image create Error: %s\r\n", err)
                //panic(err)
            }
            // close fo on exit and check for its returned error
            defer func() {
                if err := fo.Close(); err != nil {
                    panic(err)
                }
            }()

            if _, err := fo.Write(buffer.Bytes()); err != nil {
                fmt.Printf("image write Error: %s\r\n", err)
                //panic(err)
            }

            fo.Close()
        }
    }
}

Just add 2 lines of code to save sequential images

vd := NewVDecoder(CodecVP8, rtpChan)

go vd.Save("/ramdisk/image_seq/xxx")

Enjoy!

P.S. don't forget to install libvpx-go 's dependencies before running go get and make sure libvpx 's version is 1.8

user_167113700

If you need to convert interframe to jpeg the only way would be to decode every frame from your stream. I don't think there is any other way as interframes depend on previous frames.

user_88876840
Related