Encoding with image/jpeg cause image saturation / wrong pixels

I have been having this problem for some time: I'm creating a module to process images, one of my functions is to go through each pixel of an image and invert colors. The function returns the expected results when encoding .png images, but it "saturates" .jpeg/jpg images.

Example when processing a .png image (correct): https://i.imgur.com/DG35RsR.png

Example when processing a .jpeg image (error): https://i.imgur.com/DZQmxJ2.png

I was researching and the closest I found to my problem, although it's not an answer, is this issue from the Go repository: https://github.com/golang/go/issues/23936

// InvertColors function

func InvertColors(img image.Image) image.Image {
    bounds := img.Bounds()
    width := bounds.Max.X
    height := bounds.Max.Y
    inverted := image.NewRGBA(bounds)

    for y := bounds.Min.Y; y < height; y++ {
        for x := bounds.Min.X; x < width; x++ {
            r, g, b, a := img.At(x, y).RGBA()
            c := color.RGBA{uint8(255 - r), uint8(255 - g), uint8(255 - b), uint8(a)}

            inverted.SetRGBA(x, y, c)
        }
    }

    return inverted
}

// main example
func main() {
    link := "https://i.imgur.com/n5hsdl4.jpg"
    img, err := GetImageFromURL(link)
    if err != nil {
        panic(err)
    }

    buf := new(bytes.Buffer)
    Encode(buf, img, "jpg")
    ioutil.WriteFile("temp.jpg", buf.Bytes(), 0666)

    invImg := InvertColors(img)
    buf = new(bytes.Buffer)
    Encode(buf, invImg, "jpg")
    ioutil.WriteFile("temp2.jpg", buf.Bytes(), 0666)
}

// GetImageFromURL 
func GetImageFromURL(link string) (image.Image, error) {
    _, format, err := ParseURL(link)
    if err != nil {
        return nil, err
    }

    req, err := http.NewRequest("GET", link, nil)
    if err != nil {
        return nil, err
    }

    // Required to make a request.
    req.Close = true
    req.Header.Set("Content-Type", "image/"+format)

    res, err := http.DefaultClient.Do(req)
    if err != nil {
        return nil, err
    }
    defer res.Body.Close()

    b, err := ioutil.ReadAll(res.Body)
    if err != nil {
        return nil, err
    }

    img, err := Decode(bytes.NewReader(b), format)
    if err != nil {
        return nil, err
    }

    return img, nil
}

func ParseURL(link string) (u *url.URL, format string, err error) {
    u, err = url.Parse(link)
    if err != nil {
        return u, "", err
    }

    format = u.Path[len(u.Path)-4:]
    if strings.Contains(format, ".") {
        format = strings.Split(format, ".")[1]
    }

    if format != "png" && format != "jpg" && format != "jpeg" {
        return u, "", fmt.Errorf("Unsupported format: %s", format)
    }

    return u, format, nil
}

func Decode(r io.Reader, format string) (img image.Image, err error) {
    if format == "png" {
        img, err = png.Decode(r)
        if err != nil {
            return nil, err
        }

    } else if format == "jpg" || format == "jpeg" {
        img, err = jpeg.Decode(r)
        if err != nil {
            return nil, err
        }

    } else {
        return nil, fmt.Errorf("Unsupported format: %s", format)
    }

    return img, nil
}