Decode JPG byte array to Bitmap

Viewed 364

I have a raspberry Pi 4 equipped with Camera, and a process is capturing image using OpenCV library. This image is then forwarded to two destinations:

  1. Standard laptop equipped with Arch Linux
  2. Android smartphone

This is the Image server on the Raspberry PI (C++):

#define MAX_IMAGESIZE 40090
typedef struct __attribute__((packed))
{
    uint16_t len;
    uint8_t data[MAX_IMAGESIZE];
} image_msg;

void __attribute__((noreturn)) camera_task()
{
    cv::VideoCapture capture(0);
    int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
    struct sockaddr_in pcaddr, phaddr;

    memset(&pcaddr, 0x00, sizeof(struct sockaddr_in));
    memset(&phaddr, 0x00, sizeof(struct sockaddr_in));

    pcaddr.sin_family = AF_INET;
    pcaddr.sin_addr.s_addr = inet_addr(PC_ADDRESS);
    pcaddr.sin_port = htons(4321);

    phaddr.sin_family = AF_INET;
    phaddr.sin_addr.s_addr = inet_addr(PH_ADDRESS);
    phaddr.sin_port = htons(4321);

    if (!capture.isOpened())
    {
        exit(EXIT_FAILURE);
    }

    cv::Mat frame;
    while (1)
    {
        if (!capture.read(frame))
        {
            exit(EXIT_FAILURE);
        }

        cv::flip(frame, frame, -1);
        std::vector<int> params;
        std::vector<uint8_t> buffer;

        params.push_back(cv::IMWRITE_JPEG_QUALITY);
        params.push_back(60);
        cv::imencode(".jpg", frame, buffer, params);
    
        image_msg outmsg;
        outmsg.len = static_cast<uint16_t>(buffer.size());
        for(uint16_t i = 0; i < outmsg.len; i++)
        {
            outmsg.data[i] = buffer[i];
        }
        if(sendto(sock, reinterpret_cast<char*>(&outmsg), sizeof(outmsg), 0, reinterpret_cast<struct sockaddr*>(&pcaddr), sizeof(pcaddr)) > 0)
        {
             //printf("OK sendto PC\n");
        }
        else
        {
             perror("Camera task to PC");
        }
        if(sendto(sock, reinterpret_cast<char*>(&outmsg), sizeof(outmsg), 0, reinterpret_cast<struct sockaddr*>(&phaddr), sizeof(phaddr)) > 0)
        {
            //printf("OK sendto PHONE\n");
        }
        else
        {
             perror("Camera task to PHONE");
        }
    }
}

This is the Image client on the laptop (C++):

cv::Mat *imagewindow;
void init_window()
{
    init_localsock(&imu_socket, &imu_saddr, IMUPORT); //not related with camera
    init_localsock(&speed_socket, &speed_saddr, VELPORT); //not related with camera
    init_localsock(&attitude_socket, &attitude_saddr, ATTPORT); //not related with camera
    init_localsock(&radiation_socket, &radiation_saddr, RADPORT); //not related with camera
    init_localsock(&throttle_socket, &throttle_saddr, THRPORT); //not related with camera
    init_localsock(&image_socket, &image_saddr, RENPORT); //initialize socket on port 4321

    imagewindow = new cv::Mat(600, 600, CV_8UC3, cv::Scalar(0, 0, 0));

    cv::imshow(PROJNAME, *imagewindow);
}
void main_loop(const char* board_address)
{
    while(true)
    {
        imu_task();
        speed_task();
        attitude_task();

        radiation_task();

        throttle_task();

        image_task(); //this is the image receiver task

        render_window();

        cmd_out_task(board_address);
    }
}

void image_task()
{
    image_msg recv; //the same struct defined on the raspberry
    socklen_t len;
    ssize_t bytes_recv = recvfrom(image_socket, &recv, sizeof(image_msg), 0, (struct sockaddr*)&image_saddr, &len);
    if(bytes_recv > 0)
    {
        memset(imagewindow->data, 0x00, imagewindow->dataend - imagewindow->data);
        update_image(recv);
    }
}

void update_image(image_msg image)
{
    std::vector<char> data(image.data, image.data + image.len);
    *imagewindow = cv::imdecode(cv::Mat(data), 1);
}

And it works fine. Here a screenshot of what's happening: Image is correctly received from the laptop

Here we go to the problem. The receiver on android is the following:

private byte[] recv_image(DatagramSocket sock) throws Exception
{
    byte[] b = new byte[40090];
    DatagramPacket p = new DatagramPacket(b, b.length);
    
    byte[] recv_length = new byte[4];
    length[3] = b[0];
    length[2] = b[1];
    int recv_len = ByteBuffer.wrap(length).getInt();
    byte[] image_bytes = new byte[recv_len];
    for(int i = 0; i < recv_len; i++)
    {
        image_bytes[i] = b[4 + i];
    }
    return image_bytes;
}

The receive thread is doing:

initialize_videosocket(); //init a local UDP socket on port 4321
while(true)
{
     try
     {
         final byte[] imagebytes = recv_image(video_sock);
         Bitmap bmp = BitmapFactory.decodeByteArray(imagebytes, 0, imagebytes.length);
     }
     catch(Exception e)
     {
         Log.d("Exception in image thread: " + e.toString();
     }
 }

The problem is that Bitmap bmp is always null. From the documentation, decodeByteArray returns null if the JPG is unvalid. These are the attempts I did:

  1. Tried using opencv in my android app:

     Mat mat = Imgcodecs.imdecode(new MatOfByte(imagebytes), Imgcodecs.CV_LOAD_IMAGE_UNCHANGED);
     Bitmap bmp = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888);
     Utils.matToBitmap(rgb, bmp);
    

    But mat.cols() and mat.rows() returns both -1. I forced arguments to be 480,640 (as it should be) and I get an assertion failed on the execution of Utils.matToBitmap.

  2. Tried to compare received jpg buffers both on the laptop and on the phone. They are equals.

  3. Tried to write jpg buffer on a file .jpg and opening it as standard .jpg file with Gimp. It works, so the buffer is containing a correct .jpg file, but the decodeByteArray should return null if the buffer is not a valid .jpg file.

I'm really stuck in this, any help is appreciated. Thanks

1 Answers

Solved the Issue, it was trivial.

The android function which receives image was doing:

private byte[] recv_image(DatagramSocket sock) throws Exception
{
    byte[] b = new byte[40090];
    DatagramPacket p = new DatagramPacket(b, b.length);

    byte[] recv_length = new byte[4];
    length[3] = b[0];
    length[2] = b[1];
    int recv_len = ByteBuffer.wrap(length).getInt();
    byte[] image_bytes = new byte[recv_len];
    for(int i = 0; i < recv_len; i++)
    {
        image_bytes[i] = b[4 + i]; //HERE it is the problem
    }
    return image_bytes;
}

The len field of the struct is an uint16_t. I need to convert it to an int(32) in Java, in order to avoid negative values which may be evaluated inside a Java program if I use ByteBuffer.wrap(b, 0, 2).getShort() because the java short type is signed. The C++ struct the server is sending is declared __attribute__((packed)):

typedef struct __attribute__((packed))
{
    uint16_t len;
    uint8_t data[MAX_IMAGESIZE];
} image_msg;

Hence the data field begins at base+0x02 and not base+0x04:

private byte[] recv_image(DatagramSocket sock) throws Exception
{
    byte[] b = new byte[40090];
    DatagramPacket p = new DatagramPacket(b, b.length);

    byte[] recv_length = new byte[4];
    length[3] = b[0];
    length[2] = b[1];
    int recv_len = ByteBuffer.wrap(length).getInt();
    byte[] image_bytes = new byte[recv_len];
    for(int i = 0; i < recv_len; i++)
    {
        image_bytes[i] = b[2 + i]; //2 + i instead of 4 + i
    }
    return image_bytes;
}

This solution fixed my problem, as you can see here. Thanks for your support.

Related