Select/Sockets: How to catch a CTRL+D input with no data before?

I'm working with sockets and select to check if a client disconnected I check the return value of recv after a call to FD_ISSET like this:

if (FD_ISSET(cl_fd, &fdset_rd)) {
    char buf[RECV_BUFSIZE + 1];
    int rd_ret = recv(cl_fd, buf, RECV_BUFSIZE, 0);
    if (rd_ret <= 0) {
        // client disconnected
    } else {
        // handle message from client
    }
}

So it works fine, even when I input CTRL+D in netcat after some data it successfully reads from the socket, but the only case where it seems broken is when I input CTRL+D directly with no data in front: in this case FD_ISSET doesn't return true and even when I send some input after the CTRL+D the socket seems to be in a "dead state", the client is still connected but the server can't receive any data.

Which is weird because when I input for example: hello^Dhello, the server successfully reads the first hello and the second, but when I input ^Dhello it doesn't work. The logical behavior should be to receive 0 characters and disconnect the client but I receive nothing and the client stays connected but becomes unreadable.

Here's a minimal reproductible example to see the problem:

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/select.h>
#include <stdbool.h>

#define RECV_BUFSIZE 2048

int main( int argc, char **argv ) {
    if (argc < 2) {
        printf("usage: %s <port>\n", argv[0]);
        return 1;
    }

    int sockfd;
    int cl_fd[__FD_SETSIZE];
    size_t cl_cnt = 0;
    struct sockaddr_in servaddr = {
        .sin_family = AF_INET,
        .sin_addr.s_addr = htonl(2130706433), // 127.0.0.1
        .sin_port = htons(atoi(argv[1]))
    };
    fd_set fdset_all = { };
    fd_set fdset_rd;

    sockfd = socket(AF_INET, SOCK_STREAM, 0);
    if (sockfd == -1
    || bind(sockfd, (void*)&servaddr, sizeof(servaddr)) == -1
    || listen(sockfd, 32) == -1)
        return 1;
    FD_SET(sockfd, &fdset_all);

    while (true) {
        fdset_rd = fdset_all;
        if (select(__FD_SETSIZE, &fdset_rd, NULL, NULL, NULL) == -1)
            return 1;

        if (FD_ISSET(sockfd, &fdset_rd)) {
            cl_fd[cl_cnt] = accept(sockfd, NULL, NULL);
            if (cl_fd[cl_cnt] == -1)
                return 1;
            printf("client arrived (socket %d)\n", cl_fd[cl_cnt]);
            FD_SET(cl_fd[cl_cnt], &fdset_all);
            ++cl_cnt;
        }

        for (size_t i = 0; i < cl_cnt; ++i) {
            if (FD_ISSET(cl_fd[i], &fdset_rd)) {
                char buf[RECV_BUFSIZE];
                int rd_ret = recv(cl_fd[i], buf, RECV_BUFSIZE, 0);

                printf("recv returned %d (socket %d)\n", rd_ret, cl_fd[i]);
                if (rd_ret <= 0) {
                    printf("client left (socket %d)\n", cl_fd[i]);
                    FD_CLR(cl_fd[i], &fdset_all);
                    close(cl_fd[i]);
                    cl_fd[i] = cl_fd[--cl_cnt];
                }
            }
        }
    }
    return 0;
}

Launch this program, then launch nc localhost <port> and try writing hello^Dhello: both hello are received by the server.

then try writing just ^D: nothing is received, and if you try to send other messages after that nothing will be received as well, but when you CTRL+C in nc after that the client disconnection still gets received