I have a fully dockerised application:
- nginx as proxy
- a backend server (express.js)
- a database (mongodb)
- a frontend server (express js)
- goaccess for logging
The problem is when I hit my backend endpoint with a POST request, the response is never sent to the client. A 499 code is logged by nginx along with this log
epoll_wait() reported that client prematurely closed connection, so upstream connection is closed too while sending request to upstream,
The client is the browser, there is no doubt about it.
The error arise after 1min of processing in firefox and 5min of processing in chrome. As far as I know, these times match the timeout settings of theses browsers. I could increase the timeout in firefox but it is not a viable solution.
When I get rid of the proxy, the request completes and the client get the response in about 15min. So I think there is a problem with the nginx configuration but I don't know what.
So far I tried to increase all timeout you can imagine but that didn't change anything.
I also try to set the proxy_ignore_client_abort in nginx but it is not useful in my case. Indeed the connection between nginx and my backend is still alive and the request completes after 15min (code 200 in nginx logs) but the ui is not updated because the client has terminated the connection with nginx.
I think that the browser thinks nginx is dead, because it doesn't receive any data, so it closes the TCP connection.
I'll try later on to "stimulates" this TCP connection when the request is still processing by switching between my website pages (so the browser should not close the connection), but if I have to do some weird stuff to get my backend result, it is not a viable solution.
There should be a way to process long requests without facing these browser's timeout but I don't know how. Any help would be appreciated :)
My nginx configuration:
user nginx;
pid /run/nginx.pid;
worker_processes auto;
worker_rlimit_nofile 65535;
events {
multi_accept on;
worker_connections 65535;
}
http {
charset utf-8;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
server_tokens off;
log_not_found off;
types_hash_max_size 2048;
types_hash_bucket_size 64;
client_max_body_size 16M;
# mime
include mime.types;
default_type application/octet-stream;
# logging
log_format my_log '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for" ';
access_log /var/log/nginx/access.log my_log;
error_log /var/log/nginx/error.log info;
# limits
limit_req_log_level warn;
limit_req_zone $binary_remote_addr zone=main:10m rate=10r/s;
# SSL
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:10m;
ssl_session_tickets off;
# Mozilla Intermediate configuration
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
# OCSP
ssl_stapling on;
ssl_stapling_verify on;
resolver 1.1.1.1 1.0.0.1 8.8.8.8 8.8.4.4 208.67.222.222 208.67.220.220 valid=60s;
resolver_timeout 2s;
# Connection header for WebSocket reverse proxy
map $http_upgrade $connection_upgrade {
default upgrade;
"" close;
}
map $remote_addr $proxy_forwarded_elem {
# IPv4 addresses can be sent as_is
~^[0-9.]+$ "for=$remote_addr";
# IPv6 addresses need to be bracketed and quoted
~^[0-9A-Fa-f:.]+$ "for\"[$remote_addr]\"";
# Unix domain socket names cannot be represented in RFC 7239 syntax
default "for=unknown";
}
map $http_forwarded $proxy_add_forwarded {
# If the incoming Forwarded header is syntactially valid, append to it
"~^(,[ \\t]*)*([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?(;([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?)*([ \\t]*,([ \\t]*([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?(;([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?)*)?)*$" "$http_forwarded, $proxy_forwarded_elem";
# Otherwise, replace it
default "$proxy_forwarded_elem";
}
# Load configs
include /etc/nginx/conf.d/localhost.conf;
}
and localhost.conf
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name localhost;
root /usr/share/nginx/html;
ssl_certificate /etc/nginx/live/localhost/cert.pem;
ssl_certificate_key /etc/nginx/live/localhost/key.pem;
include /etc/nginx/conf.d/security.conf;
include /etc/nginx/conf.d/proxy.conf;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log info;
# nginx render files or proxy the request
location / {
try_files $uri @front;
}
location @front {
proxy_pass http://frontend:80;
}
location ^~ /api/v1 {
proxy_read_timeout 30m; # because an inference with SIMP can takes some time
proxy_send_timeout 30m;
proxy_connect_timeout 30m;
proxy_pass http://backend:4000;
}
location = /report.html {
root /usr/share/goaccess/html/;
}
location ^~ /ws {
proxy_pass http://goaccess:7890;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_read_timeout 7d;
proxy_connect_timeout 3600;
}
include /etc/nginx/conf.d/general.conf;
}
EDIT:
The request is sent via the Angular HttpClient, maybe this module is built in a way to abort requests if a response in not send in a short time frame, I'll try to investigate on that.