I am using nginx and a spring boot application with Netty server, but for some requests nginx is throwing 502 error even though netty access logs are showing 200OK for the same request. So basically the response packet is being dropped in between netty server and nginx.
This is my nginx.conf
daemon off;
worker_processes 4;
worker_rlimit_nofile 100000;
pid /var/run/nginx.pid;
error_log /opt/logs/myservice/nginx-error.log warn;
events {
worker_connections 512;
use epoll;
multi_accept off;
}
http {
#server_tokens off;
#include mime.types;
default_type application/octet-stream;
################# Gzip Settings ################
gzip on;
gzip_comp_level 4;
gzip_min_length 1024;
gzip_proxied any;
gzip_static on;
gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript text/x-js application/soap
+xml;
gzip_disable "MSIE [1-6].";
####################################################
log_format upstreamlog '$time_local $status $remote_addr to:- $upstream_addr $request -- upstream_response_time:$upstream_response_time request_time:$request_time tid_header:$http_tid status:$upstream_cache_status slot:$http_slot slotTime:$http_slotstarttime ttlReq:$http_ttl ttlResp:$upstream_http_x_accel_expires jobFlag:$http_jobflag cookies:"$http_cookie" bytes_sent:$bytes_sent gzip_ratio:$gzip_ratio "$http_referer" "$http_user_agent" $http_x_forwarded_for';
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 30;
keepalive_requests 10000;
reset_timedout_connection on;
client_body_timeout 30;
send_timeout 300;
set_real_ip_from 10.117.0.0/16;
real_ip_header X-Forwarded-For;
real_ip_recursive on;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $http_scheme;
proxy_set_header X-Real-IP $remote_addr;
server {
listen 80;
server_name 127.0.0.1;
client_header_buffer_size 64k;
large_client_header_buffers 4 64k;
client_max_body_size 2M;
if ($host ~* ^(example)) {
rewrite ^/(.*)$ https://www.example.com/$1 permanent;
}
access_log /opt/logs/myservice/nginx-frontend.log upstreamlog;
location / {
# Proxy Settings
proxy_pass http://127.0.0.1:8000$request_uri;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $http_scheme;
proxy_set_header X-Real-IP $remote_addr;
proxy_hide_header Set-Cookie;
proxy_ignore_headers Set-Cookie;
proxy_buffering off;
proxy_buffers 8 16k;
proxy_buffer_size 16k;
proxy_set_header Cookie "";
fastcgi_read_timeout 120;
proxy_read_timeout 120;
client_max_body_size 500M;
add_header Cache-Control "no-cache, , max-age=0, must-revalidate, no-store";
################# Gzip Settings ################
gzip on;
gzip_comp_level 4;
gzip_min_length 10240;
gzip_proxied expired no-cache no-store private auth;
gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript text/x-js application/soap+xml;
###################################################
gzip_disable "MSIE [1-6].";
set_real_ip_from 10.117.0.0/16;
real_ip_header X-Forwarded-For;
real_ip_recursive on;
}
location /nginx_status {
stub_status on;
access_log off;
}
}
}
I am using spring boot version of 2.5.0.
Also, during the issue, the CPU and memory usage are also below 10%.
I already have tried changing the number of worker processes and changing the reverse proxy timeouts. Tries increasing the keep alive timeouts and keep alive connections count.
2
Answers
I do not know the exact reason why this is happening but upgrading my Netty server version does sole the problem.
Does the error.log of Nginx explain the reason for 502? See if this helps you.
NGINX returning HTTP 502, but HTTP 200 in the logs