Enhance NGINX configuration with rate limiting, improved timeout settings, and backup server for unavailable services. Added logging for 429 status and refined DNS resolver settings. Updated proxy settings for admin API and pgAdmin4 with increased body size limits and error handling.

nginx/nginx.conf

@@ -6,6 +6,13 @@ events {
 }
 
 http {
+    # ## Track by client IP; 20MB ≈ ~1200 active IPs
+    limit_req_zone $binary_remote_addr zone=rl_zone:20m rate=50r/s;
+    # Return 429 instead of 503 when throttled
+    limit_req_status 429; # 429 is the HTTP status code for Too Many Requests
+    # Log 429s at warn (not error)
+    limit_req_log_level warn;
+
     geo $frontend_whitelist {
         default 1;
         127.0.0.1 1;
@@ -32,29 +39,47 @@ http {
     real_ip_header X-Forwarded-For;
     real_ip_recursive on;
 
-    resolver 127.0.0.11 valid=10s;
+    # DNS resolver configuration for better reliability
-    resolver_timeout 5s;
+    resolver 127.0.0.11 valid=60s ipv6=off;
+    resolver_timeout 60s;
 
     upstream phoenix_system_cluster {
         zone phoenix_system_cluster 64k;
         least_conn;
-        server phoenix-system:3000 resolve fail_timeout=1s max_fails=0;
+        server phoenix-system:3000 resolve fail_timeout=60s max_fails=10;
+        server 127.0.0.1:81 backup;  # Backup server for unavailable service
         # ADD_SYSTEM_SERVERS_HERE
     }
 
     upstream phoenix_worker_cluster {
         zone phoenix_worker_cluster 64k;
         least_conn;
-        server phoenix-worker:3001 resolve fail_timeout=1s max_fails=0;
+        server phoenix-worker:3001 resolve fail_timeout=60s max_fails=10;
+        server 127.0.0.1:81 backup;  # Backup server for unavailable service
         # ADD_WORKER_SERVERS_HERE
     }
 
+    upstream pgadmin4-ui {
+        zone pgadmin4-ui 64k;
+        least_conn;
+        server pgadmin4-ui:80 resolve fail_timeout=120s max_fails=20;
+        server 127.0.0.1:81 backup;  # Backup server for unavailable service
+        # ADD_PGADMIN4_SERVERS_HERE
+    }
+
     server_tokens off; # Disable NGINX version tokens to avoid leaking NGINX version.
 
     # File handling & upload limits
     sendfile on;
     client_max_body_size 64m;
 
+    # Global proxy timeout settings (can be overridden per location)
+    proxy_connect_timeout 30s;
+    proxy_send_timeout 30s;
+    proxy_read_timeout 30s;
+    proxy_next_upstream_timeout 30s;
+    proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+
     # Prevent warning when setting many proxy headers, like we do
     proxy_headers_hash_max_size 1024;
     proxy_headers_hash_bucket_size 128;
@@ -84,9 +109,9 @@ http {
         ~^/health/worker     0;
     }
 
-    log_format main_with_realip '$remote_addr - $realip_remote_addr [$time_local] '
+    # log_format main_with_realip '$remote_addr - $realip_remote_addr [$time_local] '
-                                '"$request" $status $body_bytes_sent '
+    #                             '"$request" $status $body_bytes_sent '
-                                '"$http_referer" "$http_user_agent"';
+    #                             '"$http_referer" "$http_user_agent"';
 
     log_format json_compatible escape=json '{'
         '"time":"$time_iso8601",'
@@ -105,10 +130,43 @@ http {
         '"realip":"$realip_remote_addr"'
     '}';
 
-    access_log /var/log/nginx/access_json.log json_compatible if=$loggable; # JSON format for Loki
+    access_log /var/log/nginx/access_json.log json_compatible if=$loggable; # JSON format for Loki/Grafana/Prometheus/Fail2ban
-    access_log /var/log/nginx/access.log main_with_realip if=$loggable;
+    # access_log /var/log/nginx/access.log main_with_realip if=$loggable;
     # End of logs
 
+    ##################################################################
+    # 🔧 Backup Server for Unavailable Services
+    ##################################################################
+    server {
+        listen 127.0.0.1:81;
+        server_name _;
+
+        limit_req zone=rl_zone burst=30 nodelay;
+        
+        # Return service unavailable for health checks
+        location /health {
+            add_header Content-Type application/json always;
+            return 503 '{"status":"unavailable","message":"Service is currently down"}';
+        }
+        
+        # Return service unavailable for all other requests
+        location / {
+            add_header Content-Type text/html always;
+            return 503 '<html><body><h1>Service Temporarily Unavailable</h1><p>The requested service is currently down for maintenance.</p></body></html>';
+        }
+
+        # Return service unavailable for pgAdmin4
+        location /pgadmin4 {
+            add_header Content-Type text/html always;
+            return 503 '<html><body><h1>Service Temporarily Unavailable</h1><p>The requested service is currently down for maintenance.</p></body></html>';
+        }
+
+        location /health/worker {
+            add_header Content-Type application/json always;
+            return 503 '{"status":"unavailable","message":"Service is currently down"}';
+        }
+    }
+
     ##################################################################
     # 🧩 HTTP Server Block
     ##################################################################
@@ -116,6 +174,8 @@ http {
         listen 80;
         server_name _;
 
+        limit_req zone=rl_zone burst=30 nodelay;
+
         # Security headers
         add_header X-Frame-Options "SAMEORIGIN" always;
         add_header X-XSS-Protection "1; mode=block" always;
@@ -125,7 +185,21 @@ http {
         root /usr/share/nginx/html;
         index index.html index.htm;
 
-        # Frontend SPA fallback
+        # --- Return real 404 for common bot targets (must be before SPA fallback) ---
+        # Block direct file probes like .php, .env, .git, backups, etc.
+        location ~* \.(php|env|git|sql|bak|ini|config|swp|old|backup)$ {
+            return 404;
+        }
+
+        # Block well-known bad paths used by scanners
+        location ~* ^/(wp-admin|wp-login\.php|xmlrpc\.php|phpinfo\.php|vendor/phpunit|setup\.php|manager/html|id\.php|shell\.php|\.DS_Store) {
+            return 404;
+        }
+
+        # Test for Fail2ban
+        # location = /__f2b_test_404__ { return 404; }
+
+        # Frontend SPA fallback (keep this AFTER the blocks above)
         location / {
             try_files $uri $uri/ /index.html;
         }
@@ -140,14 +214,20 @@ http {
             proxy_set_header X-Forwarded-Host $host;
             proxy_set_header X-Forwarded-Proto $forwarded_proto;
             # End of headers
-            # Increase timeout settings for file uploads
+            client_max_body_size 500m; #  Increase the body size limit for admin API requests to 500mb
-            proxy_connect_timeout 600;
+            # Increase timeout settings for file uploads  
-            proxy_send_timeout 600;
+            proxy_connect_timeout 1800; # 30 minutes for regular API with file uploads
-            proxy_read_timeout 600;
+            proxy_send_timeout 1800; # 30 minutes to send request to backend
-            send_timeout 600;
+            proxy_read_timeout 1800; # 30 minutes to read response from backend
+            send_timeout 1800; # 30 minutes to send response to client
         }
 
         location /admin-api {
+            client_max_body_size 1024m; #  Increase the body size limit for admin API requests to 1gb
+            proxy_connect_timeout 3600; # 60 minutes to establish connection to backend
+            proxy_send_timeout 3600; # 60 minutes to send request to backend  
+            proxy_read_timeout 3600; # 60 minutes to read response from backend
+            send_timeout 3600; # 60 minutes to send response to client
             proxy_pass http://phoenix_system_cluster/admin-api;
             # Include headers for proxying
             proxy_set_header Host $host;
@@ -196,7 +276,36 @@ http {
         }
 
         # Reverse proxy for pgAdmin (subpath support)
-        include /etc/nginx/includes/*.conf;
+        # include /etc/nginx/includes/*.conf;
+        location /pgadmin4 {
+            error_log /var/log/nginx/pgadmin4_error.log notice;
+            
+            proxy_pass http://pgadmin4-ui/;
+            proxy_set_header X-Script-Name /pgadmin4;
+            proxy_set_header X-Scheme $scheme;
+            proxy_set_header X-Forwarded-Proto $forwarded_proto;
+
+            # Improved timeout settings
+            proxy_connect_timeout 120s;
+            proxy_send_timeout 120s;
+            proxy_read_timeout 120s;
+
+             # Retry on errors
+            proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+            proxy_next_upstream_tries 5;
+            proxy_next_upstream_timeout 120s;
+
+            # Include headers for proxying
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Host $host;
+            # End of headers
+            proxy_redirect off;
+
+            # ⚠️ Rewrite required to remove /pgadmin4 from the path
+            rewrite ^/pgadmin4(/.*)$ $1 break;
+        }
 
         # Health check endpoints -> used by the health check exporter
         location /health/system {
@@ -212,6 +321,10 @@ http {
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Proto $forwarded_proto;
             # End of headers
+            # Timeout settings
+            proxy_connect_timeout 30s;
+            proxy_send_timeout 30s;
+            proxy_read_timeout 30s;
         }
 
         # location /health/system/metrics {
@@ -242,6 +355,10 @@ http {
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
             proxy_set_header X-Forwarded-Proto $forwarded_proto;
             # End of headers
+            # Timeout settings
+            proxy_connect_timeout 30s;
+            proxy_send_timeout 30s;
+            proxy_read_timeout 30s;
         }
 
         # location /health/worker/metrics {
@@ -277,6 +394,9 @@ http {
         http2 on;
         server_name _;
 
+        # Apply globally inside this server
+        limit_req zone=rl_zone burst=30 nodelay;
+
         # Security headers
         add_header X-Frame-Options "SAMEORIGIN" always;
         add_header X-XSS-Protection "1; mode=block" always;
@@ -303,14 +423,20 @@ http {
             proxy_set_header X-Forwarded-Host $host;
             proxy_set_header X-Forwarded-Proto $forwarded_proto;
             # End of headers
-            # Increase timeout settings for file uploads
+            client_max_body_size 500m; #  Increase the body size limit for admin API requests to 500mb
-            proxy_connect_timeout 600;
+            # Increase timeout settings for file uploads  
-            proxy_send_timeout 600;
+            proxy_connect_timeout 1800; # 30 minutes for regular API with file uploads
-            proxy_read_timeout 600;
+            proxy_send_timeout 1800; # 30 minutes to send request to backend
-            send_timeout 600;
+            proxy_read_timeout 1800; # 30 minutes to read response from backend
+            send_timeout 1800; # 30 minutes to send response to client
         }
 
         location /admin-api {
+            client_max_body_size 1024m; #  Increase the body size limit for admin API requests to 1gb
+            proxy_connect_timeout 3600; # 60 minutes to establish connection to backend
+            proxy_send_timeout 3600; # 60 minutes to send request to backend  
+            proxy_read_timeout 3600; # 60 minutes to read response from backend
+            send_timeout 3600; # 60 minutes to send response to client
             proxy_pass http://phoenix_system_cluster/admin-api;
             # Include headers for proxying
             proxy_set_header Host $host;