Update Fail2ban configuration for nginx: increase maxretry from 20 to 50 in nginx-4xx jail to enhance security measures against repeated offenses.

.gitignore

@@ -1,4 +1,4 @@
-fail2ban
+fail2ban/data
 https_portal
 logs
 database

docker-compose.yaml

@@ -86,7 +86,8 @@ services:
       retries: 5  # Grace period before health checks start
   pgadmin:
     restart: always
-    image: dpage/pgadmin4:9.6.0
+    image: dpage/pgadmin4:9.8.0
+    pull_policy: always
     container_name: pgadmin4-ui
     user: "5050:5050"
     # logging:
@@ -159,11 +160,11 @@ services:
       postgres:
         condition: service_healthy
     healthcheck:
-      test: ["CMD", "wget", "-O", "-", "http://localhost:80/misc/ping"]
+      test: ["CMD-SHELL", "wget -qO- http://localhost:80/misc/ping && wget -qO- http://localhost:80/browser/"]
       interval: 15s
       timeout: 10s
       retries: 5
-      start_period: 120s
+      start_period: 150s
   phoenix-app:
     restart: always
     image: "phxerp/phoenix-app:alpha"
@@ -322,10 +323,8 @@ services:
       
       # - "./logs:/usr/src/app/packages/dev-server/logs"
   phoenix-redis:
-    image: 'bitnami/redis:latest'
+    image: redis:7.2-alpine
     container_name: redis
-    command: /opt/bitnami/scripts/redis/run.sh # Not good, but as agreed. At some point i can start using this: --maxmemory + add eviction policy
-    user: root
     # logging:
     #   driver: loki
     #   options:
@@ -337,8 +336,6 @@ services:
       - backend
     restart: always
     environment:
-      ALLOW_EMPTY_PASSWORD: "no"
-      REDIS_PASSWORD: ${REDIS_PASSWORD}
       TZ: Europe/Berlin
     # deploy:
     #   restart_policy: # Define how the service should restart when it fails
@@ -353,10 +350,13 @@ services:
     #     reservations: # Guaranteed minimum resources for the container
     #       cpus: "0.05" # Reserved CPU cores (5% of one core)
     #       memory: 32M # Reserved memory (32 megabytes)
+    command: >
+      redis-server
+      --requirepass ${REDIS_PASSWORD}
     healthcheck:
       test: [
         "CMD-SHELL",
-        "redis-cli --no-auth-warning -a ${REDIS_PASSWORD} ping | grep PONG && test -w /bitnami/redis/data"
+        "redis-cli -a ${REDIS_PASSWORD} ping | grep PONG && test -w /data"
       ]
       interval: 5s
       retries: 10
@@ -365,7 +365,7 @@ services:
       postgres:
         condition: service_healthy
     volumes:
-      - "./redis/data:/bitnami/redis/data"
+      - ./redis/data:/data
   phoenix-health-exporter:
     image: phxerp/phoenix-health-exporter:alpha
     container_name: health_exporter

docker-compose.yaml.1140915.2025-09-21@07:55:01~

@@ -0,0 +1,596 @@
+---
+services:
+  postgres-auto-upgrade:
+    profiles:
+        - postgres-upgrade # 🟢 This isolates the service
+    image: alpine:3.19
+    container_name: postgres_auto_upgrade
+    working_dir: /opt/phx
+    environment:
+      TZ: Europe/Berlin
+    volumes:
+        - .:/opt/phx:rw
+        - /var/run/docker.sock:/var/run/docker.sock
+    entrypoint: >
+        sh -c "
+          apk add --no-cache bash coreutils grep sed findutils curl docker-cli dos2unix &&
+          mkdir -p ~/.docker/cli-plugins &&
+          curl -SL https://github.com/docker/compose/releases/download/v2.27.0/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose &&
+          chmod +x ~/.docker/cli-plugins/docker-compose &&
+          chmod +x ./postgres_upgrade.sh &&
+          ./postgres_upgrade.sh"
+    restart: 'no'
+    depends_on: []
+    network_mode: bridge
+  postgres-auto-rollback:
+    profiles:
+        - postgres-rollback # 🟢 This isolates the service
+    image: alpine:3.19
+    container_name: postgres_rollback
+    working_dir: /opt/phx
+    environment:
+      TZ: Europe/Berlin
+    volumes:
+        - .:/opt/phx:rw
+        - /var/run/docker.sock:/var/run/docker.sock
+    entrypoint: >
+        sh -c "
+          apk add --no-cache bash coreutils grep sed findutils curl docker-cli dos2unix &&
+          mkdir -p ~/.docker/cli-plugins &&
+          curl -SL https://github.com/docker/compose/releases/download/v2.27.0/docker-compose-linux-x86_64 -o ~/.docker/cli-plugins/docker-compose &&
+          chmod +x ~/.docker/cli-plugins/docker-compose &&
+          chmod +x ./rollback_postgres_upgrade.sh &&
+          ./rollback_postgres_upgrade.sh"
+    restart: 'no'
+    depends_on: []
+    network_mode: bridge
+  postgres:
+    restart: always
+    image: "postgres:17.5-alpine"
+    container_name: phoenixDB # Hostname
+    # logging:
+    #   driver: loki
+    #   options:
+    #     loki-url: "${LOKI_URL}"
+    #     loki-retries: "${LOKI_RETRIES}"
+    #     loki-batch-size: "${LOKI_BATCH_SIZE}"
+    #     loki-external-labels: "service=phx-postgres,env=prod"
+    networks:
+      - backend
+    security_opt:
+      - no-new-privileges:true
+    environment:
+      DEBUG: true
+      POSTGRES_DB: ${DB_NAME}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      TZ: Europe/Berlin
+    volumes:
+      - "./database:/var/lib/postgresql/data"
+    # deploy:
+    #   restart_policy: # Define how the service should restart when it fails
+    #     condition: on-failure # Only restart if the container exits with a non-zero code
+    #     delay: 5s # Wait 5 seconds before attempting to restart
+    #     max_attempts: 5 # Maximum number of restart attempts before giving up
+    #     window: 120s # Time window to evaluate restart attempts (resets counter after this period)
+    #   resources: # Resource allocation and limits for the container
+    #     limits: # Maximum resources the container can use
+    #       cpus: "0.75" # Maximum CPU cores (75% of one core)
+    #       memory: 768M # Maximum memory usage (768 megabytes)
+    #     reservations: # Guaranteed minimum resources for the container
+    #       cpus: "0.25" # Reserved CPU cores (25% of one core)
+    #       memory: 256M # Reserved memory (256 megabytes)
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      interval: 5s # Time between each health check
+      timeout: 2s # Number of failures before marking as unhealthy
+      retries: 5  # Grace period before health checks start
+  pgadmin:
+    restart: always
+    image: dpage/pgadmin4:9.8.0
+    container_name: pgadmin4-ui
+    user: "5050:5050"
+    # logging:
+    #   driver: loki
+    #   options:
+    #     loki-url: "${LOKI_URL}"
+    #     loki-retries: "${LOKI_RETRIES}"
+    #     loki-batch-size: "${LOKI_BATCH_SIZE}"
+    #     loki-external-labels: "service=phx-pgadmin,env=prod"
+    networks:
+      - backend
+      - frontend
+    environment:
+      PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL}
+      PGADMIN_DEFAULT_PASSWORD: ${SUPER_ADMIN_USER_PASSWORD}
+      PGADMIN_CONFIG_SERVER_MODE: 'True'
+      PGADMIN_CONFIG_WSGI_SCRIPT_NAME: "'/pgadmin4'"
+      PGADMIN_CONFIG_PROXY_X_PROTO_COUNT: 1
+      PGADMIN_SERVER_JSON_FILE: '/var/lib/pgadmin/servers.json'
+      PGADMIN_REPLACE_SERVERS_ON_STARTUP: 'True'
+      PGADMIN_CONFIG_DATA_DIR: "'/var/lib/pgadmin'"
+      PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED: 'False'
+
+      # pgpass dynamic vars
+      PGPASSFILE: /var/lib/pgadmin/pgpass
+      PGPASS_HOST: ${DB_HOST}
+      PGPASS_PORT: ${DB_PORT}
+      PGPASS_DB: ${DB_NAME}
+      PGPASS_USER: ${DB_USERNAME}
+      PGPASS_PASSWORD: ${POSTGRES_PASSWORD}
+
+      # Other config
+      ALLOW_SAVE_PASSWORD: 'False'
+      MFA_ENABLED: 'True'
+      MFA_FORCE_REGISTRATION: 'False'
+      MFA_SUPPORTED_METHODS: 'email'
+      MFA_EMAIL_SUBJECT: 'Your MFA code by PHX-ERP'
+      MAX_LOGIN_ATTEMPTS: 5
+      ENHANCED_COOKIE_PROTECTION: 'True'
+      SHOW_GRAVATAR_IMAGE: 'True'
+      SECURITY_EMAIL_SENDER: ${SECURITY_EMAIL_SENDER}
+      MAIL_SERVER: ${MAIL_SERVER}
+      MAIL_PORT: ${MAIL_PORT}
+      MAIL_USE_SSL: 'False'
+      MAIL_USE_TLS: 'False'
+      MAIL_USERNAME: ${MAIL_USERNAME}
+      MAIL_PASSWORD: ${MAIL_PASSWORD}
+      MAIL_DEBUG: 'False'
+      TZ: Europe/Berlin
+    volumes:
+      - ./pgadmin/data:/var/lib/pgadmin
+      - ./pgadmin/pgadmin-entrypoint.sh:/docker-entrypoint.sh:ro
+    mem_limit: 512M
+    memswap_limit: 512M
+    deploy:
+      restart_policy: # Define how the service should restart when it fails
+        condition: on-failure # Only restart if the container exits with a non-zero code
+        delay: 5s # Wait 5 seconds before attempting to restart
+        max_attempts: 5 # Maximum number of restart attempts before giving up
+        window: 120s # Time window to evaluate restart attempts (resets counter after this period)
+      resources: # Resource allocation and limits for the container
+        limits: # Maximum resources the container can use
+          cpus: "1.0" # Maximum CPU cores (100% of one core)
+          memory: 512M # Maximum memory usage (512 megabytes)
+        reservations: # Guaranteed minimum resources for the container
+          cpus: "0.15" # Reserved CPU cores (15% of one core)
+          memory: 250M # Reserved memory (250 megabytes)
+    entrypoint: ["/bin/sh", "/docker-entrypoint.sh"]
+    depends_on:
+      postgres:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://localhost:80/misc/ping && wget -qO- http://localhost:80/browser/"]
+      interval: 15s
+      timeout: 10s
+      retries: 5
+      start_period: 150s
+  phoenix-app:
+    restart: always
+    image: "phxerp/phoenix-app:alpha"
+    container_name: phoenix-app
+    ports:
+      - "3000:3000" # Restrict to only allow access from Grafana Server IP
+    # logging:
+    #   driver: loki
+    #   options:
+    #     loki-url: "${LOKI_URL}"
+    #     loki-retries: "${LOKI_RETRIES}"
+    #     loki-batch-size: "${LOKI_BATCH_SIZE}"
+    #     loki-external-labels: "service=phx-app,env=prod,project=phoenix"
+    environment:
+      TZ: Europe/Berlin
+    volumes:
+      - ./app_custom:/usr/share/nginx/html/assets/custom
+      - nginx-logs:/var/log/nginx # this is part of the fail2ban to avoid rotation logs cleaning setup. This will be done by docker volume rm nginx-logs
+      # - ./phoenix-app/logs:/var/log/nginx # this is part of the fail2ban config to make analysis of the logs easier
+      # - ./nginx/nginx.conf:/etc/nginx/nginx.conf # Uncomment this if you want to use override the default nginx.conf
+      # - ./nginx/includes:/etc/nginx/includes:ro # Uncomment this if you want to use override the default includes
+    networks:
+      - backend
+      - frontend
+    # deploy:
+    #   restart_policy: # Define how the service should restart when it fails
+    #     condition: on-failure # Only restart if the container exits with a non-zero code
+    #     delay: 5s # Wait 5 seconds before attempting to restart
+    #     max_attempts: 5 # Maximum number of restart attempts before giving up
+    #     window: 120s # Time window to evaluate restart attempts (resets counter after this period)
+    #   resources: # Resource allocation and limits for the container
+    #     limits: # Maximum resources the container can use
+    #       cpus: "0.35" # Maximum CPU cores (35% of one core)
+    #       memory: 384M # Maximum memory usage (384 megabytes)
+    #     reservations: # Guaranteed minimum resources for the container
+    #       cpus: "0.10" # Reserved CPU cores (10% of one core)
+    #       memory: 128M # Reserved memory (128 megabytes)
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://phoenix-app/login"] # localhost checks that the NGINX server inside the container is serving something at the root 
+      interval: 10s       # check every 10 seconds
+      timeout: 5s         # allow 5 seconds per check
+      retries: 5          # mark as unhealthy after 5 failures
+      start_period: 15s   # wait 15 seconds after container start before checking
+  phoenix-system:
+    restart: always
+    image: "phxerp/phoenix-system:alpha"
+    # logging:
+    #   driver: loki
+    #   options:
+    #     loki-url: "${LOKI_URL}"
+    #     loki-retries: "${LOKI_RETRIES}"
+    #     loki-batch-size: "${LOKI_BATCH_SIZE}"
+    #     loki-external-labels: "service=phoenix-system,env=prod"
+    environment:
+      - "DB_HOST=${DB_HOST}"
+      - "DB_NAME=${DB_NAME}"
+      - "DB_PASSWORD=${POSTGRES_PASSWORD}"
+      - "DB_USERNAME=${DB_USERNAME}"
+      - "SUPER_ADMIN_USER_PASSWORD=${SUPER_ADMIN_USER_PASSWORD}"
+      - "REDIS_PASSWORD=${REDIS_PASSWORD}"
+      - NODE_ENV=${NODE_ENV}
+      - PHX_HOST_NAME=${PHX_HOST_NAME}
+      - PERFORMANCE_STRUCTURED_LOGGING=${PERFORMANCE_STRUCTURED_LOGGING}
+      - PERFORMANCE_WARNING_THRESHOLD=${PERFORMANCE_WARNING_THRESHOLD}
+      - PERFORMANCE_DETAILED_MEMORY=${PERFORMANCE_DETAILED_MEMORY}
+      - TZ=Europe/Berlin
+    command: ["npm", "run", "start:server"]
+    deploy:
+      replicas: ${PHOENIX_SYSTEM_REPLICAS} #change here if u want to have more replicas. Cant find a way to set via variable right now
+    #   restart_policy: # Define how the service should restart when it fails
+    #     condition: on-failure # Only restart if the container exits with a non-zero code
+    #     delay: 5s # Wait 5 seconds before attempting to restart
+    #     max_attempts: 5 # Maximum number of restart attempts before giving up
+    #     window: 120s # Time window to evaluate restart attempts (resets counter after this period)
+    #   resources: # Resource allocation and limits for the container
+    #     limits: # Maximum resources the container can use
+    #       cpus: "1.50" # Maximum CPU cores (150% of one core)
+    #       memory: 1600M # Maximum memory usage (1600 megabytes)
+    #     reservations: # Guaranteed minimum resources for the container
+    #       cpus: "0.50" # Reserved CPU cores (50% of one core)
+    #       memory: 768M # Reserved memory (768 megabytes)
+    networks:
+      backend:
+        aliases:
+          - phoenix-system
+    depends_on:
+      postgres:
+        condition: service_healthy
+      phoenix-redis:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", "curl -s http://phoenix-system:3000/health | grep -q '\"admin-api\":{\"status\":\"up\"}' && curl -s http://phoenix-system:3000/health | grep -q '\"database\":{\"status\":\"up\"}'"] # Checks both admin-api and database status
+      interval: 10s # Time between each health check
+      timeout: 10s # Max time to wait for each check
+      retries: 20 # Number of failures before marking as unhealthy
+      start_period: 60s  # Grace period before health checks start
+    volumes:
+      - "./assets:/usr/src/app/packages/dev-server/assets"
+      
+      # - "./logs:/usr/src/app/packages/dev-server/logs"
+  phoenix-worker:
+    restart: always
+    image: "phxerp/phoenix-system:alpha"
+    container_name: "phoenix-worker"
+    ports:
+      - "3001:3001" # Restrict to only allow access from Grafana Server IP
+    # logging:
+    #   driver: loki
+    #   options:
+    #     loki-url: "${LOKI_URL}"
+    #     loki-retries: "${LOKI_RETRIES}"
+    #     loki-batch-size: "${LOKI_BATCH_SIZE}"
+    #     loki-external-labels: "service=phx-worker,env=prod"
+    networks:
+      - backend
+    environment:
+      - TZ=Europe/Berlin
+      - "DB_HOST=${DB_HOST}"
+      - "DB_NAME=${DB_NAME}"
+      - "DB_PASSWORD=${POSTGRES_PASSWORD}"
+      - "DB_USERNAME=${DB_USERNAME}"
+      - "SUPER_ADMIN_USER_PASSWORD=${SUPER_ADMIN_USER_PASSWORD}"
+      - REDIS_PASSWORD=${REDIS_PASSWORD}
+      - NODE_ENV=${NODE_ENV}
+      - PHX_HOST_NAME=${PHX_HOST_NAME}
+      - PERFORMANCE_STRUCTURED_LOGGING=${PERFORMANCE_STRUCTURED_LOGGING}
+      - PERFORMANCE_WARNING_THRESHOLD=${PERFORMANCE_WARNING_THRESHOLD}
+      - PERFORMANCE_DETAILED_MEMORY=${PERFORMANCE_DETAILED_MEMORY}
+    command: ['npm', 'run', 'start:worker']
+    # deploy:
+    #   restart_policy: # Define how the service should restart when it fails
+    #     condition: on-failure # Only restart if the container exits with a non-zero code
+    #     delay: 5s # Wait 5 seconds before attempting to restart
+    #     max_attempts: 5 # Maximum number of restart attempts before giving up
+    #     window: 120s # Time window to evaluate restart attempts (resets counter after this period)
+    #   resources: # Resource allocation and limits for the container
+    #     limits: # Maximum resources the container can use
+    #       cpus: '2.0' # Maximum CPU cores (200% of one core)
+    #       memory: 2G # Maximum memory usage (2 gigabytes)
+    #     reservations: # Guaranteed minimum resources for the container
+    #       cpus: '0.5' # Reserved CPU cores (50% of one core)
+    #       memory: 512M # Reserved memory (512 megabytes)
+    depends_on:
+      phoenix-system:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+    healthcheck:
+      test: [ "CMD-SHELL", "curl -s http://phoenix-worker:3001/health | grep -q '\"status\":\"ok\"'" ] # Check if worker responds with status ok
+      interval: 10s   # Time between each health check
+      timeout: 6s     # Max time to wait for each check
+      retries: 20     # Grace period before health checks start
+      start_period: 30s  # Grace period before health checks start
+    volumes:
+      - "./assets:/usr/src/app/packages/dev-server/assets"
+      
+      # - "./logs:/usr/src/app/packages/dev-server/logs"
+  phoenix-redis:
+    image: redis:7.2-alpine
+    container_name: redis
+    restart: always
+    networks:
+      - backend
+    command: >
+      redis-server
+      --requirepass ${REDIS_PASSWORD}
+    volumes:
+      - ./redis/data:/data
+    depends_on:
+      postgres:
+        condition: service_healthy
+    healthcheck:
+      test: [
+        "CMD-SHELL",
+        "redis-cli -a ${REDIS_PASSWORD} ping | grep PONG && test -w /data"
+      ]
+      interval: 5s
+      retries: 10
+      timeout: 5s
+  phoenix-health-exporter:
+    image: phxerp/phoenix-health-exporter:alpha
+    container_name: health_exporter
+    restart: unless-stopped
+    # logging:
+    #   driver: loki
+    #   options:
+    #     loki-url: "${LOKI_URL}"
+    #     loki-retries: "${LOKI_RETRIES}"
+    #     loki-batch-size: "${LOKI_BATCH_SIZE}"
+    #     loki-external-labels: "service=phx-health-exporter,env=prod"
+    ports:
+      - "9800:9800"
+    environment:
+      DB_HOST: ${DB_HOST}
+      DB_NAME: ${DB_NAME}
+      DB_PASSWORD: ${POSTGRES_PASSWORD}
+      DB_USERNAME: ${DB_USERNAME}
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+      TZ: Europe/Berlin
+    networks:
+      - frontend
+      - backend
+    volumes:
+      - /etc/hostname:/etc/host_hostname:ro # This ensures the container always uses the real machine hostname, even if restarted or recreated.
+    security_opt:
+      - no-new-privileges:true
+    memswap_limit: 512M
+    deploy:
+      restart_policy: # Define how the service should restart when it fails
+        condition: on-failure # Only restart if the container exits with a non-zero code
+        delay: 5s # Wait 5 seconds before attempting to restart
+        max_attempts: 5 # Maximum number of restart attempts before giving up
+        window: 120s # Time window to evaluate restart attempts (resets counter after this period)
+      resources: # Resource allocation and limits for the container
+        limits: # Maximum resources the container can use
+          cpus: "0.5" # Maximum CPU cores (50% of one core)
+          memory: 256M # Maximum memory usage (256 megabytes)
+        reservations: # Guaranteed minimum resources for the container
+          cpus: "0.1" # Reserved CPU cores (10% of one core)
+          memory: 64M # Reserved memory (64 megabytes)
+    depends_on:
+      phoenix-system:
+        condition: service_healthy
+      phoenix-worker:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", "curl -sf http://localhost:9800/healthz || exit 1"]
+      interval: 1m
+      timeout: 5s
+      retries: 3
+      start_period: 15s
+  node-exporter:
+    image: quay.io/prometheus/node-exporter:latest
+    container_name: node_exporter
+    # logging:
+    #   driver: loki
+    #   options:
+    #     loki-url: "${LOKI_URL}"
+    #     loki-retries: "${LOKI_RETRIES}"
+    #     loki-batch-size: "${LOKI_BATCH_SIZE}"
+    #     loki-external-labels: "service=phx-node-exporter,env=prod"
+    networks:
+      - metrics
+      - frontend
+    restart: unless-stopped
+    environment:
+      TZ: Europe/Berlin
+    ports:
+      - "9100:9100"  # Restrict to only allow access from Grafana Server IP
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.sysfs=/host/sys"
+      - "--path.rootfs=/host"
+      - "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev)($$|/)"
+    volumes:
+      - "/proc:/host/proc:ro"
+      - "/sys:/host/sys:ro"
+      - "/:/host:ro,rslave"
+    security_opt:
+      - no-new-privileges:true
+    memswap_limit: 512M
+    deploy:
+      restart_policy: # Define how the service should restart when it fails
+        condition: on-failure # Only restart if the container exits with a non-zero code
+        delay: 5s # Wait 5 seconds before attempting to restart
+        max_attempts: 5 # Maximum number of restart attempts before giving up
+        window: 120s # Time window to evaluate restart attempts (resets counter after this period)
+      resources: # Resource allocation and limits for the container
+        limits: # Maximum resources the container can use
+          cpus: "0.25" # Maximum CPU cores (25% of one core)
+          memory: 128M # Maximum memory usage (128 megabytes)
+        reservations: # Guaranteed minimum resources for the container
+          cpus: "0.05" # Reserved CPU cores (5% of one core)
+          memory: 32M # Reserved memory (32 megabytes)
+    depends_on:
+      phoenix-worker: # This is to avoid alocation of resources to the node-exporter if the phoenix-worker is not healthy yet.
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:9100/metrics"]
+      interval: 15s
+      timeout: 5s
+      retries: 3
+      start_period: 20s
+  # nginx-exporter:
+  #   image: nginx/nginx-prometheus-exporter:1.4.2
+  #   container_name: nginx_exporter
+  #   restart: unless-stopped
+  #   # logging:
+  #   #   driver: loki
+  #   #   options:
+  #   #     loki-url: "${LOKI_URL}"
+  #   #     loki-retries: "${LOKI_RETRIES}"
+  #   #     loki-batch-size: "${LOKI_BATCH_SIZE}"
+  #   #     loki-external-labels: "service=phx-nginx-exporter,env=prod"
+  #   ports:
+  #     - "9113:9113" # Restrict to only allow access from Grafana Server IP
+  #   command:
+  #     - '--nginx.scrape-uri=http://phoenix-app/stub_status'
+  #   security_opt:
+  #     - no-new-privileges:true
+  #   deploy:
+  #     resources:
+  #       limits:
+  #           cpus: '0.25'
+  #           memory: 128M
+  #   depends_on:
+  #     phoenix-app:
+  #       condition: service_healthy
+  #   networks:
+  #     - frontend
+  #     - metrics
+    # healthcheck:
+    #   test: ["CMD", "wget", "-qO-", "http://localhost:9113/metrics"] # Not working as expected
+    #   interval: 15s
+    #   timeout: 5s
+    #   retries: 3
+    #   start_period: 10s
+  https_portal:
+    container_name: https_portal
+    image: "steveltn/https-portal:1.21"
+    restart: unless-stopped
+    # logging:
+    #   driver: loki
+    #   options:
+    #     loki-url: "${LOKI_URL}"
+    #     loki-retries: "${LOKI_RETRIES}"
+    #     loki-batch-size: "${LOKI_BATCH_SIZE}"
+    #     loki-external-labels: "service=phx-https-portal,env=prod"
+    networks:
+      - frontend # [ PgAdmin, Phoenix-App ]
+    ports:
+      - "80:80"
+      - "443:443"
+      # - host:container
+    environment:
+      STAGE: "production" # Use Let's Encrypt production server
+      WEBSOCKET: "true" # Enable websocket support
+      DEBUG: "true"
+      RENEW_MARGIN_DAYS: 30
+      CLIENT_MAX_BODY_SIZE: 0
+      SERVER_NAMES_HASH_BUCKET_SIZE: 128 # Increase hash bucket size for server names - good for bigger domains names, if not set correctly, it will throw an error, break the container.
+      # FORCE_RENEW: 'true'
+      DOMAINS: "${HTTPS_PORTAL_DOMAINS}"
+      TZ: Europe/Berlin
+    volumes:
+      - ./https_portal/data:/var/lib/https-portal # ssl_certs, vhost.d, htdocs
+      - ./https_portal/log:/var/log/nginx # nginx logs
+      # - ./https_portal/config/custom_nginx.conf:/opt/custom_nginx.conf:ro  # ✅ Mount file in a safe path
+    memswap_limit: 512M
+    deploy:
+      restart_policy: # Define how the service should restart when it fails
+        condition: on-failure # Only restart if the container exits with a non-zero code
+        delay: 5s # Wait 5 seconds before attempting to restart
+        max_attempts: 5 # Maximum number of restart attempts before giving up
+        window: 120s # Time window to evaluate restart attempts (resets counter after this period)
+      resources: # Resource allocation and limits for the container
+        limits: # Maximum resources the container can use
+          cpus: "0.5" # Maximum CPU cores (50% of one core)
+          memory: 256M # Maximum memory usage (256 megabytes)
+        reservations: # Guaranteed minimum resources for the container
+          cpus: "0.1" # Reserved CPU cores (10% of one core)
+          memory: 64M # Reserved memory (64 megabytes)
+    depends_on:
+      # pgadmin:
+      #   condition: service_healthy
+      postgres:
+        condition: service_healthy
+  fail2ban:
+    restart: always
+    image: crazymax/fail2ban:latest
+    container_name: fail2ban
+    network_mode: host # important: act on host network
+    cap_add:
+        - NET_ADMIN # needed to manage firewall
+        - NET_RAW
+    environment:
+      TZ: Europe/Berlin
+      F2B_LOG_LEVEL: WARNING
+    volumes:
+        - ./fail2ban/data:/data
+        - ./fail2ban/jail.d:/etc/fail2ban/jail.d
+        - ./fail2ban/filter.d:/data/filter.d
+        - /var/log:/var/log:ro # Parse host logs to the sshd
+        - nginx-logs:/data/nginx-logs:ro
+        # - ./phoenix-app/logs:/logs/phoenix-app:ro # not needed anymore, but keep here for manual/testing purposes.
+    memswap_limit: 512M
+    deploy:
+      restart_policy: # Define how the service should restart when it fails
+        condition: on-failure # Only restart if the container exits with a non-zero code
+        delay: 5s # Wait 5 seconds before attempting to restart
+        max_attempts: 5 # Maximum number of restart attempts before giving up
+        window: 120s # Time window to evaluate restart attempts (resets counter after this period)
+      resources: # Resource allocation and limits for the container
+        limits: # Maximum resources the container can use
+          cpus: "0.5" # Maximum CPU cores (50% of one core)
+          memory: 100M # Maximum memory usage (100 megabytes)
+        reservations: # Guaranteed minimum resources for the container
+          cpus: "0.1" # Reserved CPU cores (10% of one core)
+          memory: 35M # Reserved memory (32 megabytes)
+    depends_on:
+      phoenix-worker: # This is to avoid alocation of resources to the fail2ban if the phoenix-worker is not healthy yet.
+        condition: service_healthy
+
+volumes:
+  nginx-logs:
+    name: nginx-logs
+
+networks:
+  backend:
+    driver: bridge
+    external: false
+    ipam:
+      config:
+        - subnet: 172.19.0.0/16
+
+  frontend:
+    driver: bridge
+    external: false
+    ipam:
+      config:
+        - subnet: 172.20.0.0/16
+  
+  metrics:
+    driver: bridge
+    external: false
+    ipam:
+      config:
+        - subnet: 172.22.0.0/16

fail2ban/filter.d/http-get-dos-compressed.conf

@@ -0,0 +1,8 @@
+[Definition]
+# Count any GET/POST from the same client IP. We prefer X-Forwarded-For
+# (real client IP) when present; otherwise fall back to remote_addr.
+failregex = ^.*"x_forwarded_for":"<HOST>(?:, [^"]+)?".*"request_method":"(?:GET|POST)".*$
+            ^.*"remote_addr":"<HOST>".*"request_method":"(?:GET|POST)".*$
+
+# Ignore safe/health endpoints (adjust to your env)
+ignoreregex = ^.*"request_uri":"\/(?:stub_status|health\/system|health\/worker|pgadmin4(?:\/|$)|\.well-known\/acme-challenge\/|.*\.(?:css|js|png|jpg|jpeg|gif|svg|ico|webp|woff2?))".*$

fail2ban/filter.d/http-get-dos.conf

@@ -0,0 +1,8 @@
+[Definition]
+# Count lots of requests from same IP regardless of status code.
+# Prefer X-Forwarded-For (client IP behind proxy), fallback to remote_addr.
+failregex = ^.*"x_forwarded_for":"<HOST>".*"(GET|POST|HEAD)".*$
+            ^.*"remote_addr":"<HOST>".*"(GET|POST|HEAD)".*$
+
+# Ignore health and stub_status hits
+ignoreregex = ^.*"request_uri":"\/(?:stub_status|health\/system|health\/worker|pgadmin4(?:\/|$)|\.well-known\/acme-challenge\/|.*\.(?:css|js|png|jpg|jpeg|gif|svg|ico|webp|woff2?))".*$

fail2ban/filter.d/nginx-429.conf

@@ -0,0 +1,6 @@
+[Definition]
+# JSON access log with x_forwarded_for preferred, else remote_addr
+failregex = ^.*"x_forwarded_for":"<HOST>(?:, [^"]+)?".*"status":429.*$
+            ^.*"remote_addr":"<HOST>".*"status":429.*$
+
+ignoreregex = ^.*"request_uri":"\/(?:stub_status|health\/system|health\/worker|pgadmin4(?:\/|$)|\.well-known\/acme-challenge\/|.*\.(?:css|js|png|jpg|jpeg|gif|svg|ico|webp|woff2?))".*$

fail2ban/filter.d/nginx-4xx.conf

@@ -0,0 +1,6 @@
+[Definition]
+# Match either x_forwarded_for (preferred) or remote_addr
+failregex = ^.*"x_forwarded_for":"<HOST>".*"status":4\d\d.*$
+            ^.*"remote_addr":"<HOST>".*"status":4\d\d.*$
+
+ignoreregex = ^.*"request_uri":"\/(?:stub_status|health\/system|health\/worker|pgadmin4(?:\/|$)|\.well-known\/acme-challenge\/|.*\.(?:css|js|png|jpg|jpeg|gif|svg|ico|webp|woff2?))".*$

fail2ban/filter.d/nginx-badbots.conf

@@ -0,0 +1,15 @@
+[Definition]
+# JSON logs with ISO-8601 timestamps
+datepattern = {^LN-BEG}%%Y-%%m-%%dT%%H:%%M:%%S(?:[.,]\\d+)?(?:Z|[+\\-]\\d{2}:\\d{2})?
+
+# Define a custom IP regex for <HOST>
+hostname = (?:\d{1,3}\.){3}\d{1,3}
+
+# Detect scanners in user-agent
+failregex = (?i)^.*"x_forwarded_for":"<HOST>(?:, [^"]+)?".*"http_user_agent":"[^"]*(?:sqlmap|nikto|acunetix|wpscan|dirbuster|gobuster|masscan|zgrab|ZmEu|nessus|openvas|libwww-perl|mechanize|lwp-trivial|python-requests|python-urllib|urllib|aiohttp|httpx|scrapy|curl|wget|Go-http-client|okhttp|httpclient|jakarta|java)[^"]*".*$
+            (?i)^.*"remote_addr":"<HOST>".*"http_user_agent":"[^"]*(?:sqlmap|nikto|acunetix|wpscan|dirbuster|gobuster|masscan|zgrab|ZmEu|nessus|openvas|libwww-perl|mechanize|lwp-trivial|python-requests|python-urllib|urllib|aiohttp|httpx|scrapy|curl|wget|Go-http-client|okhttp|httpclient|jakarta|java)[^"]*".*$
+
+# Ignore internal/harmless endpoints and broken IPs
+ignoreregex = ^.*"request_uri":"\/(?:stub_status|health\/system|health\/worker|pgadmin4(?:\/|$)|\.well-known\/acme-challenge\/|.*\.(?:css|js|png|jpg|jpeg|gif|svg|ico|webp|woff2?))".*$
+              ^.*"x_forwarded_for":"\:?".*$
+              ^.*"remote_addr":"\:?".*$

fail2ban/filter.d/nginx-botsearch.conf

@@ -0,0 +1,6 @@
+[Definition]
+datepattern = {^LN-BEG}%%Y-%%m-%%dT%%H:%%M:%%S(?:[.,]\\d+)?(?:Z|[+\\-]\\d{2}:\\d{2})?
+failregex = ^.*"x_forwarded_for":"<HOST>(?:, [^"]+)?".*"request_uri":"/(?:\\.env|\\.git/|wp-login\\.php|xmlrpc\\.php|wp-admin(?:/|$)|phpinfo\\.php|vendor/phpunit|setup\\.php|manager/html|id\\.php|shell\\.php|\\.DS_Store)[^"]*".*"status":(?:40[0-4]|403|404).*$
+            ^.*"remote_addr":"<HOST>".*"request_uri":"/(?:\\.env|\\.git/|wp-login\\.php|xmlrpc\\.php|wp-admin(?:/|$)|phpinfo\\.php|vendor/phpunit|setup\\.php|manager/html|id\\.php|shell\\.php|\\.DS_Store)[^"]*".*"status":(?:40[0-4]|403|404).*$
+
+ignoreregex = ^.*"request_uri":"\/(?:stub_status|health\/system|health\/worker|pgadmin4(?:\/|$)|\.well-known\/acme-challenge\/|.*\.(?:css|js|png|jpg|jpeg|gif|svg|ico|webp|woff2?))".*$

fail2ban/filter.d/sshd-pubkey.conf

@@ -0,0 +1,7 @@
+[Definition]
+failregex = ^%(__prefix_line)s(?:error: )?Received disconnect from <HOST> port \d+: .* \[preauth\]$
+            ^%(__prefix_line)sInvalid user .* from <HOST> port \d+ \[preauth\]$
+            ^%(__prefix_line)sFailed publickey for .* from <HOST> port \d+ ssh2$
+            ^%(__prefix_line)sConnection closed by (invalid user )?.* <HOST> port \d+ \[preauth\]$
+
+ignoreregex =

fail2ban/jail.d/00-defaults.local

@@ -0,0 +1,7 @@
+[DEFAULT]
+banaction = iptables-multiport
+# Push the rule into DOCKER-USER and cover your ports
+action    = iptables-multiport[name=nginx, port="80,443,3000", protocol=tcp, chain=DOCKER-USER]
+ignoreip = 127.0.0.1/8 ::1 \
+           172.19.0.0/16 172.20.0.0/16 172.22.0.0/16 \
+           5.75.153.161 167.235.254.4

fail2ban/jail.d/nginx-phoenix.local

@@ -0,0 +1,88 @@
+# /etc/fail2ban/jail.d/nginx-phoenix.local
+[DEFAULT]
+backend    = polling
+findtime   = 10m
+
+# ban timing: start short, escalate for repeat offenders, add jitter to avoid stampedes
+bantime              = 5m
+bantime.increment    = true
+bantime.factor       = 1.5
+bantime.overalljails = true
+bantime.rndtime      = 60s
+
+# honor your global ignore list from 00-defaults.local (don’t repeat here)
+
+# -----------------------------
+# Common scanners / bad bots
+# -----------------------------
+[nginx-badbots]
+enabled   = true
+filter    = nginx-badbots
+logpath   = /data/nginx-logs/access_json.log
+port      = 80,443,3000
+findtime  = 2m
+maxretry  = 5
+bantime   = 30m
+
+# -----------------------------
+# Bot search / generic probing
+# (odd paths, wp-admin, phpinfo, etc.)
+# -----------------------------
+[nginx-botsearch]
+enabled   = true
+filter    = nginx-botsearch
+logpath   = /data/nginx-logs/access_json.log
+port      = 80,443,3000
+findtime  = 5m
+maxretry  = 6
+bantime   = 30m
+
+# -----------------------------
+# Many 4xx in a short window
+# (likely brute/scan or broken client)
+# -----------------------------
+[nginx-4xx]
+enabled   = true
+filter    = nginx-4xx
+logpath   = /data/nginx-logs/access_json.log
+port      = 80,443,3000
+findtime  = 5m
+maxretry  = 50
+bantime   = 15m
+
+# -----------------------------
+# Simple HTTP GET/POST flood
+# (rate-based; pairs well with nginx rate limit) http-get-dos
+# Not in use anymore, because to avoid bloking phx url paths, i would have to manually add all of them to avoi be banned. Instead we use 
+# nginx-429 witch is managed by nginx and once it hits the rate limit, it send back a 429 status code.
+# -----------------------------
+# [http-get-dos]
+# enabled   = true
+# filter    = http-get-dos-compressed
+# logpath   = /data/nginx-logs/access_json.log
+# port      = 80,443,3000
+# findtime  = 60s
+# maxretry  = 20
+# bantime   = 10m
+
+# -----------------------------
+# (Optional) recidive: longer ban for repeat offenders across jails
+# Requires fail2ban.log inside the container (already present by default)
+# -----------------------------
+# [recidive]
+# enabled   = true
+# logpath   = /data/fail2ban.log /var/log/fail2ban.log
+# backend   = auto
+# banaction = nftables-allports
+# findtime  = 12h
+# maxretry  = 4
+# bantime   = 24h
+
+[nginx-429]
+enabled   = true
+filter    = nginx-429
+logpath   = /data/nginx-logs/access_json.log
+port      = 80,443,3000
+findtime  = 60s
+maxretry  = 10
+bantime   = 10m

fail2ban/jail.d/nginx-phoenix.local.2454726.2025-09-24@15:59:59~

@@ -0,0 +1,88 @@
+# /etc/fail2ban/jail.d/nginx-phoenix.local
+[DEFAULT]
+backend    = polling
+findtime   = 10m
+
+# ban timing: start short, escalate for repeat offenders, add jitter to avoid stampedes
+bantime              = 5m
+bantime.increment    = true
+bantime.factor       = 1.5
+bantime.overalljails = true
+bantime.rndtime      = 60s
+
+# honor your global ignore list from 00-defaults.local (don’t repeat here)
+
+# -----------------------------
+# Common scanners / bad bots
+# -----------------------------
+[nginx-badbots]
+enabled   = true
+filter    = nginx-badbots
+logpath   = /data/nginx-logs/access_json.log
+port      = 80,443,3000
+findtime  = 2m
+maxretry  = 5
+bantime   = 30m
+
+# -----------------------------
+# Bot search / generic probing
+# (odd paths, wp-admin, phpinfo, etc.)
+# -----------------------------
+[nginx-botsearch]
+enabled   = true
+filter    = nginx-botsearch
+logpath   = /data/nginx-logs/access_json.log
+port      = 80,443,3000
+findtime  = 5m
+maxretry  = 6
+bantime   = 30m
+
+# -----------------------------
+# Many 4xx in a short window
+# (likely brute/scan or broken client)
+# -----------------------------
+[nginx-4xx]
+enabled   = true
+filter    = nginx-4xx
+logpath   = /data/nginx-logs/access_json.log
+port      = 80,443,3000
+findtime  = 5m
+maxretry  = 20
+bantime   = 15m
+
+# -----------------------------
+# Simple HTTP GET/POST flood
+# (rate-based; pairs well with nginx rate limit) http-get-dos
+# Not in use anymore, because to avoid bloking phx url paths, i would have to manually add all of them to avoi be banned. Instead we use 
+# nginx-429 witch is managed by nginx and once it hits the rate limit, it send back a 429 status code.
+# -----------------------------
+# [http-get-dos]
+# enabled   = true
+# filter    = http-get-dos-compressed
+# logpath   = /data/nginx-logs/access_json.log
+# port      = 80,443,3000
+# findtime  = 60s
+# maxretry  = 20
+# bantime   = 10m
+
+# -----------------------------
+# (Optional) recidive: longer ban for repeat offenders across jails
+# Requires fail2ban.log inside the container (already present by default)
+# -----------------------------
+# [recidive]
+# enabled   = true
+# logpath   = /data/fail2ban.log /var/log/fail2ban.log
+# backend   = auto
+# banaction = nftables-allports
+# findtime  = 12h
+# maxretry  = 4
+# bantime   = 24h
+
+[nginx-429]
+enabled   = true
+filter    = nginx-429
+logpath   = /data/nginx-logs/access_json.log
+port      = 80,443,3000
+findtime  = 60s
+maxretry  = 10
+bantime   = 10m

fail2ban/jail.d/sshd.local

@@ -0,0 +1,17 @@
+[DEFAULT]
+banaction = nftables-multiport
+backend   = auto
+findtime  = 10m
+bantime   = 30m
+maxretry  = 3
+bantime.increment = false
+bantime.factor = 1
+bantime.maxtime = 24h
+mode = aggressive
+
+[sshd]
+enabled = true
+port = 2522
+filter = sshd
+logpath = /var/log/auth.log
+action  = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="tcp"]

nginx/nginx.conf

@@ -7,11 +7,11 @@ events {
 
 http {
     # ## Track by client IP; 20MB ≈ ~1200 active IPs
-    limit_req_zone $binary_remote_addr zone=rl_zone:20m rate=50r/s;
+    # limit_req_zone $binary_remote_addr zone=rl_zone:20m rate=50r/s;
-    # Return 429 instead of 503 when throttled
+    # # Return 429 instead of 503 when throttled
-    limit_req_status 429; # 429 is the HTTP status code for Too Many Requests
+    # limit_req_status 429; # 429 is the HTTP status code for Too Many Requests
-    # Log 429s at warn (not error)
+    # # Log 429s at warn (not error)
-    limit_req_log_level warn;
+    # limit_req_log_level warn;
 
     geo $frontend_whitelist {
         default 1;
@@ -63,7 +63,7 @@ http {
         zone pgadmin4-ui 64k;
         least_conn;
         server pgadmin4-ui:80 resolve fail_timeout=120s max_fails=20;
-        server 127.0.0.1:81 backup;  # Backup server for unavailable service
+        # server 127.0.0.1:81 backup;  # Backup server for unavailable service
         # ADD_PGADMIN4_SERVERS_HERE
     }
 
@@ -141,7 +141,7 @@ http {
         listen 127.0.0.1:81;
         server_name _;
 
-        limit_req zone=rl_zone burst=30 nodelay;
+        # limit_req zone=rl_zone burst=30 nodelay;
         
         # Return service unavailable for health checks
         location /health {
@@ -174,7 +174,7 @@ http {
         listen 80;
         server_name _;
 
-        limit_req zone=rl_zone burst=30 nodelay;
+        # limit_req zone=rl_zone burst=30 nodelay;
 
         # Security headers
         add_header X-Frame-Options "SAMEORIGIN" always;
@@ -279,6 +279,9 @@ http {
         # include /etc/nginx/includes/*.conf;
         location /pgadmin4 {
             error_log /var/log/nginx/pgadmin4_error.log notice;
+
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
             
             proxy_pass http://pgadmin4-ui/;
             proxy_set_header X-Script-Name /pgadmin4;
@@ -395,7 +398,7 @@ http {
         server_name _;
 
         # Apply globally inside this server
-        limit_req zone=rl_zone burst=30 nodelay;
+        # limit_req zone=rl_zone burst=30 nodelay;
 
         # Security headers
         add_header X-Frame-Options "SAMEORIGIN" always;