👌 IMPROVE:

config/blackbox_targets.json

@@ -0,0 +1,12 @@
+[
+    {
+        "targets": [
+            "https://alpha.phx-erp.de"
+        ],
+        "labels": {
+            "job": "blackbox-dynamic",
+            "instance": "Alpha",
+            "probe": "http"
+        }
+    }
+]

config/hetzner_dns_mapping.json

@@ -0,0 +1,102 @@
+[
+    {
+        "dns_name": "PHX-DEV-001.Alpha",
+        "ip_address": "157.90.161.42"
+    },
+    {
+        "dns_name": "ANSIBLE-MASTER",
+        "ip_address": "167.235.254.4"
+    },
+    {
+        "dns_name": "cts",
+        "ip_address": "116.203.75.215"
+    },
+    {
+        "dns_name": "Phx-Yuri",
+        "ip_address": "162.55.52.253"
+    },
+    {
+        "dns_name": "benzinger-demo",
+        "ip_address": "162.55.54.75"
+    },
+    {
+        "dns_name": "phx-internal",
+        "ip_address": "5.75.183.139"
+    },
+    {
+        "dns_name": "trachtenmode-schmid",
+        "ip_address": "142.132.165.231"
+    },
+    {
+        "dns_name": "teamcity",
+        "ip_address": "23.88.107.109"
+    },
+    {
+        "dns_name": "gitea",
+        "ip_address": "49.13.146.138"
+    },
+    {
+        "dns_name": "phx-beta-rc",
+        "ip_address": "159.69.200.205"
+    },
+    {
+        "dns_name": "lhl",
+        "ip_address": "116.203.53.137"
+    },
+    {
+        "dns_name": "Grafana-Prometheus",
+        "ip_address": "5.75.153.161"
+    },
+    {
+        "dns_name": "cooper",
+        "ip_address": "159.69.44.39"
+    },
+    {
+        "dns_name": "shipxpert",
+        "ip_address": "49.13.165.13"
+    },
+    {
+        "dns_name": "dss",
+        "ip_address": "49.13.197.152"
+    },
+    {
+        "dns_name": "sartissohn",
+        "ip_address": "188.245.44.219"
+    },
+    {
+        "dns_name": "guntli",
+        "ip_address": "116.203.92.218"
+    },
+    {
+        "dns_name": "kolb",
+        "ip_address": "94.130.77.57"
+    },
+    {
+        "dns_name": "ried",
+        "ip_address": "116.203.151.20"
+    },
+    {
+        "dns_name": "heba",
+        "ip_address": "116.203.68.120"
+    },
+    {
+        "dns_name": "eicsoft",
+        "ip_address": "116.203.128.69"
+    },
+    {
+        "dns_name": "ck-vechta",
+        "ip_address": "159.69.93.252"
+    },
+    {
+        "dns_name": "eeparts",
+        "ip_address": "116.203.46.171"
+    },
+    {
+        "dns_name": "big-break-changes",
+        "ip_address": "167.235.130.242"
+    },
+    {
+        "dns_name": "bug",
+        "ip_address": "138.199.196.189"
+    }
+]

config/hetzner_excluded_servers.json

@@ -0,0 +1,8 @@
+[
+    {
+        "server_id": 52530041,
+        "name": "docuvita",
+        "datacenter": "nbg1",
+        "reason": "Darklisted server"
+    }
+]

config/hetzner_targets.json

@@ -161,15 +161,6 @@
             "datacenter": "nbg1"
         }
     },
-    {
-        "targets": [
-            "128.140.15.177:9100"
-        ],
-        "labels": {
-            "instance": "docuvita",
-            "datacenter": "nbg1"
-        }
-    },
     {
         "targets": [
             "116.203.151.20:9100"
@@ -223,5 +214,14 @@
             "instance": "big-break-changes",
             "datacenter": "nbg1"
         }
+    },
+    {
+        "targets": [
+            "138.199.196.189:9100"
+        ],
+        "labels": {
+            "instance": "bug",
+            "datacenter": "nbg1"
+        }
     }
 ]

config/prometheus.yaml

@@ -2,9 +2,28 @@ global:
   scrape_interval: 15s  # Default for all jobs unless overridden
 
 scrape_configs:
+  # Existing Hetzner Dynamic Configuration
   - job_name: "hetzner-dynamic"
+    scheme: http
     file_sd_configs:
       - files:
-          - "/hetzner_targets.json"
+          - "/opt/phx/main/config/hetzner_targets.json"
         refresh_interval: 30s
     scrape_interval: 5s  # Custom interval for Hetzner servers
+
+  # Blackbox Exporter with Dynamic File SD
+  # - job_name: 'blackbox-dynamic'
+  #   metrics_path: /probe
+  #   params:
+  #     module: [http_2xx]  # HTTP check to see if target is up
+  #   file_sd_configs:
+  #     - files:
+  #         - "/opt/phx/main/config/blackbox_targets.json"  # Dynamic file like hetzner_targets.json
+  #       refresh_interval: 30s
+  #   relabel_configs:
+  #     - source_labels: [__address__]
+  #       target_label: __param_target
+  #     - source_labels: [__param_target]
+  #       target_label: instance
+  #     - target_label: __address__
+  #       replacement: blackbox_exporter:9115  # Blackbox Exporter container/service

docker-compose.yml

@@ -73,6 +73,14 @@ services:
       - "/proc:/host/proc:ro"
       - "/sys:/host/sys:ro"
       - "/:/host:ro,rslave"
+  # blackbox_exporter:
+  #   image: prom/blackbox-exporter
+  #   container_name: blackbox_exporter
+  #   ports:
+  #     - "9115:9115"
+  #   restart: unless-stopped
+  #   networks:
+  #     - default
   https_portal:
     container_name: https_portal
     image: "steveltn/https-portal:1.21"

hetzner_discovery.py

@@ -1,6 +1,7 @@
 import requests
 import json
 import os
+import time
 from dotenv import load_dotenv
 
 # Load environment variables from .env file
@@ -13,9 +14,11 @@ HETZNER_API_TOKEN = os.getenv("HETZNER_API_TOKEN")
 if not HETZNER_API_TOKEN:
     raise ValueError("❌ HETZNER_API_TOKEN is missing! Make sure it's set in the .env file.")
 
-# 📂 Path to the Prometheus service discovery file
+# 📂 Paths to output files
 PROMETHEUS_TARGETS_FILE = "/opt/phx/main/config/hetzner_targets.json"
 ERROR_LOG_FILE = "/opt/phx/main/config/hetzner_error_servers.json"
+EXCLUDED_SERVERS_FILE = "/opt/phx/main/config/hetzner_excluded_servers.json"
+DNS_MAPPING_FILE = "/opt/phx/main/config/hetzner_dns_mapping.json"
 
 # 📌 Hetzner API URL
 HETZNER_API_URL = os.getenv("HETZNER_API_URL")
@@ -24,31 +27,74 @@ HETZNER_API_URL = os.getenv("HETZNER_API_URL")
 if not HETZNER_API_URL:
     raise ValueError("❌ HETZNER_API_URL is missing! Make sure it's set in the .env file.")
 
+# 🛑 List of server names to exclude (DARKLIST)
+DARKLISTED_SERVERS = ["docuvita"]
 
-# 📡 Fetch Hetzner server list
+# 📡 Fetch Hetzner server list with pagination support and Rate Limiting handling
 def get_hetzner_servers():
     headers = {"Authorization": f"Bearer {HETZNER_API_TOKEN}"}
-    response = requests.get(HETZNER_API_URL, headers=headers)
+    all_servers = []
+    page = 1
+    per_page = 50  # Max per request
+
+    while True:
+        response = requests.get(
+            f"{HETZNER_API_URL}?page={page}&per_page={per_page}", headers=headers
+        )
+
+        # Handle Rate Limiting (429 Too Many Requests)
+        if response.status_code == 429:
+            reset_time = int(response.headers.get("RateLimit-Reset", time.time() + 60))
+            wait_time = reset_time - int(time.time())
+            print(f"⏳ Rate limit exceeded! Waiting {wait_time} seconds until reset...")
+            time.sleep(wait_time)
+            continue  # Retry after waiting
 
         if response.status_code != 200:
             print(f"❌ Error fetching servers: {response.text}")
-        return []
+            break  # Stop fetching if error
 
-    return response.json().get("servers", [])
+        # Read rate limit headers
+        remaining = response.headers.get("RateLimit-Remaining", "unknown")
+        reset_time = int(response.headers.get("RateLimit-Reset", time.time() + 60))
+        print(f"📊 API Rate Limit: {remaining} requests remaining. Next reset at {reset_time}.")
 
+        data = response.json()
+        servers = data.get("servers", [])
+        all_servers.extend(servers)
 
-# 🏗️ Generate Prometheus JSON file
+        # Check if there's a next page
+        pagination = data.get("meta", {}).get("pagination", {})
+        if not pagination.get("next_page"):
+            break  # Exit if no more pages
+
+        page = pagination["next_page"]  # Move to the next page
+
+    return all_servers
+
+# 🏗️ Generate Prometheus JSON file and DNS Mapping
 def generate_prometheus_sd_config():
     servers = get_hetzner_servers()
     targets = []
     error_servers = []
+    excluded_servers = []
+    dns_mappings = []  # New list for storing DNS-IP mappings
 
     for server in servers:
-        if "public_net" in server and "ipv4" in server["public_net"]:
+        ipv4 = server.get("public_net", {}).get("ipv4", {}).get("ip")
-            ipv4 = server["public_net"]["ipv4"]["ip"]
         server_name = server["name"]
         datacenter = server["datacenter"]["location"]["name"]
 
+        if server_name in DARKLISTED_SERVERS:
+            excluded_servers.append({
+                "server_id": server["id"],
+                "name": server_name,
+                "datacenter": datacenter,
+                "reason": "Darklisted server"
+            })
+            continue  # Skip adding to Prometheus targets
+
+        if ipv4:
             targets.append({
                 "targets": [f"{ipv4}:9100"],
                 "labels": {
@@ -56,13 +102,18 @@ def generate_prometheus_sd_config():
                     "datacenter": datacenter
                 }
             })
+            # Add to DNS mapping file
+            dns_mappings.append({
+                "dns_name": server_name,
+                "ip_address": ipv4
+            })
         else:
             # Log the server that couldn't be added
             error_servers.append({
                 "server_id": server["id"],
-                "name": server["name"],
+                "name": server_name,
                 "status": server["status"],
-                "datacenter": server["datacenter"]["location"]["name"],
+                "datacenter": datacenter,
                 "reason": "Missing public_net or IPv4"
             })
 
@@ -72,14 +123,25 @@ def generate_prometheus_sd_config():
 
     print(f"✅ Updated Prometheus targets in {PROMETHEUS_TARGETS_FILE}")
 
-    # Save error logs if any servers were skipped
+    # Save DNS Mappings file
+    with open(DNS_MAPPING_FILE, "w") as f:
+        json.dump(dns_mappings, f, indent=4)
+
+    print(f"📡 Created DNS Mapping file: {DNS_MAPPING_FILE}")
+
+    # Save error logs if any servers were skipped due to missing data
     if error_servers:
         with open(ERROR_LOG_FILE, "w") as f:
             json.dump(error_servers, f, indent=4)
         print(f"⚠️ Some servers could not be added. Check {ERROR_LOG_FILE} for details.")
-    else:
-        print("✅ All servers were added successfully, no errors detected.")
 
+    # Save excluded servers log
+    if excluded_servers:
+        with open(EXCLUDED_SERVERS_FILE, "w") as f:
+            json.dump(excluded_servers, f, indent=4)
+        print(f"🚫 Darklisted servers were skipped. See {EXCLUDED_SERVERS_FILE} for details.")
+    else:
+        print("✅ No servers were excluded due to the darklist.")
 
 # 🔄 Run the script
 if __name__ == "__main__":

https_portal/log/error.log

@@ -8,3 +8,10 @@
 2025/01/30 12:49:57 [notice] 265#265: signal process started
 2025/01/30 12:53:15 [notice] 266#266: signal process started
 2025/01/30 12:56:58 [notice] 265#265: signal process started
+2025/01/30 13:20:47 [notice] 265#265: signal process started
+2025/01/30 13:21:52 [notice] 265#265: signal process started
+2025/01/30 13:25:46 [notice] 264#264: signal process started
+2025/01/30 13:35:24 [notice] 265#265: signal process started
+2025/01/30 13:41:55 [notice] 265#265: signal process started
+2025/01/30 14:35:34 [notice] 265#265: signal process started
+2025/01/30 14:45:52 [notice] 265#265: signal process started