From ebac8ff7e5eba49979b4e094d419fba7233c647c Mon Sep 17 00:00:00 2001 From: Yuri Lima Date: Thu, 30 Jan 2025 14:47:40 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/blackbox_targets.json | 12 ++++ config/hetzner_dns_mapping.json | 102 +++++++++++++++++++++++++++ config/hetzner_excluded_servers.json | 8 +++ config/hetzner_targets.json | 18 ++--- config/prometheus.yaml | 23 +++++- docker-compose.yml | 8 +++ hetzner_discovery.py | 96 ++++++++++++++++++++----- https_portal/log/error.log | 7 ++ 8 files changed, 246 insertions(+), 28 deletions(-) create mode 100644 config/blackbox_targets.json create mode 100644 config/hetzner_dns_mapping.json create mode 100644 config/hetzner_excluded_servers.json diff --git a/config/blackbox_targets.json b/config/blackbox_targets.json new file mode 100644 index 0000000..aa352d7 --- /dev/null +++ b/config/blackbox_targets.json @@ -0,0 +1,12 @@ +[ + { + "targets": [ + "https://alpha.phx-erp.de" + ], + "labels": { + "job": "blackbox-dynamic", + "instance": "Alpha", + "probe": "http" + } + } +] \ No newline at end of file diff --git a/config/hetzner_dns_mapping.json b/config/hetzner_dns_mapping.json new file mode 100644 index 0000000..91e6ef8 --- /dev/null +++ b/config/hetzner_dns_mapping.json @@ -0,0 +1,102 @@ +[ + { + "dns_name": "PHX-DEV-001.Alpha", + "ip_address": "157.90.161.42" + }, + { + "dns_name": "ANSIBLE-MASTER", + "ip_address": "167.235.254.4" + }, + { + "dns_name": "cts", + "ip_address": "116.203.75.215" + }, + { + "dns_name": "Phx-Yuri", + "ip_address": "162.55.52.253" + }, + { + "dns_name": "benzinger-demo", + "ip_address": "162.55.54.75" + }, + { + "dns_name": "phx-internal", + "ip_address": "5.75.183.139" + }, + { + "dns_name": "trachtenmode-schmid", + "ip_address": "142.132.165.231" + }, + { + "dns_name": "teamcity", + "ip_address": "23.88.107.109" + }, + { + "dns_name": "gitea", + "ip_address": "49.13.146.138" + }, + { + "dns_name": "phx-beta-rc", + "ip_address": "159.69.200.205" + }, + { + "dns_name": "lhl", + "ip_address": "116.203.53.137" + }, + { + "dns_name": "Grafana-Prometheus", + "ip_address": "5.75.153.161" + }, + { + "dns_name": "cooper", + "ip_address": "159.69.44.39" + }, + { + "dns_name": "shipxpert", + "ip_address": "49.13.165.13" + }, + { + "dns_name": "dss", + "ip_address": "49.13.197.152" + }, + { + "dns_name": "sartissohn", + "ip_address": "188.245.44.219" + }, + { + "dns_name": "guntli", + "ip_address": "116.203.92.218" + }, + { + "dns_name": "kolb", + "ip_address": "94.130.77.57" + }, + { + "dns_name": "ried", + "ip_address": "116.203.151.20" + }, + { + "dns_name": "heba", + "ip_address": "116.203.68.120" + }, + { + "dns_name": "eicsoft", + "ip_address": "116.203.128.69" + }, + { + "dns_name": "ck-vechta", + "ip_address": "159.69.93.252" + }, + { + "dns_name": "eeparts", + "ip_address": "116.203.46.171" + }, + { + "dns_name": "big-break-changes", + "ip_address": "167.235.130.242" + }, + { + "dns_name": "bug", + "ip_address": "138.199.196.189" + } +] \ No newline at end of file diff --git a/config/hetzner_excluded_servers.json b/config/hetzner_excluded_servers.json new file mode 100644 index 0000000..56df0c3 --- /dev/null +++ b/config/hetzner_excluded_servers.json @@ -0,0 +1,8 @@ +[ + { + "server_id": 52530041, + "name": "docuvita", + "datacenter": "nbg1", + "reason": "Darklisted server" + } +] \ No newline at end of file diff --git a/config/hetzner_targets.json b/config/hetzner_targets.json index 421c71b..353c211 100644 --- a/config/hetzner_targets.json +++ b/config/hetzner_targets.json @@ -161,15 +161,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "128.140.15.177:9100" - ], - "labels": { - "instance": "docuvita", - "datacenter": "nbg1" - } - }, { "targets": [ "116.203.151.20:9100" @@ -223,5 +214,14 @@ "instance": "big-break-changes", "datacenter": "nbg1" } + }, + { + "targets": [ + "138.199.196.189:9100" + ], + "labels": { + "instance": "bug", + "datacenter": "nbg1" + } } ] \ No newline at end of file diff --git a/config/prometheus.yaml b/config/prometheus.yaml index c4302c0..d1f3bbd 100644 --- a/config/prometheus.yaml +++ b/config/prometheus.yaml @@ -2,9 +2,28 @@ global: scrape_interval: 15s # Default for all jobs unless overridden scrape_configs: + # Existing Hetzner Dynamic Configuration - job_name: "hetzner-dynamic" + scheme: http file_sd_configs: - files: - - "/hetzner_targets.json" + - "/opt/phx/main/config/hetzner_targets.json" refresh_interval: 30s - scrape_interval: 5s # Custom interval for Hetzner servers \ No newline at end of file + scrape_interval: 5s # Custom interval for Hetzner servers + + # Blackbox Exporter with Dynamic File SD + # - job_name: 'blackbox-dynamic' + # metrics_path: /probe + # params: + # module: [http_2xx] # HTTP check to see if target is up + # file_sd_configs: + # - files: + # - "/opt/phx/main/config/blackbox_targets.json" # Dynamic file like hetzner_targets.json + # refresh_interval: 30s + # relabel_configs: + # - source_labels: [__address__] + # target_label: __param_target + # - source_labels: [__param_target] + # target_label: instance + # - target_label: __address__ + # replacement: blackbox_exporter:9115 # Blackbox Exporter container/service \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index c58889d..f659636 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,6 +73,14 @@ services: - "/proc:/host/proc:ro" - "/sys:/host/sys:ro" - "/:/host:ro,rslave" + # blackbox_exporter: + # image: prom/blackbox-exporter + # container_name: blackbox_exporter + # ports: + # - "9115:9115" + # restart: unless-stopped + # networks: + # - default https_portal: container_name: https_portal image: "steveltn/https-portal:1.21" diff --git a/hetzner_discovery.py b/hetzner_discovery.py index a9501b0..74cd313 100644 --- a/hetzner_discovery.py +++ b/hetzner_discovery.py @@ -1,6 +1,7 @@ import requests import json import os +import time from dotenv import load_dotenv # Load environment variables from .env file @@ -13,9 +14,11 @@ HETZNER_API_TOKEN = os.getenv("HETZNER_API_TOKEN") if not HETZNER_API_TOKEN: raise ValueError("❌ HETZNER_API_TOKEN is missing! Make sure it's set in the .env file.") -# 📂 Path to the Prometheus service discovery file +# 📂 Paths to output files PROMETHEUS_TARGETS_FILE = "/opt/phx/main/config/hetzner_targets.json" ERROR_LOG_FILE = "/opt/phx/main/config/hetzner_error_servers.json" +EXCLUDED_SERVERS_FILE = "/opt/phx/main/config/hetzner_excluded_servers.json" +DNS_MAPPING_FILE = "/opt/phx/main/config/hetzner_dns_mapping.json" # 📌 Hetzner API URL HETZNER_API_URL = os.getenv("HETZNER_API_URL") @@ -24,31 +27,74 @@ HETZNER_API_URL = os.getenv("HETZNER_API_URL") if not HETZNER_API_URL: raise ValueError("❌ HETZNER_API_URL is missing! Make sure it's set in the .env file.") +# 🛑 List of server names to exclude (DARKLIST) +DARKLISTED_SERVERS = ["docuvita"] -# 📡 Fetch Hetzner server list +# 📡 Fetch Hetzner server list with pagination support and Rate Limiting handling def get_hetzner_servers(): headers = {"Authorization": f"Bearer {HETZNER_API_TOKEN}"} - response = requests.get(HETZNER_API_URL, headers=headers) + all_servers = [] + page = 1 + per_page = 50 # Max per request - if response.status_code != 200: - print(f"❌ Error fetching servers: {response.text}") - return [] + while True: + response = requests.get( + f"{HETZNER_API_URL}?page={page}&per_page={per_page}", headers=headers + ) - return response.json().get("servers", []) + # Handle Rate Limiting (429 Too Many Requests) + if response.status_code == 429: + reset_time = int(response.headers.get("RateLimit-Reset", time.time() + 60)) + wait_time = reset_time - int(time.time()) + print(f"⏳ Rate limit exceeded! Waiting {wait_time} seconds until reset...") + time.sleep(wait_time) + continue # Retry after waiting + if response.status_code != 200: + print(f"❌ Error fetching servers: {response.text}") + break # Stop fetching if error -# 🏗️ Generate Prometheus JSON file + # Read rate limit headers + remaining = response.headers.get("RateLimit-Remaining", "unknown") + reset_time = int(response.headers.get("RateLimit-Reset", time.time() + 60)) + print(f"📊 API Rate Limit: {remaining} requests remaining. Next reset at {reset_time}.") + + data = response.json() + servers = data.get("servers", []) + all_servers.extend(servers) + + # Check if there's a next page + pagination = data.get("meta", {}).get("pagination", {}) + if not pagination.get("next_page"): + break # Exit if no more pages + + page = pagination["next_page"] # Move to the next page + + return all_servers + +# 🏗️ Generate Prometheus JSON file and DNS Mapping def generate_prometheus_sd_config(): servers = get_hetzner_servers() targets = [] error_servers = [] + excluded_servers = [] + dns_mappings = [] # New list for storing DNS-IP mappings for server in servers: - if "public_net" in server and "ipv4" in server["public_net"]: - ipv4 = server["public_net"]["ipv4"]["ip"] - server_name = server["name"] - datacenter = server["datacenter"]["location"]["name"] + ipv4 = server.get("public_net", {}).get("ipv4", {}).get("ip") + server_name = server["name"] + datacenter = server["datacenter"]["location"]["name"] + if server_name in DARKLISTED_SERVERS: + excluded_servers.append({ + "server_id": server["id"], + "name": server_name, + "datacenter": datacenter, + "reason": "Darklisted server" + }) + continue # Skip adding to Prometheus targets + + if ipv4: targets.append({ "targets": [f"{ipv4}:9100"], "labels": { @@ -56,13 +102,18 @@ def generate_prometheus_sd_config(): "datacenter": datacenter } }) + # Add to DNS mapping file + dns_mappings.append({ + "dns_name": server_name, + "ip_address": ipv4 + }) else: # Log the server that couldn't be added error_servers.append({ "server_id": server["id"], - "name": server["name"], + "name": server_name, "status": server["status"], - "datacenter": server["datacenter"]["location"]["name"], + "datacenter": datacenter, "reason": "Missing public_net or IPv4" }) @@ -72,14 +123,25 @@ def generate_prometheus_sd_config(): print(f"✅ Updated Prometheus targets in {PROMETHEUS_TARGETS_FILE}") - # Save error logs if any servers were skipped + # Save DNS Mappings file + with open(DNS_MAPPING_FILE, "w") as f: + json.dump(dns_mappings, f, indent=4) + + print(f"📡 Created DNS Mapping file: {DNS_MAPPING_FILE}") + + # Save error logs if any servers were skipped due to missing data if error_servers: with open(ERROR_LOG_FILE, "w") as f: json.dump(error_servers, f, indent=4) print(f"⚠️ Some servers could not be added. Check {ERROR_LOG_FILE} for details.") - else: - print("✅ All servers were added successfully, no errors detected.") + # Save excluded servers log + if excluded_servers: + with open(EXCLUDED_SERVERS_FILE, "w") as f: + json.dump(excluded_servers, f, indent=4) + print(f"🚫 Darklisted servers were skipped. See {EXCLUDED_SERVERS_FILE} for details.") + else: + print("✅ No servers were excluded due to the darklist.") # 🔄 Run the script if __name__ == "__main__": diff --git a/https_portal/log/error.log b/https_portal/log/error.log index 048c8da..a6cdfd4 100644 --- a/https_portal/log/error.log +++ b/https_portal/log/error.log @@ -8,3 +8,10 @@ 2025/01/30 12:49:57 [notice] 265#265: signal process started 2025/01/30 12:53:15 [notice] 266#266: signal process started 2025/01/30 12:56:58 [notice] 265#265: signal process started +2025/01/30 13:20:47 [notice] 265#265: signal process started +2025/01/30 13:21:52 [notice] 265#265: signal process started +2025/01/30 13:25:46 [notice] 264#264: signal process started +2025/01/30 13:35:24 [notice] 265#265: signal process started +2025/01/30 13:41:55 [notice] 265#265: signal process started +2025/01/30 14:35:34 [notice] 265#265: signal process started +2025/01/30 14:45:52 [notice] 265#265: signal process started