diff --git a/config/hetzner_targets.json b/config/hetzner_targets.json index 03ab19b..3759be4 100644 --- a/config/hetzner_targets.json +++ b/config/hetzner_targets.json @@ -9,16 +9,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "157.90.161.42:9800" - ], - "labels": { - "instance": "PHX-DEV-001.Alpha", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "167.235.254.4:9100", @@ -29,16 +19,6 @@ "datacenter": "fsn1" } }, - { - "targets": [ - "167.235.254.4:9800" - ], - "labels": { - "instance": "ANSIBLE-MASTER", - "datacenter": "fsn1", - "job": "health-exporter" - } - }, { "targets": [ "116.203.75.215:9100", @@ -49,16 +29,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "116.203.75.215:9800" - ], - "labels": { - "instance": "cts", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "162.55.52.253:9100", @@ -69,16 +39,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "162.55.52.253:9800" - ], - "labels": { - "instance": "Phx-Yuri", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "162.55.54.75:9100", @@ -89,16 +49,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "162.55.54.75:9800" - ], - "labels": { - "instance": "benzinger-demo", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "5.75.183.139:9100", @@ -109,16 +59,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "5.75.183.139:9800" - ], - "labels": { - "instance": "phx-internal", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "142.132.165.231:9100", @@ -129,16 +69,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "142.132.165.231:9800" - ], - "labels": { - "instance": "trachtenmode-schmid", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "159.69.200.205:9100", @@ -149,16 +79,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "159.69.200.205:9800" - ], - "labels": { - "instance": "phx-beta-rc", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "116.203.53.137:9100", @@ -169,16 +89,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "116.203.53.137:9800" - ], - "labels": { - "instance": "lhl", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "5.75.153.161:9100", @@ -189,16 +99,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "5.75.153.161:9800" - ], - "labels": { - "instance": "Grafana-Prometheus", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "159.69.44.39:9100", @@ -209,16 +109,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "159.69.44.39:9800" - ], - "labels": { - "instance": "cooper", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "49.13.165.13:9100", @@ -229,16 +119,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "49.13.165.13:9800" - ], - "labels": { - "instance": "shipxpert", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "188.245.44.219:9100", @@ -249,16 +129,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "188.245.44.219:9800" - ], - "labels": { - "instance": "sartissohn", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "116.203.92.218:9100", @@ -269,16 +139,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "116.203.92.218:9800" - ], - "labels": { - "instance": "guntli", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "94.130.77.57:9100", @@ -289,16 +149,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "94.130.77.57:9800" - ], - "labels": { - "instance": "kolb", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "116.203.151.20:9100", @@ -309,16 +159,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "116.203.151.20:9800" - ], - "labels": { - "instance": "ried", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "116.203.68.120:9100", @@ -329,16 +169,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "116.203.68.120:9800" - ], - "labels": { - "instance": "heba", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "116.203.128.69:9100", @@ -349,16 +179,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "116.203.128.69:9800" - ], - "labels": { - "instance": "eicsoft", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "159.69.93.252:9100", @@ -369,16 +189,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "159.69.93.252:9800" - ], - "labels": { - "instance": "ck-vechta", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "116.203.46.171:9100", @@ -389,16 +199,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "116.203.46.171:9800" - ], - "labels": { - "instance": "eeparts", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "167.235.130.242:9100", @@ -409,16 +209,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "167.235.130.242:9800" - ], - "labels": { - "instance": "big-break-changes", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "188.245.32.214:9100", @@ -429,16 +219,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "188.245.32.214:9800" - ], - "labels": { - "instance": "PHX-License-Server", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "138.199.201.35:9100", @@ -449,16 +229,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "138.199.201.35:9800" - ], - "labels": { - "instance": "PHX-SkyNodus", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "188.245.108.133:9100", @@ -469,16 +239,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "188.245.108.133:9800" - ], - "labels": { - "instance": "bode", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "168.119.174.181:9100", @@ -489,16 +249,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "168.119.174.181:9800" - ], - "labels": { - "instance": "skr04", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "91.99.24.53:9100", @@ -509,16 +259,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "91.99.24.53:9800" - ], - "labels": { - "instance": "artwin", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "168.119.231.96:9100", @@ -529,16 +269,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "168.119.231.96:9800" - ], - "labels": { - "instance": "amplid", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "116.203.143.63:9100", @@ -549,16 +279,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "116.203.143.63:9800" - ], - "labels": { - "instance": "schwarz", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "91.99.53.67:9100", @@ -569,16 +289,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "91.99.53.67:9800" - ], - "labels": { - "instance": "skr03", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "91.99.52.253:9100", @@ -589,16 +299,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "91.99.52.253:9800" - ], - "labels": { - "instance": "moonich", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "91.99.76.156:9100", @@ -609,16 +309,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "91.99.76.156:9800" - ], - "labels": { - "instance": "comp-sys", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "78.46.214.224:9100", @@ -629,16 +319,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "78.46.214.224:9800" - ], - "labels": { - "instance": "skr-ch", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "116.203.18.156:9100", @@ -649,16 +329,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "116.203.18.156:9800" - ], - "labels": { - "instance": "wsoft", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "91.99.71.224:9100", @@ -669,16 +339,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "91.99.71.224:9800" - ], - "labels": { - "instance": "bzp", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "49.13.197.152:9100", @@ -689,16 +349,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "49.13.197.152:9800" - ], - "labels": { - "instance": "dss", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "128.140.103.178:9100", @@ -709,16 +359,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "128.140.103.178:9800" - ], - "labels": { - "instance": "new-full-test", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "157.90.228.52:9100", @@ -729,16 +369,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "157.90.228.52:9800" - ], - "labels": { - "instance": "modern", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "23.88.56.89:9100", @@ -749,16 +379,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "23.88.56.89:9800" - ], - "labels": { - "instance": "burg-itc", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "116.203.73.240:9100", @@ -769,16 +389,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "116.203.73.240:9800" - ], - "labels": { - "instance": "test-ansible-ip-permission", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "91.99.29.55:9100", @@ -789,16 +399,6 @@ "datacenter": "nbg1" } }, - { - "targets": [ - "91.99.29.55:9800" - ], - "labels": { - "instance": "test-ansible-ip-permission-2", - "datacenter": "nbg1", - "job": "health-exporter" - } - }, { "targets": [ "91.99.107.178:9100", @@ -808,15 +408,5 @@ "instance": "bpp", "datacenter": "nbg1" } - }, - { - "targets": [ - "91.99.107.178:9800" - ], - "labels": { - "instance": "bpp", - "datacenter": "nbg1", - "job": "health-exporter" - } } ] \ No newline at end of file diff --git a/config/phoenix_health_exporter_hetzner_targets.json b/config/phoenix_health_exporter_hetzner_targets.json new file mode 100644 index 0000000..f905dd3 --- /dev/null +++ b/config/phoenix_health_exporter_hetzner_targets.json @@ -0,0 +1,371 @@ +[ + { + "targets": [ + "157.90.161.42:9800" + ], + "labels": { + "instance": "PHX-DEV-001.Alpha-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "167.235.254.4:9800" + ], + "labels": { + "instance": "ANSIBLE-MASTER-health-exporter", + "datacenter": "fsn1" + } + }, + { + "targets": [ + "116.203.75.215:9800" + ], + "labels": { + "instance": "cts-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "162.55.52.253:9800" + ], + "labels": { + "instance": "Phx-Yuri-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "162.55.54.75:9800" + ], + "labels": { + "instance": "benzinger-demo-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "5.75.183.139:9800" + ], + "labels": { + "instance": "phx-internal-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "142.132.165.231:9800" + ], + "labels": { + "instance": "trachtenmode-schmid-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "159.69.200.205:9800" + ], + "labels": { + "instance": "phx-beta-rc-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "116.203.53.137:9800" + ], + "labels": { + "instance": "lhl-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "5.75.153.161:9800" + ], + "labels": { + "instance": "Grafana-Prometheus-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "159.69.44.39:9800" + ], + "labels": { + "instance": "cooper-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "49.13.165.13:9800" + ], + "labels": { + "instance": "shipxpert-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "188.245.44.219:9800" + ], + "labels": { + "instance": "sartissohn-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "116.203.92.218:9800" + ], + "labels": { + "instance": "guntli-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "94.130.77.57:9800" + ], + "labels": { + "instance": "kolb-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "116.203.151.20:9800" + ], + "labels": { + "instance": "ried-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "116.203.68.120:9800" + ], + "labels": { + "instance": "heba-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "116.203.128.69:9800" + ], + "labels": { + "instance": "eicsoft-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "159.69.93.252:9800" + ], + "labels": { + "instance": "ck-vechta-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "116.203.46.171:9800" + ], + "labels": { + "instance": "eeparts-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "167.235.130.242:9800" + ], + "labels": { + "instance": "big-break-changes-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "188.245.32.214:9800" + ], + "labels": { + "instance": "PHX-License-Server-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "138.199.201.35:9800" + ], + "labels": { + "instance": "PHX-SkyNodus-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "188.245.108.133:9800" + ], + "labels": { + "instance": "bode-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "168.119.174.181:9800" + ], + "labels": { + "instance": "skr04-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "91.99.24.53:9800" + ], + "labels": { + "instance": "artwin-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "168.119.231.96:9800" + ], + "labels": { + "instance": "amplid-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "116.203.143.63:9800" + ], + "labels": { + "instance": "schwarz-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "91.99.53.67:9800" + ], + "labels": { + "instance": "skr03-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "91.99.52.253:9800" + ], + "labels": { + "instance": "moonich-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "91.99.76.156:9800" + ], + "labels": { + "instance": "comp-sys-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "78.46.214.224:9800" + ], + "labels": { + "instance": "skr-ch-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "116.203.18.156:9800" + ], + "labels": { + "instance": "wsoft-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "91.99.71.224:9800" + ], + "labels": { + "instance": "bzp-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "49.13.197.152:9800" + ], + "labels": { + "instance": "dss-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "128.140.103.178:9800" + ], + "labels": { + "instance": "new-full-test-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "157.90.228.52:9800" + ], + "labels": { + "instance": "modern-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "23.88.56.89:9800" + ], + "labels": { + "instance": "burg-itc-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "116.203.73.240:9800" + ], + "labels": { + "instance": "test-ansible-ip-permission-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "91.99.29.55:9800" + ], + "labels": { + "instance": "test-ansible-ip-permission-2-health-exporter", + "datacenter": "nbg1" + } + }, + { + "targets": [ + "91.99.107.178:9800" + ], + "labels": { + "instance": "bpp-health-exporter", + "datacenter": "nbg1" + } + } +] \ No newline at end of file diff --git a/config/prometheus.yaml b/config/prometheus.yaml index 459c36a..c5cd06c 100644 --- a/config/prometheus.yaml +++ b/config/prometheus.yaml @@ -8,8 +8,16 @@ scrape_configs: - files: - "/opt/phx/main/config/hetzner_targets.json" # 📁 JSON file containing the list of targets (e.g. IPs, ports, labels). refresh_interval: 30s # 🔁 How often Prometheus checks the file for changes and updates the target list. - scrape_interval: 5s # ⏱️ Override global default: scrape metrics from each target in this job every 5 seconds. + scrape_interval: 5s # ⏱️ Override global default: scrape metrics from each target in this job every x seconds. scheme: http # 🌐 Use plain HTTP when scraping the targets (not HTTPS). + + - job_name: "phoenix-health-exporter" + file_sd_configs: + - files: + - "/opt/phx/main/config/phoenix_health_exporter_hetzner_targets.json" + refresh_interval: 30s + scrape_interval: 2m # Change it carefully, it is a heavy job to keep open and close to check the health of DB connection, there annotation to refactor it and use a new pgDebounce image. + scheme: http # - job_name: "phoenix-system" # file_sd_configs: diff --git a/hetzner_discovery.py b/hetzner_discovery.py index 9914310..6992771 100644 --- a/hetzner_discovery.py +++ b/hetzner_discovery.py @@ -28,6 +28,7 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__)) PROMETHEUS_TARGETS_FILE = os.path.join(BASE_DIR, "config/hetzner_targets.json") PHX_SYSTEM_FILE = os.path.join(BASE_DIR, "config/phoenix_system_hetzner_targets.json") PHX_WORKER_FILE = os.path.join(BASE_DIR, "config/phoenix_worker_hetzner_targets.json") +PHX_HEALTH_EXPORTER_FILE = os.path.join(BASE_DIR, "config/phoenix_health_exporter_hetzner_targets.json") DNS_MAPPING_FILE = os.path.join(BASE_DIR, "config/hetzner_dns_mapping.json") ERROR_LOG_FILE = os.path.join(BASE_DIR, "config/hetzner_error_servers.json") EXCLUDED_SERVERS_FILE = os.path.join(BASE_DIR, "config/hetzner_excluded_servers.json") @@ -90,6 +91,7 @@ def generate_prometheus_sd_config(): targets = [] phx_system_targets = [] phx_worker_targets = [] + phx_health_exporter_targets = [] error_servers = [] excluded_servers = [] dns_mappings = [] # New list for storing DNS-IP mappings @@ -119,12 +121,11 @@ def generate_prometheus_sd_config(): }) # This is with Python Flask server for health checks - targets.append({ + phx_health_exporter_targets.append({ "targets": [f"{ipv4}:9800"], "labels": { - "instance": f"{server_name}", - "datacenter": datacenter, - "job": "health-exporter" + "instance": f"{server_name}-health-exporter", + "datacenter": datacenter } }) @@ -168,13 +169,17 @@ def generate_prometheus_sd_config(): print(f"✅ Updated Prometheus targets in {PROMETHEUS_TARGETS_FILE}") - with open(PHX_SYSTEM_FILE, "w") as f: - json.dump(phx_system_targets, f, indent=4) - print(f"✅ phoenix-system targets saved to {PHX_SYSTEM_FILE}") + with open(PHX_HEALTH_EXPORTER_FILE, "w") as f: + json.dump(phx_health_exporter_targets, f, indent=4) + print(f"✅ phoenix-health-exporter targets saved to {PHX_HEALTH_EXPORTER_FILE}") - with open(PHX_WORKER_FILE, "w") as f: - json.dump(phx_worker_targets, f, indent=4) - print(f"✅ phoenix-worker targets saved to {PHX_WORKER_FILE}") + # with open(PHX_SYSTEM_FILE, "w") as f: + # json.dump(phx_system_targets, f, indent=4) + # print(f"✅ phoenix-system targets saved to {PHX_SYSTEM_FILE}") + + # with open(PHX_WORKER_FILE, "w") as f: + # json.dump(phx_worker_targets, f, indent=4) + # print(f"✅ phoenix-worker targets saved to {PHX_WORKER_FILE}") # Save DNS Mappings file with open(DNS_MAPPING_FILE, "w") as f: