From ddfdb91401231053bf2beab980ce7b485121a0c5 Mon Sep 17 00:00:00 2001 From: Yuri Lima Date: Mon, 4 Aug 2025 16:58:39 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20Add=20heartbeat=20tar?= =?UTF-8?q?gets=20configuration=20for=20Prometheus=20monitoring=20and=20sa?= =?UTF-8?q?ve=20to=20new=20JSON=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/hetzner_heartbeat_targets.json | 416 ++++++++++++++++++++++++++ config/prometheus.yaml | 7 + hetzner_discovery.py | 15 + 3 files changed, 438 insertions(+) create mode 100644 config/hetzner_heartbeat_targets.json diff --git a/config/hetzner_heartbeat_targets.json b/config/hetzner_heartbeat_targets.json new file mode 100644 index 0000000..c60478d --- /dev/null +++ b/config/hetzner_heartbeat_targets.json @@ -0,0 +1,416 @@ +[ + { + "targets": [ + "157.90.161.42:9091" + ], + "labels": { + "instance": "PHX-DEV-001.Alpha", + "job": "heartbeat" + } + }, + { + "targets": [ + "167.235.254.4:9091" + ], + "labels": { + "instance": "ANSIBLE-MASTER", + "job": "heartbeat" + } + }, + { + "targets": [ + "116.203.75.215:9091" + ], + "labels": { + "instance": "cts", + "job": "heartbeat" + } + }, + { + "targets": [ + "162.55.52.253:9091" + ], + "labels": { + "instance": "Phx-Yuri", + "job": "heartbeat" + } + }, + { + "targets": [ + "162.55.54.75:9091" + ], + "labels": { + "instance": "benzinger-demo", + "job": "heartbeat" + } + }, + { + "targets": [ + "5.75.183.139:9091" + ], + "labels": { + "instance": "phx-internal", + "job": "heartbeat" + } + }, + { + "targets": [ + "142.132.165.231:9091" + ], + "labels": { + "instance": "trachtenmode-schmid", + "job": "heartbeat" + } + }, + { + "targets": [ + "159.69.200.205:9091" + ], + "labels": { + "instance": "phx-beta", + "job": "heartbeat" + } + }, + { + "targets": [ + "116.203.53.137:9091" + ], + "labels": { + "instance": "lhl", + "job": "heartbeat" + } + }, + { + "targets": [ + "5.75.153.161:9091" + ], + "labels": { + "instance": "Grafana-Prometheus", + "job": "heartbeat" + } + }, + { + "targets": [ + "159.69.44.39:9091" + ], + "labels": { + "instance": "cooper", + "job": "heartbeat" + } + }, + { + "targets": [ + "49.13.165.13:9091" + ], + "labels": { + "instance": "shipxpert", + "job": "heartbeat" + } + }, + { + "targets": [ + "188.245.44.219:9091" + ], + "labels": { + "instance": "sartissohn", + "job": "heartbeat" + } + }, + { + "targets": [ + "116.203.92.218:9091" + ], + "labels": { + "instance": "guntli", + "job": "heartbeat" + } + }, + { + "targets": [ + "94.130.77.57:9091" + ], + "labels": { + "instance": "kolb", + "job": "heartbeat" + } + }, + { + "targets": [ + "116.203.151.20:9091" + ], + "labels": { + "instance": "ried", + "job": "heartbeat" + } + }, + { + "targets": [ + "116.203.68.120:9091" + ], + "labels": { + "instance": "heba", + "job": "heartbeat" + } + }, + { + "targets": [ + "116.203.128.69:9091" + ], + "labels": { + "instance": "eicsoft", + "job": "heartbeat" + } + }, + { + "targets": [ + "159.69.93.252:9091" + ], + "labels": { + "instance": "ck-vechta", + "job": "heartbeat" + } + }, + { + "targets": [ + "116.203.46.171:9091" + ], + "labels": { + "instance": "eeparts", + "job": "heartbeat" + } + }, + { + "targets": [ + "167.235.130.242:9091" + ], + "labels": { + "instance": "big-break-changes", + "job": "heartbeat" + } + }, + { + "targets": [ + "188.245.32.214:9091" + ], + "labels": { + "instance": "PHX-License-Server", + "job": "heartbeat" + } + }, + { + "targets": [ + "138.199.201.35:9091" + ], + "labels": { + "instance": "PHX-SkyNodus", + "job": "heartbeat" + } + }, + { + "targets": [ + "188.245.108.133:9091" + ], + "labels": { + "instance": "bode", + "job": "heartbeat" + } + }, + { + "targets": [ + "168.119.174.181:9091" + ], + "labels": { + "instance": "skr04", + "job": "heartbeat" + } + }, + { + "targets": [ + "91.99.24.53:9091" + ], + "labels": { + "instance": "artwin", + "job": "heartbeat" + } + }, + { + "targets": [ + "168.119.231.96:9091" + ], + "labels": { + "instance": "amplid", + "job": "heartbeat" + } + }, + { + "targets": [ + "116.203.143.63:9091" + ], + "labels": { + "instance": "schwarz", + "job": "heartbeat" + } + }, + { + "targets": [ + "91.99.53.67:9091" + ], + "labels": { + "instance": "skr03", + "job": "heartbeat" + } + }, + { + "targets": [ + "91.99.52.253:9091" + ], + "labels": { + "instance": "moonich", + "job": "heartbeat" + } + }, + { + "targets": [ + "91.99.76.156:9091" + ], + "labels": { + "instance": "comp-sys", + "job": "heartbeat" + } + }, + { + "targets": [ + "78.46.214.224:9091" + ], + "labels": { + "instance": "skr-ch", + "job": "heartbeat" + } + }, + { + "targets": [ + "116.203.18.156:9091" + ], + "labels": { + "instance": "wsoft", + "job": "heartbeat" + } + }, + { + "targets": [ + "91.99.71.224:9091" + ], + "labels": { + "instance": "bzp", + "job": "heartbeat" + } + }, + { + "targets": [ + "49.13.197.152:9091" + ], + "labels": { + "instance": "dss", + "job": "heartbeat" + } + }, + { + "targets": [ + "157.90.228.52:9091" + ], + "labels": { + "instance": "modern", + "job": "heartbeat" + } + }, + { + "targets": [ + "23.88.56.89:9091" + ], + "labels": { + "instance": "burg-itc", + "job": "heartbeat" + } + }, + { + "targets": [ + "91.99.109.3:9091" + ], + "labels": { + "instance": "capricorn-it", + "job": "heartbeat" + } + }, + { + "targets": [ + "91.99.140.34:9091" + ], + "labels": { + "instance": "kundw", + "job": "heartbeat" + } + }, + { + "targets": [ + "128.140.33.105:9091" + ], + "labels": { + "instance": "dieweingoetter", + "job": "heartbeat" + } + }, + { + "targets": [ + "94.130.108.153:9091" + ], + "labels": { + "instance": "Rustdesk-Relay-Server", + "job": "heartbeat" + } + }, + { + "targets": [ + "91.99.226.250:9091" + ], + "labels": { + "instance": "rihnet", + "job": "heartbeat" + } + }, + { + "targets": [ + "49.13.205.27:9091" + ], + "labels": { + "instance": "infoplus", + "job": "heartbeat" + } + }, + { + "targets": [ + "195.201.31.160:9091" + ], + "labels": { + "instance": "ai-it", + "job": "heartbeat" + } + }, + { + "targets": [ + "91.99.143.88:9091" + ], + "labels": { + "instance": "guder", + "job": "heartbeat" + } + }, + { + "targets": [ + "91.107.235.238:9091" + ], + "labels": { + "instance": "testRedisHeathCheck", + "job": "heartbeat" + } + } +] \ No newline at end of file diff --git a/config/prometheus.yaml b/config/prometheus.yaml index c5cd06c..f095c12 100644 --- a/config/prometheus.yaml +++ b/config/prometheus.yaml @@ -19,6 +19,13 @@ scrape_configs: scrape_interval: 2m # Change it carefully, it is a heavy job to keep open and close to check the health of DB connection, there annotation to refactor it and use a new pgDebounce image. scheme: http + - job_name: "heartbeat" + file_sd_configs: + - files: + - "/opt/phx/main/config/hetzner_heartbeat_targets.json" + refresh_interval: 30s + honor_labels: true + # - job_name: "phoenix-system" # file_sd_configs: # - files: diff --git a/hetzner_discovery.py b/hetzner_discovery.py index 6992771..c296e3d 100644 --- a/hetzner_discovery.py +++ b/hetzner_discovery.py @@ -25,6 +25,7 @@ if not HETZNER_API_TOKEN: BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +HEARTBEAT_TARGETS_FILE = os.path.join(BASE_DIR, "config/hetzner_heartbeat_targets.json") PROMETHEUS_TARGETS_FILE = os.path.join(BASE_DIR, "config/hetzner_targets.json") PHX_SYSTEM_FILE = os.path.join(BASE_DIR, "config/phoenix_system_hetzner_targets.json") PHX_WORKER_FILE = os.path.join(BASE_DIR, "config/phoenix_worker_hetzner_targets.json") @@ -95,6 +96,7 @@ def generate_prometheus_sd_config(): error_servers = [] excluded_servers = [] dns_mappings = [] # New list for storing DNS-IP mappings + heartbeat_targets = [] for server in servers: ipv4 = server.get("public_net", {}).get("ipv4", {}).get("ip") @@ -129,6 +131,15 @@ def generate_prometheus_sd_config(): } }) + # Add Pushgateway heartbeat endpoint (default Pushgateway port is 9091) + heartbeat_targets.append({ + "targets": [f"{ipv4}:9091"], # assuming Pushgateway is running on each server + "labels": { + "instance": server_name, + "job": "heartbeat" + } + }) + # Phoenix System metrics (port 3000) # phx_system_targets.append({ # "targets": [f"{ipv4}:3000"], @@ -173,6 +184,10 @@ def generate_prometheus_sd_config(): json.dump(phx_health_exporter_targets, f, indent=4) print(f"✅ phoenix-health-exporter targets saved to {PHX_HEALTH_EXPORTER_FILE}") + with open(HEARTBEAT_TARGETS_FILE, "w") as f: + json.dump(heartbeat_targets, f, indent=4) + print(f"✅ heartbeat targets saved to {HEARTBEAT_TARGETS_FILE}") + # with open(PHX_SYSTEM_FILE, "w") as f: # json.dump(phx_system_targets, f, indent=4) # print(f"✅ phoenix-system targets saved to {PHX_SYSTEM_FILE}")