yurilima/Grafana-Prometheus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: yurilima:7f9f7b1aa40366a993db769e02c598662a54d6b6
Branches Tags
yurilima:main
...
compare: yurilima:f4a6be714b908e5755e4067ee1833605a217c3f6
Branches Tags
yurilima:main

2 Commits
7f9f7b1aa4 ... f4a6be714b

Author SHA1 Message Date
Yuri Lima
f4a6be714b 👌 IMPROVE: Update Hetzner DNS mapping and targets configuration by renaming instances, removing obsolete entries, and adding new targets for enhanced monitoring 2025-08-04 16:58:50 +02:00
Yuri Lima
ddfdb91401 👌 IMPROVE: Add heartbeat targets configuration for Prometheus monitoring and save to new JSON file 2025-08-04 16:58:39 +02:00
6 changed files with 600 additions and 47 deletions
Expand all files Collapse all files

42
config/hetzner_dns_mapping.json
View File

@@ -28,7 +28,7 @@
"ip_address": "142.132.165.231" "ip_address": "142.132.165.231"
}, },
{ {
"dns_name": "phx-beta-rc", "dns_name": "phx-beta",
"ip_address": "159.69.200.205" "ip_address": "159.69.200.205"
}, },
{ {
@@ -139,10 +139,6 @@
"dns_name": "dss", "dns_name": "dss",
"ip_address": "49.13.197.152" "ip_address": "49.13.197.152"
}, },
{
"dns_name": "new-full-test",
"ip_address": "128.140.103.178"
},
{ {
"dns_name": "modern", "dns_name": "modern",
"ip_address": "157.90.228.52" "ip_address": "157.90.228.52"
@@ -152,15 +148,39 @@
"ip_address": "23.88.56.89" "ip_address": "23.88.56.89"
}, },
{ {
"dns_name": "test-ansible-ip-permission", "dns_name": "capricorn-it",
"ip_address": "116.203.73.240" "ip_address": "91.99.109.3"
}, },
{ {
"dns_name": "test-ansible-ip-permission-2", "dns_name": "kundw",
"ip_address": "91.99.29.55" "ip_address": "91.99.140.34"
}, },
{ {
"dns_name": "bpp", "dns_name": "dieweingoetter",
"ip_address": "91.99.107.178" "ip_address": "128.140.33.105"
},
{
"dns_name": "Rustdesk-Relay-Server",
"ip_address": "94.130.108.153"
},
{
"dns_name": "rihnet",
"ip_address": "91.99.226.250"
},
{
"dns_name": "infoplus",
"ip_address": "49.13.205.27"
},
{
"dns_name": "ai-it",
"ip_address": "195.201.31.160"
},
{
"dns_name": "guder",
"ip_address": "91.99.143.88"
},
{
"dns_name": "testRedisHeathCheck",
"ip_address": "91.107.235.238"
} }
] ]

416
config/hetzner_heartbeat_targets.json Normal file
View File

@@ -0,0 +1,416 @@
[
{
"targets": [
"157.90.161.42:9091"
],
"labels": {
"instance": "PHX-DEV-001.Alpha",
"job": "heartbeat"
}
},
{
"targets": [
"167.235.254.4:9091"
],
"labels": {
"instance": "ANSIBLE-MASTER",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.75.215:9091"
],
"labels": {
"instance": "cts",
"job": "heartbeat"
}
},
{
"targets": [
"162.55.52.253:9091"
],
"labels": {
"instance": "Phx-Yuri",
"job": "heartbeat"
}
},
{
"targets": [
"162.55.54.75:9091"
],
"labels": {
"instance": "benzinger-demo",
"job": "heartbeat"
}
},
{
"targets": [
"5.75.183.139:9091"
],
"labels": {
"instance": "phx-internal",
"job": "heartbeat"
}
},
{
"targets": [
"142.132.165.231:9091"
],
"labels": {
"instance": "trachtenmode-schmid",
"job": "heartbeat"
}
},
{
"targets": [
"159.69.200.205:9091"
],
"labels": {
"instance": "phx-beta",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.53.137:9091"
],
"labels": {
"instance": "lhl",
"job": "heartbeat"
}
},
{
"targets": [
"5.75.153.161:9091"
],
"labels": {
"instance": "Grafana-Prometheus",
"job": "heartbeat"
}
},
{
"targets": [
"159.69.44.39:9091"
],
"labels": {
"instance": "cooper",
"job": "heartbeat"
}
},
{
"targets": [
"49.13.165.13:9091"
],
"labels": {
"instance": "shipxpert",
"job": "heartbeat"
}
},
{
"targets": [
"188.245.44.219:9091"
],
"labels": {
"instance": "sartissohn",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.92.218:9091"
],
"labels": {
"instance": "guntli",
"job": "heartbeat"
}
},
{
"targets": [
"94.130.77.57:9091"
],
"labels": {
"instance": "kolb",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.151.20:9091"
],
"labels": {
"instance": "ried",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.68.120:9091"
],
"labels": {
"instance": "heba",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.128.69:9091"
],
"labels": {
"instance": "eicsoft",
"job": "heartbeat"
}
},
{
"targets": [
"159.69.93.252:9091"
],
"labels": {
"instance": "ck-vechta",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.46.171:9091"
],
"labels": {
"instance": "eeparts",
"job": "heartbeat"
}
},
{
"targets": [
"167.235.130.242:9091"
],
"labels": {
"instance": "big-break-changes",
"job": "heartbeat"
}
},
{
"targets": [
"188.245.32.214:9091"
],
"labels": {
"instance": "PHX-License-Server",
"job": "heartbeat"
}
},
{
"targets": [
"138.199.201.35:9091"
],
"labels": {
"instance": "PHX-SkyNodus",
"job": "heartbeat"
}
},
{
"targets": [
"188.245.108.133:9091"
],
"labels": {
"instance": "bode",
"job": "heartbeat"
}
},
{
"targets": [
"168.119.174.181:9091"
],
"labels": {
"instance": "skr04",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.24.53:9091"
],
"labels": {
"instance": "artwin",
"job": "heartbeat"
}
},
{
"targets": [
"168.119.231.96:9091"
],
"labels": {
"instance": "amplid",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.143.63:9091"
],
"labels": {
"instance": "schwarz",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.53.67:9091"
],
"labels": {
"instance": "skr03",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.52.253:9091"
],
"labels": {
"instance": "moonich",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.76.156:9091"
],
"labels": {
"instance": "comp-sys",
"job": "heartbeat"
}
},
{
"targets": [
"78.46.214.224:9091"
],
"labels": {
"instance": "skr-ch",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.18.156:9091"
],
"labels": {
"instance": "wsoft",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.71.224:9091"
],
"labels": {
"instance": "bzp",
"job": "heartbeat"
}
},
{
"targets": [
"49.13.197.152:9091"
],
"labels": {
"instance": "dss",
"job": "heartbeat"
}
},
{
"targets": [
"157.90.228.52:9091"
],
"labels": {
"instance": "modern",
"job": "heartbeat"
}
},
{
"targets": [
"23.88.56.89:9091"
],
"labels": {
"instance": "burg-itc",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.109.3:9091"
],
"labels": {
"instance": "capricorn-it",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.140.34:9091"
],
"labels": {
"instance": "kundw",
"job": "heartbeat"
}
},
{
"targets": [
"128.140.33.105:9091"
],
"labels": {
"instance": "dieweingoetter",
"job": "heartbeat"
}
},
{
"targets": [
"94.130.108.153:9091"
],
"labels": {
"instance": "Rustdesk-Relay-Server",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.226.250:9091"
],
"labels": {
"instance": "rihnet",
"job": "heartbeat"
}
},
{
"targets": [
"49.13.205.27:9091"
],
"labels": {
"instance": "infoplus",
"job": "heartbeat"
}
},
{
"targets": [
"195.201.31.160:9091"
],
"labels": {
"instance": "ai-it",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.143.88:9091"
],
"labels": {
"instance": "guder",
"job": "heartbeat"
}
},
{
"targets": [
"91.107.235.238:9091"
],
"labels": {
"instance": "testRedisHeathCheck",
"job": "heartbeat"
}
}
]

90
config/hetzner_targets.json
View File

@@ -75,7 +75,7 @@
"159.69.200.205:9113" "159.69.200.205:9113"
], ],
"labels": { "labels": {
"instance": "phx-beta-rc", "instance": "phx-beta",
"datacenter": "nbg1" "datacenter": "nbg1"
} }
}, },
@@ -349,16 +349,6 @@
"datacenter": "nbg1" "datacenter": "nbg1"
} }
}, },
{
"targets": [
"128.140.103.178:9100",
"128.140.103.178:9113"
],
"labels": {
"instance": "new-full-test",
"datacenter": "nbg1"
}
},
{ {
"targets": [ "targets": [
"157.90.228.52:9100", "157.90.228.52:9100",
@@ -381,31 +371,91 @@
}, },
{ {
"targets": [ "targets": [
"116.203.73.240:9100", "91.99.109.3:9100",
"116.203.73.240:9113" "91.99.109.3:9113"
], ],
"labels": { "labels": {
"instance": "test-ansible-ip-permission", "instance": "capricorn-it",
"datacenter": "nbg1" "datacenter": "nbg1"
} }
}, },
{ {
"targets": [ "targets": [
"91.99.29.55:9100", "91.99.140.34:9100",
"91.99.29.55:9113" "91.99.140.34:9113"
], ],
"labels": { "labels": {
"instance": "test-ansible-ip-permission-2", "instance": "kundw",
"datacenter": "nbg1" "datacenter": "nbg1"
} }
}, },
{ {
"targets": [ "targets": [
"91.99.107.178:9100", "128.140.33.105:9100",
"91.99.107.178:9113" "128.140.33.105:9113"
], ],
"labels": { "labels": {
"instance": "bpp", "instance": "dieweingoetter",
"datacenter": "nbg1"
}
},
{
"targets": [
"94.130.108.153:9100",
"94.130.108.153:9113"
],
"labels": {
"instance": "Rustdesk-Relay-Server",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.226.250:9100",
"91.99.226.250:9113"
],
"labels": {
"instance": "rihnet",
"datacenter": "nbg1"
}
},
{
"targets": [
"49.13.205.27:9100",
"49.13.205.27:9113"
],
"labels": {
"instance": "infoplus",
"datacenter": "nbg1"
}
},
{
"targets": [
"195.201.31.160:9100",
"195.201.31.160:9113"
],
"labels": {
"instance": "ai-it",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.143.88:9100",
"91.99.143.88:9113"
],
"labels": {
"instance": "guder",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.107.235.238:9100",
"91.107.235.238:9113"
],
"labels": {
"instance": "testRedisHeathCheck",
"datacenter": "nbg1" "datacenter": "nbg1"
} }
} }

77
config/phoenix_health_exporter_hetzner_targets.json
View File

@@ -67,7 +67,7 @@
"159.69.200.205:9800" "159.69.200.205:9800"
], ],
"labels": { "labels": {
"instance": "phx-beta-rc-health-exporter", "instance": "phx-beta-health-exporter",
"datacenter": "nbg1" "datacenter": "nbg1"
} }
}, },
@@ -314,15 +314,6 @@
"datacenter": "nbg1" "datacenter": "nbg1"
} }
}, },
{
"targets": [
"128.140.103.178:9800"
],
"labels": {
"instance": "new-full-test-health-exporter",
"datacenter": "nbg1"
}
},
{ {
"targets": [ "targets": [
"157.90.228.52:9800" "157.90.228.52:9800"
@@ -343,28 +334,82 @@
}, },
{ {
"targets": [ "targets": [
"116.203.73.240:9800" "91.99.109.3:9800"
], ],
"labels": { "labels": {
"instance": "test-ansible-ip-permission-health-exporter", "instance": "capricorn-it-health-exporter",
"datacenter": "nbg1" "datacenter": "nbg1"
} }
}, },
{ {
"targets": [ "targets": [
"91.99.29.55:9800" "91.99.140.34:9800"
], ],
"labels": { "labels": {
"instance": "test-ansible-ip-permission-2-health-exporter", "instance": "kundw-health-exporter",
"datacenter": "nbg1" "datacenter": "nbg1"
} }
}, },
{ {
"targets": [ "targets": [
"91.99.107.178:9800" "128.140.33.105:9800"
], ],
"labels": { "labels": {
"instance": "bpp-health-exporter", "instance": "dieweingoetter-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"94.130.108.153:9800"
],
"labels": {
"instance": "Rustdesk-Relay-Server-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.226.250:9800"
],
"labels": {
"instance": "rihnet-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"49.13.205.27:9800"
],
"labels": {
"instance": "infoplus-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"195.201.31.160:9800"
],
"labels": {
"instance": "ai-it-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.143.88:9800"
],
"labels": {
"instance": "guder-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.107.235.238:9800"
],
"labels": {
"instance": "testRedisHeathCheck-health-exporter",
"datacenter": "nbg1" "datacenter": "nbg1"
} }
} }

7
config/prometheus.yaml
View File

@@ -19,6 +19,13 @@ scrape_configs:
scrape_interval: 2m # Change it carefully, it is a heavy job to keep open and close to check the health of DB connection, there annotation to refactor it and use a new pgDebounce image. scrape_interval: 2m # Change it carefully, it is a heavy job to keep open and close to check the health of DB connection, there annotation to refactor it and use a new pgDebounce image.
scheme: http scheme: http
- job_name: "heartbeat"
file_sd_configs:
- files:
- "/opt/phx/main/config/hetzner_heartbeat_targets.json"
refresh_interval: 30s
honor_labels: true
# - job_name: "phoenix-system" # - job_name: "phoenix-system"
# file_sd_configs: # file_sd_configs:
# - files: # - files:

15
hetzner_discovery.py
View File

@@ -25,6 +25,7 @@ if not HETZNER_API_TOKEN:
BASE_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_DIR = os.path.dirname(os.path.abspath(__file__))
HEARTBEAT_TARGETS_FILE = os.path.join(BASE_DIR, "config/hetzner_heartbeat_targets.json")
PROMETHEUS_TARGETS_FILE = os.path.join(BASE_DIR, "config/hetzner_targets.json") PROMETHEUS_TARGETS_FILE = os.path.join(BASE_DIR, "config/hetzner_targets.json")
PHX_SYSTEM_FILE = os.path.join(BASE_DIR, "config/phoenix_system_hetzner_targets.json") PHX_SYSTEM_FILE = os.path.join(BASE_DIR, "config/phoenix_system_hetzner_targets.json")
PHX_WORKER_FILE = os.path.join(BASE_DIR, "config/phoenix_worker_hetzner_targets.json") PHX_WORKER_FILE = os.path.join(BASE_DIR, "config/phoenix_worker_hetzner_targets.json")
@@ -95,6 +96,7 @@ def generate_prometheus_sd_config():
error_servers = [] error_servers = []
excluded_servers = [] excluded_servers = []
dns_mappings = [] # New list for storing DNS-IP mappings dns_mappings = [] # New list for storing DNS-IP mappings
heartbeat_targets = []
for server in servers: for server in servers:
ipv4 = server.get("public_net", {}).get("ipv4", {}).get("ip") ipv4 = server.get("public_net", {}).get("ipv4", {}).get("ip")
@@ -129,6 +131,15 @@ def generate_prometheus_sd_config():
} }
}) })
# Add Pushgateway heartbeat endpoint (default Pushgateway port is 9091)
heartbeat_targets.append({
"targets": [f"{ipv4}:9091"], # assuming Pushgateway is running on each server
"labels": {
"instance": server_name,
"job": "heartbeat"
}
})
# Phoenix System metrics (port 3000) # Phoenix System metrics (port 3000)
# phx_system_targets.append({ # phx_system_targets.append({
# "targets": [f"{ipv4}:3000"], # "targets": [f"{ipv4}:3000"],
@@ -173,6 +184,10 @@ def generate_prometheus_sd_config():
json.dump(phx_health_exporter_targets, f, indent=4) json.dump(phx_health_exporter_targets, f, indent=4)
print(f"✅ phoenix-health-exporter targets saved to {PHX_HEALTH_EXPORTER_FILE}") print(f"✅ phoenix-health-exporter targets saved to {PHX_HEALTH_EXPORTER_FILE}")
with open(HEARTBEAT_TARGETS_FILE, "w") as f:
json.dump(heartbeat_targets, f, indent=4)
print(f"✅ heartbeat targets saved to {HEARTBEAT_TARGETS_FILE}")
# with open(PHX_SYSTEM_FILE, "w") as f: # with open(PHX_SYSTEM_FILE, "w") as f:
# json.dump(phx_system_targets, f, indent=4) # json.dump(phx_system_targets, f, indent=4)
# print(f"✅ phoenix-system targets saved to {PHX_SYSTEM_FILE}") # print(f"✅ phoenix-system targets saved to {PHX_SYSTEM_FILE}")