yurilima/Grafana-Prometheus
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: yurilima:938eaf2f5ff1346f35bd22a1b13f9dc23c78db46
Branches Tags
yurilima:main
..
compare: yurilima:main
Branches Tags
yurilima:main

11 Commits
938eaf2f5f ... main

Author SHA1 Message Date
Yuri Lima
f4a6be714b 👌 IMPROVE: Update Hetzner DNS mapping and targets configuration by renaming instances, removing obsolete entries, and adding new targets for enhanced monitoring 2025-08-04 16:58:50 +02:00
Yuri Lima
ddfdb91401 👌 IMPROVE: Add heartbeat targets configuration for Prometheus monitoring and save to new JSON file 2025-08-04 16:58:39 +02:00
Yuri Lima
7f9f7b1aa4 👌 IMPROVE: Add new health exporter targets configuration and update Prometheus to include the new health exporter job 2025-05-23 09:30:33 +00:00
Yuri Lima
945cae3908 👌 IMPROVE: Update Hetzner configuration files and scripts to include new targets, enhance health check metrics, and organize file paths for better maintainability 2025-05-22 10:28:08 +00:00
Yuri Lima
61d056dcab 👌 IMPROVE: Enhance Loki configuration with additional server settings and limits configuration 2025-05-19 11:13:48 +00:00
Yuri Lima
011aa2d613 👌 IMPROVE: Update Loki service configuration to expand environment variables and fix healthcheck URL 2025-05-19 11:13:31 +00:00
Yuri Lima
cab9ccf1af 👌 IMPROVE: Update Hetzner DNS mapping and targets with new instances and IP addresses 2025-05-19 10:18:31 +00:00
Yuri Lima
50a4670c53 👌 IMPROVE: Refactor Grafana environment variables for consistency and add resource limits for Loki service 2025-05-19 10:18:18 +00:00
Yuri Lima
38a2439b98 👌 IMPROVE: Fix Prometheus reload URL to use the correct port 2025-05-19 10:18:03 +00:00
Yuri Lima
43ba5e60b7 👌 IMPROVE: Add additional metrics endpoint for Hetzner server targets in configuration 2025-05-13 14:21:38 +00:00
Yuri Lima
89519ef1f3 👌 IMPROVE: Update Hetzner server targets in discovery script to include additional metrics endpoint 2025-05-13 14:18:19 +00:00
11 changed files with 1221 additions and 120 deletions
Expand all files Collapse all files

54
config/hetzner_dns_mapping.json
View File

@@ -28,15 +28,7 @@
"ip_address": "142.132.165.231" "ip_address": "142.132.165.231"
}, },
{ {
"dns_name": "teamcity", "dns_name": "phx-beta",
"ip_address": "23.88.107.109"
},
{
"dns_name": "gitea",
"ip_address": "49.13.146.138"
},
{
"dns_name": "phx-beta-rc",
"ip_address": "159.69.200.205" "ip_address": "159.69.200.205"
}, },
{ {
@@ -148,7 +140,47 @@
"ip_address": "49.13.197.152" "ip_address": "49.13.197.152"
}, },
{ {
"dns_name": "phx-healthcheck-phoenixApp-pgAdmin-Test-2", "dns_name": "modern",
"ip_address": "116.203.103.215" "ip_address": "157.90.228.52"
},
{
"dns_name": "burg-itc",
"ip_address": "23.88.56.89"
},
{
"dns_name": "capricorn-it",
"ip_address": "91.99.109.3"
},
{
"dns_name": "kundw",
"ip_address": "91.99.140.34"
},
{
"dns_name": "dieweingoetter",
"ip_address": "128.140.33.105"
},
{
"dns_name": "Rustdesk-Relay-Server",
"ip_address": "94.130.108.153"
},
{
"dns_name": "rihnet",
"ip_address": "91.99.226.250"
},
{
"dns_name": "infoplus",
"ip_address": "49.13.205.27"
},
{
"dns_name": "ai-it",
"ip_address": "195.201.31.160"
},
{
"dns_name": "guder",
"ip_address": "91.99.143.88"
},
{
"dns_name": "testRedisHeathCheck",
"ip_address": "91.107.235.238"
} }
] ]

10
config/hetzner_excluded_servers.json
View File

@@ -1,7 +1,13 @@
[ [
{ {
"server_id": 52530041, "server_id": 41663080,
"name": "docuvita", "name": "teamcity",
"datacenter": "nbg1",
"reason": "Darklisted server"
},
{
"server_id": 41739626,
"name": "gitea",
"datacenter": "nbg1", "datacenter": "nbg1",
"reason": "Darklisted server" "reason": "Darklisted server"
} }

416
config/hetzner_heartbeat_targets.json Normal file
View File

@@ -0,0 +1,416 @@
[
{
"targets": [
"157.90.161.42:9091"
],
"labels": {
"instance": "PHX-DEV-001.Alpha",
"job": "heartbeat"
}
},
{
"targets": [
"167.235.254.4:9091"
],
"labels": {
"instance": "ANSIBLE-MASTER",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.75.215:9091"
],
"labels": {
"instance": "cts",
"job": "heartbeat"
}
},
{
"targets": [
"162.55.52.253:9091"
],
"labels": {
"instance": "Phx-Yuri",
"job": "heartbeat"
}
},
{
"targets": [
"162.55.54.75:9091"
],
"labels": {
"instance": "benzinger-demo",
"job": "heartbeat"
}
},
{
"targets": [
"5.75.183.139:9091"
],
"labels": {
"instance": "phx-internal",
"job": "heartbeat"
}
},
{
"targets": [
"142.132.165.231:9091"
],
"labels": {
"instance": "trachtenmode-schmid",
"job": "heartbeat"
}
},
{
"targets": [
"159.69.200.205:9091"
],
"labels": {
"instance": "phx-beta",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.53.137:9091"
],
"labels": {
"instance": "lhl",
"job": "heartbeat"
}
},
{
"targets": [
"5.75.153.161:9091"
],
"labels": {
"instance": "Grafana-Prometheus",
"job": "heartbeat"
}
},
{
"targets": [
"159.69.44.39:9091"
],
"labels": {
"instance": "cooper",
"job": "heartbeat"
}
},
{
"targets": [
"49.13.165.13:9091"
],
"labels": {
"instance": "shipxpert",
"job": "heartbeat"
}
},
{
"targets": [
"188.245.44.219:9091"
],
"labels": {
"instance": "sartissohn",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.92.218:9091"
],
"labels": {
"instance": "guntli",
"job": "heartbeat"
}
},
{
"targets": [
"94.130.77.57:9091"
],
"labels": {
"instance": "kolb",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.151.20:9091"
],
"labels": {
"instance": "ried",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.68.120:9091"
],
"labels": {
"instance": "heba",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.128.69:9091"
],
"labels": {
"instance": "eicsoft",
"job": "heartbeat"
}
},
{
"targets": [
"159.69.93.252:9091"
],
"labels": {
"instance": "ck-vechta",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.46.171:9091"
],
"labels": {
"instance": "eeparts",
"job": "heartbeat"
}
},
{
"targets": [
"167.235.130.242:9091"
],
"labels": {
"instance": "big-break-changes",
"job": "heartbeat"
}
},
{
"targets": [
"188.245.32.214:9091"
],
"labels": {
"instance": "PHX-License-Server",
"job": "heartbeat"
}
},
{
"targets": [
"138.199.201.35:9091"
],
"labels": {
"instance": "PHX-SkyNodus",
"job": "heartbeat"
}
},
{
"targets": [
"188.245.108.133:9091"
],
"labels": {
"instance": "bode",
"job": "heartbeat"
}
},
{
"targets": [
"168.119.174.181:9091"
],
"labels": {
"instance": "skr04",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.24.53:9091"
],
"labels": {
"instance": "artwin",
"job": "heartbeat"
}
},
{
"targets": [
"168.119.231.96:9091"
],
"labels": {
"instance": "amplid",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.143.63:9091"
],
"labels": {
"instance": "schwarz",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.53.67:9091"
],
"labels": {
"instance": "skr03",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.52.253:9091"
],
"labels": {
"instance": "moonich",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.76.156:9091"
],
"labels": {
"instance": "comp-sys",
"job": "heartbeat"
}
},
{
"targets": [
"78.46.214.224:9091"
],
"labels": {
"instance": "skr-ch",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.18.156:9091"
],
"labels": {
"instance": "wsoft",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.71.224:9091"
],
"labels": {
"instance": "bzp",
"job": "heartbeat"
}
},
{
"targets": [
"49.13.197.152:9091"
],
"labels": {
"instance": "dss",
"job": "heartbeat"
}
},
{
"targets": [
"157.90.228.52:9091"
],
"labels": {
"instance": "modern",
"job": "heartbeat"
}
},
{
"targets": [
"23.88.56.89:9091"
],
"labels": {
"instance": "burg-itc",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.109.3:9091"
],
"labels": {
"instance": "capricorn-it",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.140.34:9091"
],
"labels": {
"instance": "kundw",
"job": "heartbeat"
}
},
{
"targets": [
"128.140.33.105:9091"
],
"labels": {
"instance": "dieweingoetter",
"job": "heartbeat"
}
},
{
"targets": [
"94.130.108.153:9091"
],
"labels": {
"instance": "Rustdesk-Relay-Server",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.226.250:9091"
],
"labels": {
"instance": "rihnet",
"job": "heartbeat"
}
},
{
"targets": [
"49.13.205.27:9091"
],
"labels": {
"instance": "infoplus",
"job": "heartbeat"
}
},
{
"targets": [
"195.201.31.160:9091"
],
"labels": {
"instance": "ai-it",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.143.88:9091"
],
"labels": {
"instance": "guder",
"job": "heartbeat"
}
},
{
"targets": [
"91.107.235.238:9091"
],
"labels": {
"instance": "testRedisHeathCheck",
"job": "heartbeat"
}
}
]

230
config/hetzner_targets.json
View File

@@ -1,7 +1,8 @@
[ [
{ {
"targets": [ "targets": [
"157.90.161.42:9100" "157.90.161.42:9100",
"157.90.161.42:9113"
], ],
"labels": { "labels": {
"instance": "PHX-DEV-001.Alpha", "instance": "PHX-DEV-001.Alpha",
@@ -10,7 +11,8 @@
}, },
{ {
"targets": [ "targets": [
"167.235.254.4:9100" "167.235.254.4:9100",
"167.235.254.4:9113"
], ],
"labels": { "labels": {
"instance": "ANSIBLE-MASTER", "instance": "ANSIBLE-MASTER",
@@ -19,7 +21,8 @@
}, },
{ {
"targets": [ "targets": [
"116.203.75.215:9100" "116.203.75.215:9100",
"116.203.75.215:9113"
], ],
"labels": { "labels": {
"instance": "cts", "instance": "cts",
@@ -28,7 +31,8 @@
}, },
{ {
"targets": [ "targets": [
"162.55.52.253:9100" "162.55.52.253:9100",
"162.55.52.253:9113"
], ],
"labels": { "labels": {
"instance": "Phx-Yuri", "instance": "Phx-Yuri",
@@ -37,7 +41,8 @@
}, },
{ {
"targets": [ "targets": [
"162.55.54.75:9100" "162.55.54.75:9100",
"162.55.54.75:9113"
], ],
"labels": { "labels": {
"instance": "benzinger-demo", "instance": "benzinger-demo",
@@ -46,7 +51,8 @@
}, },
{ {
"targets": [ "targets": [
"5.75.183.139:9100" "5.75.183.139:9100",
"5.75.183.139:9113"
], ],
"labels": { "labels": {
"instance": "phx-internal", "instance": "phx-internal",
@@ -55,7 +61,8 @@
}, },
{ {
"targets": [ "targets": [
"142.132.165.231:9100" "142.132.165.231:9100",
"142.132.165.231:9113"
], ],
"labels": { "labels": {
"instance": "trachtenmode-schmid", "instance": "trachtenmode-schmid",
@@ -64,34 +71,18 @@
}, },
{ {
"targets": [ "targets": [
"23.88.107.109:9100" "159.69.200.205:9100",
"159.69.200.205:9113"
], ],
"labels": { "labels": {
"instance": "teamcity", "instance": "phx-beta",
"datacenter": "nbg1" "datacenter": "nbg1"
} }
}, },
{ {
"targets": [ "targets": [
"49.13.146.138:9100" "116.203.53.137:9100",
], "116.203.53.137:9113"
"labels": {
"instance": "gitea",
"datacenter": "nbg1"
}
},
{
"targets": [
"159.69.200.205:9100"
],
"labels": {
"instance": "phx-beta-rc",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.53.137:9100"
], ],
"labels": { "labels": {
"instance": "lhl", "instance": "lhl",
@@ -100,7 +91,8 @@
}, },
{ {
"targets": [ "targets": [
"5.75.153.161:9100" "5.75.153.161:9100",
"5.75.153.161:9113"
], ],
"labels": { "labels": {
"instance": "Grafana-Prometheus", "instance": "Grafana-Prometheus",
@@ -109,7 +101,8 @@
}, },
{ {
"targets": [ "targets": [
"159.69.44.39:9100" "159.69.44.39:9100",
"159.69.44.39:9113"
], ],
"labels": { "labels": {
"instance": "cooper", "instance": "cooper",
@@ -118,7 +111,8 @@
}, },
{ {
"targets": [ "targets": [
"49.13.165.13:9100" "49.13.165.13:9100",
"49.13.165.13:9113"
], ],
"labels": { "labels": {
"instance": "shipxpert", "instance": "shipxpert",
@@ -127,7 +121,8 @@
}, },
{ {
"targets": [ "targets": [
"188.245.44.219:9100" "188.245.44.219:9100",
"188.245.44.219:9113"
], ],
"labels": { "labels": {
"instance": "sartissohn", "instance": "sartissohn",
@@ -136,7 +131,8 @@
}, },
{ {
"targets": [ "targets": [
"116.203.92.218:9100" "116.203.92.218:9100",
"116.203.92.218:9113"
], ],
"labels": { "labels": {
"instance": "guntli", "instance": "guntli",
@@ -145,7 +141,8 @@
}, },
{ {
"targets": [ "targets": [
"94.130.77.57:9100" "94.130.77.57:9100",
"94.130.77.57:9113"
], ],
"labels": { "labels": {
"instance": "kolb", "instance": "kolb",
@@ -154,7 +151,8 @@
}, },
{ {
"targets": [ "targets": [
"116.203.151.20:9100" "116.203.151.20:9100",
"116.203.151.20:9113"
], ],
"labels": { "labels": {
"instance": "ried", "instance": "ried",
@@ -163,7 +161,8 @@
}, },
{ {
"targets": [ "targets": [
"116.203.68.120:9100" "116.203.68.120:9100",
"116.203.68.120:9113"
], ],
"labels": { "labels": {
"instance": "heba", "instance": "heba",
@@ -172,7 +171,8 @@
}, },
{ {
"targets": [ "targets": [
"116.203.128.69:9100" "116.203.128.69:9100",
"116.203.128.69:9113"
], ],
"labels": { "labels": {
"instance": "eicsoft", "instance": "eicsoft",
@@ -181,7 +181,8 @@
}, },
{ {
"targets": [ "targets": [
"159.69.93.252:9100" "159.69.93.252:9100",
"159.69.93.252:9113"
], ],
"labels": { "labels": {
"instance": "ck-vechta", "instance": "ck-vechta",
@@ -190,7 +191,8 @@
}, },
{ {
"targets": [ "targets": [
"116.203.46.171:9100" "116.203.46.171:9100",
"116.203.46.171:9113"
], ],
"labels": { "labels": {
"instance": "eeparts", "instance": "eeparts",
@@ -199,7 +201,8 @@
}, },
{ {
"targets": [ "targets": [
"167.235.130.242:9100" "167.235.130.242:9100",
"167.235.130.242:9113"
], ],
"labels": { "labels": {
"instance": "big-break-changes", "instance": "big-break-changes",
@@ -208,7 +211,8 @@
}, },
{ {
"targets": [ "targets": [
"188.245.32.214:9100" "188.245.32.214:9100",
"188.245.32.214:9113"
], ],
"labels": { "labels": {
"instance": "PHX-License-Server", "instance": "PHX-License-Server",
@@ -217,7 +221,8 @@
}, },
{ {
"targets": [ "targets": [
"138.199.201.35:9100" "138.199.201.35:9100",
"138.199.201.35:9113"
], ],
"labels": { "labels": {
"instance": "PHX-SkyNodus", "instance": "PHX-SkyNodus",
@@ -226,7 +231,8 @@
}, },
{ {
"targets": [ "targets": [
"188.245.108.133:9100" "188.245.108.133:9100",
"188.245.108.133:9113"
], ],
"labels": { "labels": {
"instance": "bode", "instance": "bode",
@@ -235,7 +241,8 @@
}, },
{ {
"targets": [ "targets": [
"168.119.174.181:9100" "168.119.174.181:9100",
"168.119.174.181:9113"
], ],
"labels": { "labels": {
"instance": "skr04", "instance": "skr04",
@@ -244,7 +251,8 @@
}, },
{ {
"targets": [ "targets": [
"91.99.24.53:9100" "91.99.24.53:9100",
"91.99.24.53:9113"
], ],
"labels": { "labels": {
"instance": "artwin", "instance": "artwin",
@@ -253,7 +261,8 @@
}, },
{ {
"targets": [ "targets": [
"168.119.231.96:9100" "168.119.231.96:9100",
"168.119.231.96:9113"
], ],
"labels": { "labels": {
"instance": "amplid", "instance": "amplid",
@@ -262,7 +271,8 @@
}, },
{ {
"targets": [ "targets": [
"116.203.143.63:9100" "116.203.143.63:9100",
"116.203.143.63:9113"
], ],
"labels": { "labels": {
"instance": "schwarz", "instance": "schwarz",
@@ -271,7 +281,8 @@
}, },
{ {
"targets": [ "targets": [
"91.99.53.67:9100" "91.99.53.67:9100",
"91.99.53.67:9113"
], ],
"labels": { "labels": {
"instance": "skr03", "instance": "skr03",
@@ -280,7 +291,8 @@
}, },
{ {
"targets": [ "targets": [
"91.99.52.253:9100" "91.99.52.253:9100",
"91.99.52.253:9113"
], ],
"labels": { "labels": {
"instance": "moonich", "instance": "moonich",
@@ -289,7 +301,8 @@
}, },
{ {
"targets": [ "targets": [
"91.99.76.156:9100" "91.99.76.156:9100",
"91.99.76.156:9113"
], ],
"labels": { "labels": {
"instance": "comp-sys", "instance": "comp-sys",
@@ -298,7 +311,8 @@
}, },
{ {
"targets": [ "targets": [
"78.46.214.224:9100" "78.46.214.224:9100",
"78.46.214.224:9113"
], ],
"labels": { "labels": {
"instance": "skr-ch", "instance": "skr-ch",
@@ -307,7 +321,8 @@
}, },
{ {
"targets": [ "targets": [
"116.203.18.156:9100" "116.203.18.156:9100",
"116.203.18.156:9113"
], ],
"labels": { "labels": {
"instance": "wsoft", "instance": "wsoft",
@@ -316,7 +331,8 @@
}, },
{ {
"targets": [ "targets": [
"91.99.71.224:9100" "91.99.71.224:9100",
"91.99.71.224:9113"
], ],
"labels": { "labels": {
"instance": "bzp", "instance": "bzp",
@@ -325,7 +341,8 @@
}, },
{ {
"targets": [ "targets": [
"49.13.197.152:9100" "49.13.197.152:9100",
"49.13.197.152:9113"
], ],
"labels": { "labels": {
"instance": "dss", "instance": "dss",
@@ -334,10 +351,111 @@
}, },
{ {
"targets": [ "targets": [
"116.203.103.215:9100" "157.90.228.52:9100",
"157.90.228.52:9113"
], ],
"labels": { "labels": {
"instance": "phx-healthcheck-phoenixApp-pgAdmin-Test-2", "instance": "modern",
"datacenter": "nbg1"
}
},
{
"targets": [
"23.88.56.89:9100",
"23.88.56.89:9113"
],
"labels": {
"instance": "burg-itc",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.109.3:9100",
"91.99.109.3:9113"
],
"labels": {
"instance": "capricorn-it",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.140.34:9100",
"91.99.140.34:9113"
],
"labels": {
"instance": "kundw",
"datacenter": "nbg1"
}
},
{
"targets": [
"128.140.33.105:9100",
"128.140.33.105:9113"
],
"labels": {
"instance": "dieweingoetter",
"datacenter": "nbg1"
}
},
{
"targets": [
"94.130.108.153:9100",
"94.130.108.153:9113"
],
"labels": {
"instance": "Rustdesk-Relay-Server",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.226.250:9100",
"91.99.226.250:9113"
],
"labels": {
"instance": "rihnet",
"datacenter": "nbg1"
}
},
{
"targets": [
"49.13.205.27:9100",
"49.13.205.27:9113"
],
"labels": {
"instance": "infoplus",
"datacenter": "nbg1"
}
},
{
"targets": [
"195.201.31.160:9100",
"195.201.31.160:9113"
],
"labels": {
"instance": "ai-it",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.143.88:9100",
"91.99.143.88:9113"
],
"labels": {
"instance": "guder",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.107.235.238:9100",
"91.107.235.238:9113"
],
"labels": {
"instance": "testRedisHeathCheck",
"datacenter": "nbg1" "datacenter": "nbg1"
} }
} }

25
config/loki-config.yaml
View File

@@ -1,8 +1,12 @@
---
auth_enabled: false auth_enabled: false
server: server:
http_listen_port: 3100 http_listen_port: 3100
http_server_read_timeout: 1m
http_server_write_timeout: 1m
grpc_listen_port: 9095
log_level: info
log_format: logfmt
common: common:
instance_addr: 127.0.0.1 instance_addr: 127.0.0.1
@@ -14,7 +18,15 @@ common:
replication_factor: 1 replication_factor: 1
ring: ring:
kvstore: kvstore:
store: inmemory store: inmemory # ✅ Use inmemory for single-node setups
limits_config:
reject_old_samples: true
reject_old_samples_max_age: 168h
ingestion_rate_mb: 10
ingestion_burst_size_mb: 20
max_query_parallelism: 8
max_query_lookback: 168h
schema_config: schema_config:
configs: configs:
@@ -26,5 +38,12 @@ schema_config:
prefix: index_ prefix: index_
period: 24h period: 24h
table_manager:
retention_deletes_enabled: true
retention_period: 168h
ruler: ruler:
alertmanager_url: http://localhost:9093 alertmanager_url: http://localhost:9093
rule_path: /loki/rules
enable_alertmanager_v2: true
enable_api: true

416
config/phoenix_health_exporter_hetzner_targets.json Normal file
View File

@@ -0,0 +1,416 @@
[
{
"targets": [
"157.90.161.42:9800"
],
"labels": {
"instance": "PHX-DEV-001.Alpha-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"167.235.254.4:9800"
],
"labels": {
"instance": "ANSIBLE-MASTER-health-exporter",
"datacenter": "fsn1"
}
},
{
"targets": [
"116.203.75.215:9800"
],
"labels": {
"instance": "cts-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"162.55.52.253:9800"
],
"labels": {
"instance": "Phx-Yuri-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"162.55.54.75:9800"
],
"labels": {
"instance": "benzinger-demo-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"5.75.183.139:9800"
],
"labels": {
"instance": "phx-internal-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"142.132.165.231:9800"
],
"labels": {
"instance": "trachtenmode-schmid-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"159.69.200.205:9800"
],
"labels": {
"instance": "phx-beta-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.53.137:9800"
],
"labels": {
"instance": "lhl-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"5.75.153.161:9800"
],
"labels": {
"instance": "Grafana-Prometheus-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"159.69.44.39:9800"
],
"labels": {
"instance": "cooper-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"49.13.165.13:9800"
],
"labels": {
"instance": "shipxpert-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"188.245.44.219:9800"
],
"labels": {
"instance": "sartissohn-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.92.218:9800"
],
"labels": {
"instance": "guntli-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"94.130.77.57:9800"
],
"labels": {
"instance": "kolb-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.151.20:9800"
],
"labels": {
"instance": "ried-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.68.120:9800"
],
"labels": {
"instance": "heba-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.128.69:9800"
],
"labels": {
"instance": "eicsoft-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"159.69.93.252:9800"
],
"labels": {
"instance": "ck-vechta-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.46.171:9800"
],
"labels": {
"instance": "eeparts-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"167.235.130.242:9800"
],
"labels": {
"instance": "big-break-changes-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"188.245.32.214:9800"
],
"labels": {
"instance": "PHX-License-Server-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"138.199.201.35:9800"
],
"labels": {
"instance": "PHX-SkyNodus-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"188.245.108.133:9800"
],
"labels": {
"instance": "bode-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"168.119.174.181:9800"
],
"labels": {
"instance": "skr04-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.24.53:9800"
],
"labels": {
"instance": "artwin-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"168.119.231.96:9800"
],
"labels": {
"instance": "amplid-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.143.63:9800"
],
"labels": {
"instance": "schwarz-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.53.67:9800"
],
"labels": {
"instance": "skr03-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.52.253:9800"
],
"labels": {
"instance": "moonich-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.76.156:9800"
],
"labels": {
"instance": "comp-sys-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"78.46.214.224:9800"
],
"labels": {
"instance": "skr-ch-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.18.156:9800"
],
"labels": {
"instance": "wsoft-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.71.224:9800"
],
"labels": {
"instance": "bzp-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"49.13.197.152:9800"
],
"labels": {
"instance": "dss-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"157.90.228.52:9800"
],
"labels": {
"instance": "modern-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"23.88.56.89:9800"
],
"labels": {
"instance": "burg-itc-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.109.3:9800"
],
"labels": {
"instance": "capricorn-it-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.140.34:9800"
],
"labels": {
"instance": "kundw-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"128.140.33.105:9800"
],
"labels": {
"instance": "dieweingoetter-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"94.130.108.153:9800"
],
"labels": {
"instance": "Rustdesk-Relay-Server-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.226.250:9800"
],
"labels": {
"instance": "rihnet-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"49.13.205.27:9800"
],
"labels": {
"instance": "infoplus-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"195.201.31.160:9800"
],
"labels": {
"instance": "ai-it-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.143.88:9800"
],
"labels": {
"instance": "guder-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.107.235.238:9800"
],
"labels": {
"instance": "testRedisHeathCheck-health-exporter",
"datacenter": "nbg1"
}
}
]

1
config/phoenix_system_hetzner_targets.json Normal file
View File

@@ -0,0 +1 @@
[]

1
config/phoenix_worker_hetzner_targets.json Normal file
View File

@@ -0,0 +1 @@
[]

50
config/prometheus.yaml
View File

@@ -1,29 +1,43 @@
global: global:
scrape_interval: 15s # Default for all jobs unless overridden scrape_interval: 15s # ⏱️ Default interval Prometheus uses to scrape metrics from all targets, unless overridden in specific jobs.
scrape_configs: scrape_configs:
# Existing Hetzner Dynamic Configuration # Existing Hetzner Dynamic Configuration
- job_name: "hetzner-dynamic" - job_name: "hetzner-dynamic" # 🏷️ Logical name for the scrape job (used in metrics as job="hetzner-dynamic").
scheme: http file_sd_configs: # 📂 Enable file-based service discovery. Prometheus will look at files to dynamically load targets.
- files:
- "/opt/phx/main/config/hetzner_targets.json" # 📁 JSON file containing the list of targets (e.g. IPs, ports, labels).
refresh_interval: 30s # 🔁 How often Prometheus checks the file for changes and updates the target list.
scrape_interval: 5s # ⏱️ Override global default: scrape metrics from each target in this job every x seconds.
scheme: http # 🌐 Use plain HTTP when scraping the targets (not HTTPS).
- job_name: "phoenix-health-exporter"
file_sd_configs: file_sd_configs:
- files: - files:
- "/opt/phx/main/config/hetzner_targets.json" - "/opt/phx/main/config/phoenix_health_exporter_hetzner_targets.json"
refresh_interval: 30s refresh_interval: 30s
scrape_interval: 5s # Custom interval for Hetzner servers scrape_interval: 2m # Change it carefully, it is a heavy job to keep open and close to check the health of DB connection, there annotation to refactor it and use a new pgDebounce image.
scheme: http
# Blackbox Exporter with Dynamic File SD - job_name: "heartbeat"
# - job_name: 'blackbox-dynamic' file_sd_configs:
# metrics_path: /probe - files:
# params: - "/opt/phx/main/config/hetzner_heartbeat_targets.json"
# module: [http_2xx] # HTTP check to see if target is up refresh_interval: 30s
honor_labels: true
# - job_name: "phoenix-system"
# file_sd_configs: # file_sd_configs:
# - files: # - files:
# - "/opt/phx/main/config/blackbox_targets.json" # Dynamic file like hetzner_targets.json # - "/opt/phx/main/config/phoenix_system_hetzner_targets.json"
# refresh_interval: 30s # refresh_interval: 30s
# relabel_configs: # scrape_interval: 5s
# - source_labels: [__address__] # scheme: http
# target_label: __param_target
# - source_labels: [__param_target] # - job_name: "phoenix-worker"
# target_label: instance # file_sd_configs:
# - target_label: __address__ # - files:
# replacement: blackbox_exporter:9115 # Blackbox Exporter container/service # - "/opt/phx/main/config/phoenix_worker_hetzner_targets.json"
# refresh_interval: 30s
# scrape_interval: 5s
# scheme: http

51
docker-compose.yml
View File

@@ -14,24 +14,26 @@ services:
image: docker.io/grafana/grafana-oss:12.0.0 image: docker.io/grafana/grafana-oss:12.0.0
container_name: grafana container_name: grafana
environment: environment:
- GF_SECURITY_ADMIN_PASSWORD=admin GF_PATHS_PROVISIONING: /var/lib/grafana/provisioning/
GF_SECURITY_ADMIN_PASSWORD: admin
# - GF_SERVER_DOMAIN=%(protocol)s://%(domain)s:%(http_port)s/ # - GF_SERVER_DOMAIN=%(protocol)s://%(domain)s:%(http_port)s/
# - GF_SERVER_ROOT_URL=%(protocol)s://%(domain)s/ # - GF_SERVER_ROOT_URL=%(protocol)s://%(domain)s/
- GF_SERVER_DOMAIN=grafana.phx-erp.de GF_SERVER_DOMAIN: grafana.phx-erp.de
- GF_SERVER_PROTOCOL=https GF_SERVER_PROTOCOL: https
# - GF_SERVER_HTTP_PORT=3000 # - GF_SERVER_HTTP_PORT=3000
- GF_LOG_LEVEL=debug GF_LOG_LEVEL: debug
- GF_SMTP_ENABLED=true GF_SMTP_ENABLED: true
- GF_SMTP_FROM_ADDRESS=admin@phx-erp.de GF_SMTP_FROM_ADDRESS: admin@phx-erp.de
- GF_SMTP_FROM_NAME=Grafana GF_SMTP_FROM_NAME: Grafana
- GF_SMTP_HOST=mail.phx-erp.de:465 GF_SMTP_HOST: mail.phx-erp.de:465
- GF_SMTP_PASSWORD=0rB0@et68 GF_SMTP_PASSWORD: 0rB0@et68
- GF_SMTP_USER=yuri.lima@phx-erp.de GF_SMTP_USER: yuri.lima@phx-erp.de
- GF_SMTP_STARTTLS_POLICY=false GF_SMTP_STARTTLS_POLICY: false
- GF_SMTP_SKIP_VERIFY=true GF_SMTP_SKIP_VERIFY: true
- GF_SMTP_ENABLE_TRACING=false GF_SMTP_ENABLE_TRACING: false
volumes: volumes:
- grafana-data:/var/lib/grafana - grafana-data:/var/lib/grafana
- ./config/provisioning:/var/lib/grafana/provisioning/
restart: unless-stopped restart: unless-stopped
networks: networks:
- grafana-prometheus - grafana-prometheus
@@ -41,8 +43,8 @@ services:
image: docker.io/prom/prometheus:v3.1.0 image: docker.io/prom/prometheus:v3.1.0
container_name: prometheus container_name: prometheus
user: "65534:65534" user: "65534:65534"
ports: # ports:
- "9090:9090" # - "9090:9090"
command: command:
- "--config.file=/etc/prometheus/prometheus.yaml" - "--config.file=/etc/prometheus/prometheus.yaml"
- "--web.config.file=/etc/prometheus/web-config.yaml" - "--web.config.file=/etc/prometheus/web-config.yaml"
@@ -63,11 +65,11 @@ services:
loki: loki:
container_name: loki container_name: loki
image: docker.io/grafana/loki:3.5.0 image: docker.io/grafana/loki:3.5.0
command: "-config.file=/etc/loki/config.yaml" # Tells Grafana Loki to start with the config file. command:
- "-config.file=/etc/loki/config.yaml" # Tells Grafana Loki to start with the config file.
- "-config.expand-env=true"
ports: ports:
# --> (Optional) Remove when using traefik...
- "3100:3100" - "3100:3100"
# <--
volumes: volumes:
- ./config/loki-config.yaml:/etc/loki/config.yaml:ro - ./config/loki-config.yaml:/etc/loki/config.yaml:ro
- data_loki:/loki:rw - data_loki:/loki:rw
@@ -75,7 +77,7 @@ services:
- grafana-prometheus - grafana-prometheus
restart: unless-stopped restart: unless-stopped
healthcheck: healthcheck:
test: ["CMD", "wget", "--spider", "-q", "http://localhost:3100/ready"] test: ["CMD", "wget", "--spider", "-q", "http://loki:3100/ready"]
interval: 10s # check every 10 seconds interval: 10s # check every 10 seconds
timeout: 5s # wait max 5 seconds per check timeout: 5s # wait max 5 seconds per check
retries: 5 # mark as unhealthy after 5 failures retries: 5 # mark as unhealthy after 5 failures
@@ -83,8 +85,7 @@ services:
https_portal: https_portal:
container_name: https_portal container_name: https_portal
image: "steveltn/https-portal:1.21" image: "steveltn/https-portal:1.21"
restart: unless-stopped restart: always
user: "root"
networks: networks:
- grafana-prometheus - grafana-prometheus
ports: ports:
@@ -98,10 +99,16 @@ services:
CLIENT_MAX_BODY_SIZE: 0 CLIENT_MAX_BODY_SIZE: 0
# FORCE_RENEW: 'true' # FORCE_RENEW: 'true'
DOMAINS: 'grafana.phx-erp.de -> https://grafana:3000, prometheus.phx-erp.de -> https://prometheus:9090' DOMAINS: 'grafana.phx-erp.de -> https://grafana:3000, prometheus.phx-erp.de -> https://prometheus:9090'
# loki-logs.phx-erp.de -> http://localhost:3100
volumes: volumes:
- ./https_portal/data:/var/lib/https-portal # ssl_certs, vhost.d, htdocs - ./https_portal/data:/var/lib/https-portal # ssl_certs, vhost.d, htdocs
- ./https_portal/log:/var/log/nginx # nginx logs - ./https_portal/log:/var/log/nginx # nginx logs
depends_on: depends_on:
- prometheus - prometheus
- grafana - grafana
- loki - loki
deploy:
resources:
limits:
cpus: '0.50'
memory: 512M

87
hetzner_discovery.py
View File

@@ -10,17 +10,29 @@ load_dotenv()
# 🔑 Load Hetzner API Token & Prometheus URL # 🔑 Load Hetzner API Token & Prometheus URL
HETZNER_API_TOKEN = os.getenv("HETZNER_API_TOKEN") HETZNER_API_TOKEN = os.getenv("HETZNER_API_TOKEN")
# https://prometheus.io/docs/prometheus/latest/configuration/configuration/ # https://prometheus.io/docs/prometheus/latest/configuration/configuration/
PROMETHEUS_RELOAD_URL = os.getenv("PROMETHEUS_RELOAD_URL", "https://prometheus.phx-erp.de:9090/-/reload") PROMETHEUS_RELOAD_URL = os.getenv("PROMETHEUS_RELOAD_URL", "https://prometheus.phx-erp.de/-/reload")
# Check if token is loaded # Check if token is loaded
if not HETZNER_API_TOKEN: if not HETZNER_API_TOKEN:
raise ValueError("❌ HETZNER_API_TOKEN is missing! Make sure it's set in the .env file.") raise ValueError("❌ HETZNER_API_TOKEN is missing! Make sure it's set in the .env file.")
# 📂 Paths to output files # 📂 Paths to output files
PROMETHEUS_TARGETS_FILE = "/opt/phx/main/config/hetzner_targets.json" # PROMETHEUS_TARGETS_FILE = "/opt/phx/main/config/hetzner_targets.json"
ERROR_LOG_FILE = "/opt/phx/main/config/hetzner_error_servers.json" # ERROR_LOG_FILE = "/opt/phx/main/config/hetzner_error_servers.json"
EXCLUDED_SERVERS_FILE = "/opt/phx/main/config/hetzner_excluded_servers.json" # EXCLUDED_SERVERS_FILE = "/opt/phx/main/config/hetzner_excluded_servers.json"
DNS_MAPPING_FILE = "/opt/phx/main/config/hetzner_dns_mapping.json" # DNS_MAPPING_FILE = "/opt/phx/main/config/hetzner_dns_mapping.json"
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
HEARTBEAT_TARGETS_FILE = os.path.join(BASE_DIR, "config/hetzner_heartbeat_targets.json")
PROMETHEUS_TARGETS_FILE = os.path.join(BASE_DIR, "config/hetzner_targets.json")
PHX_SYSTEM_FILE = os.path.join(BASE_DIR, "config/phoenix_system_hetzner_targets.json")
PHX_WORKER_FILE = os.path.join(BASE_DIR, "config/phoenix_worker_hetzner_targets.json")
PHX_HEALTH_EXPORTER_FILE = os.path.join(BASE_DIR, "config/phoenix_health_exporter_hetzner_targets.json")
DNS_MAPPING_FILE = os.path.join(BASE_DIR, "config/hetzner_dns_mapping.json")
ERROR_LOG_FILE = os.path.join(BASE_DIR, "config/hetzner_error_servers.json")
EXCLUDED_SERVERS_FILE = os.path.join(BASE_DIR, "config/hetzner_excluded_servers.json")
# 📌 Hetzner API URL # 📌 Hetzner API URL
HETZNER_API_URL = os.getenv("HETZNER_API_URL") HETZNER_API_URL = os.getenv("HETZNER_API_URL")
@@ -30,7 +42,7 @@ if not HETZNER_API_URL:
raise ValueError("❌ HETZNER_API_URL is missing! Make sure it's set in the .env file.") raise ValueError("❌ HETZNER_API_URL is missing! Make sure it's set in the .env file.")
# 🛑 List of server names to exclude (DARKLIST) # 🛑 List of server names to exclude (DARKLIST)
DARKLISTED_SERVERS = ["docuvita"] DARKLISTED_SERVERS = ["docuvita", "teamcity", "gitea"]
# 📡 Fetch Hetzner server list with pagination support and Rate Limiting handling # 📡 Fetch Hetzner server list with pagination support and Rate Limiting handling
def get_hetzner_servers(): def get_hetzner_servers():
@@ -78,9 +90,13 @@ def get_hetzner_servers():
def generate_prometheus_sd_config(): def generate_prometheus_sd_config():
servers = get_hetzner_servers() servers = get_hetzner_servers()
targets = [] targets = []
phx_system_targets = []
phx_worker_targets = []
phx_health_exporter_targets = []
error_servers = [] error_servers = []
excluded_servers = [] excluded_servers = []
dns_mappings = [] # New list for storing DNS-IP mappings dns_mappings = [] # New list for storing DNS-IP mappings
heartbeat_targets = []
for server in servers: for server in servers:
ipv4 = server.get("public_net", {}).get("ipv4", {}).get("ip") ipv4 = server.get("public_net", {}).get("ipv4", {}).get("ip")
@@ -97,14 +113,53 @@ def generate_prometheus_sd_config():
continue # Skip adding to Prometheus targets continue # Skip adding to Prometheus targets
if ipv4: if ipv4:
# Add to DNS mapping file
targets.append({ targets.append({
"targets": [f"{ipv4}:9100"], "targets": [f"{ipv4}:9100", f"{ipv4}:9113"],
"labels": { "labels": {
"instance": server_name, "instance": server_name,
"datacenter": datacenter "datacenter": datacenter
} }
}) })
# Add to DNS mapping file
# This is with Python Flask server for health checks
phx_health_exporter_targets.append({
"targets": [f"{ipv4}:9800"],
"labels": {
"instance": f"{server_name}-health-exporter",
"datacenter": datacenter
}
})
# Add Pushgateway heartbeat endpoint (default Pushgateway port is 9091)
heartbeat_targets.append({
"targets": [f"{ipv4}:9091"], # assuming Pushgateway is running on each server
"labels": {
"instance": server_name,
"job": "heartbeat"
}
})
# Phoenix System metrics (port 3000)
# phx_system_targets.append({
# "targets": [f"{ipv4}:3000"],
# "labels": {
# "instance": f"{server_name}",
# "datacenter": datacenter,
# "__metrics_path__": "/health/metrics"
# }
# })
# # Phoenix Worker metrics (port 3001)
# phx_worker_targets.append({
# "targets": [f"{ipv4}:3001"],
# "labels": {
# "instance": f"{server_name}",
# "datacenter": datacenter,
# "__metrics_path__": "/health/metrics"
# }
# })
dns_mappings.append({ dns_mappings.append({
"dns_name": server_name, "dns_name": server_name,
"ip_address": ipv4 "ip_address": ipv4
@@ -125,6 +180,22 @@ def generate_prometheus_sd_config():
print(f"✅ Updated Prometheus targets in {PROMETHEUS_TARGETS_FILE}") print(f"✅ Updated Prometheus targets in {PROMETHEUS_TARGETS_FILE}")
with open(PHX_HEALTH_EXPORTER_FILE, "w") as f:
json.dump(phx_health_exporter_targets, f, indent=4)
print(f"✅ phoenix-health-exporter targets saved to {PHX_HEALTH_EXPORTER_FILE}")
with open(HEARTBEAT_TARGETS_FILE, "w") as f:
json.dump(heartbeat_targets, f, indent=4)
print(f"✅ heartbeat targets saved to {HEARTBEAT_TARGETS_FILE}")
# with open(PHX_SYSTEM_FILE, "w") as f:
# json.dump(phx_system_targets, f, indent=4)
# print(f"✅ phoenix-system targets saved to {PHX_SYSTEM_FILE}")
# with open(PHX_WORKER_FILE, "w") as f:
# json.dump(phx_worker_targets, f, indent=4)
# print(f"✅ phoenix-worker targets saved to {PHX_WORKER_FILE}")
# Save DNS Mappings file # Save DNS Mappings file
with open(DNS_MAPPING_FILE, "w") as f: with open(DNS_MAPPING_FILE, "w") as f:
json.dump(dns_mappings, f, indent=4) json.dump(dns_mappings, f, indent=4)