👌 IMPROVE: Update Prometheus and Grafana images, enhance Hetzner DNS mapping, and add Prometheus reload functionality

This commit is contained in:
2025-05-13 10:58:11 +00:00
parent ebac8ff7e5
commit 84db17fa0e
8 changed files with 250 additions and 102 deletions

View File

@@ -1,4 +1,4 @@
FROM docker.io/prom/prometheus:v2.51.2
FROM docker.io/prom/prometheus:v3.1.0
# Set the proper permissions for certificates
RUN if [ -f ./https_portal/data/prometheus.phx-erp.de/production/signed.crt ]; then chmod 644 ./https_portal/data/prometheus.phx-erp.de/production/signed.crt; fi && \

View File

@@ -55,10 +55,6 @@
"dns_name": "shipxpert",
"ip_address": "49.13.165.13"
},
{
"dns_name": "dss",
"ip_address": "49.13.197.152"
},
{
"dns_name": "sartissohn",
"ip_address": "188.245.44.219"
@@ -96,7 +92,63 @@
"ip_address": "167.235.130.242"
},
{
"dns_name": "bug",
"ip_address": "138.199.196.189"
"dns_name": "PHX-License-Server",
"ip_address": "188.245.32.214"
},
{
"dns_name": "PHX-SkyNodus",
"ip_address": "138.199.201.35"
},
{
"dns_name": "bode",
"ip_address": "188.245.108.133"
},
{
"dns_name": "skr04",
"ip_address": "168.119.174.181"
},
{
"dns_name": "artwin",
"ip_address": "91.99.24.53"
},
{
"dns_name": "amplid",
"ip_address": "168.119.231.96"
},
{
"dns_name": "schwarz",
"ip_address": "116.203.143.63"
},
{
"dns_name": "skr03",
"ip_address": "91.99.53.67"
},
{
"dns_name": "moonich",
"ip_address": "91.99.52.253"
},
{
"dns_name": "comp-sys",
"ip_address": "91.99.76.156"
},
{
"dns_name": "skr-ch",
"ip_address": "78.46.214.224"
},
{
"dns_name": "wsoft",
"ip_address": "116.203.18.156"
},
{
"dns_name": "bzp",
"ip_address": "91.99.71.224"
},
{
"dns_name": "dss",
"ip_address": "49.13.197.152"
},
{
"dns_name": "phx-healthcheck-phoenixApp-pgAdmin-Test-2",
"ip_address": "116.203.103.215"
}
]

View File

@@ -125,15 +125,6 @@
"datacenter": "nbg1"
}
},
{
"targets": [
"49.13.197.152:9100"
],
"labels": {
"instance": "dss",
"datacenter": "nbg1"
}
},
{
"targets": [
"188.245.44.219:9100"
@@ -217,10 +208,136 @@
},
{
"targets": [
"138.199.196.189:9100"
"188.245.32.214:9100"
],
"labels": {
"instance": "bug",
"instance": "PHX-License-Server",
"datacenter": "nbg1"
}
},
{
"targets": [
"138.199.201.35:9100"
],
"labels": {
"instance": "PHX-SkyNodus",
"datacenter": "nbg1"
}
},
{
"targets": [
"188.245.108.133:9100"
],
"labels": {
"instance": "bode",
"datacenter": "nbg1"
}
},
{
"targets": [
"168.119.174.181:9100"
],
"labels": {
"instance": "skr04",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.24.53:9100"
],
"labels": {
"instance": "artwin",
"datacenter": "nbg1"
}
},
{
"targets": [
"168.119.231.96:9100"
],
"labels": {
"instance": "amplid",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.143.63:9100"
],
"labels": {
"instance": "schwarz",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.53.67:9100"
],
"labels": {
"instance": "skr03",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.52.253:9100"
],
"labels": {
"instance": "moonich",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.76.156:9100"
],
"labels": {
"instance": "comp-sys",
"datacenter": "nbg1"
}
},
{
"targets": [
"78.46.214.224:9100"
],
"labels": {
"instance": "skr-ch",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.18.156:9100"
],
"labels": {
"instance": "wsoft",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.71.224:9100"
],
"labels": {
"instance": "bzp",
"datacenter": "nbg1"
}
},
{
"targets": [
"49.13.197.152:9100"
],
"labels": {
"instance": "dss",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.103.215:9100"
],
"labels": {
"instance": "phx-healthcheck-phoenixApp-pgAdmin-Test-2",
"datacenter": "nbg1"
}
}

View File

@@ -5,19 +5,18 @@ volumes:
prometheus-data:
driver: local
networks:
default:
name: grafana-prometheus
grafana-prometheus:
external: true
services:
grafana:
image: docker.io/grafana/grafana-oss:10.4.2
image: docker.io/grafana/grafana-oss:12.0.0
container_name: grafana
environment:
- GF_SECURITY_ADMIN_PASSWORD=admin
# - GF_SERVER_DOMAIN=%(protocol)s://%(domain)s:%(http_port)s/
# - GF_SERVER_ROOT_URL=%(protocol)s://%(domain)s/
- GF_SERVER_DOMAIN=grafana.phx-erp.de
# - GF_SERVER_PROTOCOL=https
- GF_SERVER_PROTOCOL=https
# - GF_SERVER_HTTP_PORT=3000
- GF_LOG_LEVEL=debug
- GF_SMTP_ENABLED=true
@@ -29,23 +28,24 @@ services:
- GF_SMTP_STARTTLS_POLICY=false
- GF_SMTP_SKIP_VERIFY=true
- GF_SMTP_ENABLE_TRACING=false
ports:
- "3000:3000"
volumes:
- grafana-data:/var/lib/grafana
restart: unless-stopped
networks:
- default
- grafana-prometheus
depends_on:
- prometheus
prometheus:
image: docker.io/prom/prometheus:v2.51.2
image: docker.io/prom/prometheus:v3.1.0
container_name: prometheus
user: "65534:65534"
ports:
- 9090:9090
- "9090:9090"
command:
- "--config.file=/etc/prometheus/prometheus.yaml"
- "--web.config.file=/etc/prometheus/web-config.yaml"
- "--web.external-url=https://prometheus.phx-erp.de"
- "--web.enable-lifecycle" # 🔹 Enable dynamic config reload
volumes:
# - '--web.enable-lifecycle' # Enable reload of configuration automatically without restart !!!
- ./config/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro # RO Read-only
@@ -57,37 +57,14 @@ services:
- ./https_portal/data/prometheus.phx-erp.de/production/domain.key:/etc/prometheus/certs/domain.key:ro
restart: unless-stopped
networks:
- default
node_exporter:
image: quay.io/prometheus/node-exporter:latest
container_name: node_exporter
network_mode: host
pid: host
restart: unless-stopped
command:
- "--path.procfs=/host/proc"
- "--path.sysfs=/host/sys"
- "--path.rootfs=/host"
- "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev)($$|/)"
volumes:
- "/proc:/host/proc:ro"
- "/sys:/host/sys:ro"
- "/:/host:ro,rslave"
# blackbox_exporter:
# image: prom/blackbox-exporter
# container_name: blackbox_exporter
# ports:
# - "9115:9115"
# restart: unless-stopped
# networks:
# - default
- grafana-prometheus
https_portal:
container_name: https_portal
image: "steveltn/https-portal:1.21"
restart: unless-stopped
user: "root"
networks:
- default
- grafana-prometheus
ports:
- "80:80"
- "443:443"
@@ -98,7 +75,10 @@ services:
RENEW_MARGIN_DAYS: 30
CLIENT_MAX_BODY_SIZE: 0
# FORCE_RENEW: 'true'
DOMAINS: 'grafana.phx-erp.de -> 5.75.153.161:3000, prometheus.phx-erp.de -> 5.75.153.161:9090'
DOMAINS: 'grafana.phx-erp.de -> https://grafana:3000, prometheus.phx-erp.de -> https://prometheus:9090'
volumes:
- ./https_portal/data:/var/lib/https-portal # ssl_certs, vhost.d, htdocs
- ./https_portal/log:/var/log/nginx # nginx logs
- ./https_portal/log:/var/log/nginx # nginx logs
depends_on:
- prometheus
- grafana

View File

@@ -7,8 +7,10 @@ from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
# 🔑 Load Hetzner API Token
# 🔑 Load Hetzner API Token & Prometheus URL
HETZNER_API_TOKEN = os.getenv("HETZNER_API_TOKEN")
# https://prometheus.io/docs/prometheus/latest/configuration/configuration/
PROMETHEUS_RELOAD_URL = os.getenv("PROMETHEUS_RELOAD_URL", "https://prometheus.phx-erp.de:9090/-/reload")
# Check if token is loaded
if not HETZNER_API_TOKEN:
@@ -143,6 +145,16 @@ def generate_prometheus_sd_config():
else:
print("✅ No servers were excluded due to the darklist.")
# 🔄 **Trigger Prometheus Reload**
try:
response = requests.post(PROMETHEUS_RELOAD_URL)
if response.status_code == 200:
print("🔄 ✅ Prometheus configuration reloaded successfully!")
else:
print(f"⚠️ Warning: Prometheus reload failed with status {response.status_code}: {response.text}")
except requests.exceptions.RequestException as e:
print(f"❌ Error reloading Prometheus: {e}")
# 🔄 Run the script
if __name__ == "__main__":
generate_prometheus_sd_config()

View File

@@ -1,31 +1,32 @@
-----BEGIN CERTIFICATE-----
MIIE9TCCA92gAwIBAgISAwzhqoLZ0fS63QkInFmDAX4pMA0GCSqGSIb3DQEBCwUA
MIIFIzCCBAugAwIBAgISBVolsu7Gt+CzF7nMdCREzqXYMA0GCSqGSIb3DQEBCwUA
MDMxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MQwwCgYDVQQD
EwNSMTEwHhcNMjQxMjI4MjMyNDAxWhcNMjUwMzI4MjMyNDAwWjAdMRswGQYDVQQD
EwNSMTEwHhcNMjUwNDI3MTAyMjI3WhcNMjUwNzI2MTAyMjI2WjAdMRswGQYDVQQD
ExJncmFmYW5hLnBoeC1lcnAuZGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQDBdjepETkdm9xE+6NG4FvXAhEE16t0aBGkIBgs/DUUyMcx8f8b4TMDqEtl
AMXtkRAC4pM/eeAMNq3yfeuLX9vG+E6Jra9vrcF6ThikARlnrEmNHD5qq7FfGiWM
vAsRrxeOW22TFQu7AlUK3mxQodDQkvKRhwHflo/f6dL6Z7jPTgVteZb5U0cEWxba
4kW/voO5aI/vm4kABoUdyzSy1cwza5fO0OQRLnAnV9wdtbx/47BLuacHSK4L8n30
KLhqrnnftT9XQiT18L9L4N8laBPuIGD+9Xix74izRs6kJCc/fm21hFTA+NuT5URM
UizyBp17yeX/vXH7d1v9TYL59UjdAgMBAAGjggIXMIICEzAOBgNVHQ8BAf8EBAMC
UizyBp17yeX/vXH7d1v9TYL59UjdAgMBAAGjggJFMIICQTAOBgNVHQ8BAf8EBAMC
BaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAw
HQYDVR0OBBYEFLuJi7SLpiYjNoDoBnLeTIb1Hh50MB8GA1UdIwQYMBaAFMXPRqTq
9MPAemyVxC2wXpIvJuO5MFcGCCsGAQUFBwEBBEswSTAiBggrBgEFBQcwAYYWaHR0
cDovL3IxMS5vLmxlbmNyLm9yZzAjBggrBgEFBQcwAoYXaHR0cDovL3IxMS5pLmxl
bmNyLm9yZy8wHQYDVR0RBBYwFIISZ3JhZmFuYS5waHgtZXJwLmRlMBMGA1UdIAQM
MAowCAYGZ4EMAQIBMIIBBQYKKwYBBAHWeQIEAgSB9gSB8wDxAHcAcyAiDwgWivnz
xKaLCrJqmkoA7vV3hYoITQUA1KVCRFkAAAGUD8sF2gAABAMASDBGAiEAwvxEK9nh
4wnf0vaUn4rWl35EznLoiOD+oWIaxyzsUv0CIQCFT1MF/AWr1/zJiBWZoV6Pce57
T4d2MEGaGy35QU1FiwB2AKLjCuRF772tm3447Udnd1PXgluElNcrXhssxLlQpEfn
AAABlA/LBdoAAAQDAEcwRQIgAy0GLb9048EDmhT4EzCEIWd+f5ze1SBY0IrUi/Nq
ipkCIQDxX1DEkbBnxnO6U2YgSeKRU/BiUA24fWkdN5brf5KnKzANBgkqhkiG9w0B
AQsFAAOCAQEAnH1I6R966b5Bl/Sbm1uQy2ayflXFdq+p9th9+WIy5Z1PMmW/oeVa
TNpjO+M8UjGznf6tzd2Ur9YgAWYR1EeawmkDJqp5lQfNHWhEVLU4CTgWVfBrIagM
mVs54WKmUAxkPYftEzBXV3rAvz8WLy4dDnEUGrr7ZwJNB8JTWOzB+akPKUkiOrPx
p9vmvYc6Yxv9YtaMjcVvU61RBxQo6XobT1Zg6O1gPU9JmNBw9vgfVITCBv01oGCa
uFBqjdkn2DZdOpZk2orrKKll6CFSVCrLcI55Ja1kRJLoMtbMfOmAEewWfGbcxnZv
hI9isR7X8MJ9rsJhaqOQOQ84mnNteR4+ww==
MAowCAYGZ4EMAQIBMC4GA1UdHwQnMCUwI6AhoB+GHWh0dHA6Ly9yMTEuYy5sZW5j
ci5vcmcvODkuY3JsMIIBAwYKKwYBBAHWeQIEAgSB9ASB8QDvAHYAzPsPaoVxCWX+
lZtTzumyfCLphVwNl422qX5UwP5MDbAAAAGWdvqaYwAABAMARzBFAiEA0e6Cw5IB
RmotsjC9qcc1X9Ab0NODHRs5JOLEp7CB/J8CIG4b6L+EH2qhYed9TSjln8Nt1Tym
Yq4PI29PmNnbaCnzAHUAfVkeEuF4KnscYWd8Xv340IdcFKBOlZ65Ay/ZDowuebgA
AAGWdvqaUAAABAMARjBEAiBcksSA/FONdr4vvl6Y6b2AaAh+wHzByinHdn6Jc09q
AgIgJfM1XdTgmY/QJb29OGpH8kgLkTZn+MnwHEoHYI2nobMwDQYJKoZIhvcNAQEL
BQADggEBALfI55bDe7jZLZiD+VdpPDhGyx6Rf7rAkHZ4xtZL4v6lhd2pWZeyV3bJ
DNj7IQ/iVIxe388G9MhfmyYjn4fPmDvczGrJzoA2z45bCAS7x/chYjeej4MPq/Ud
VwCrU2OUFH9dMB0q+3qHr6aGnKRuaG4DS0qIkTEa4ou2KfSbaDOR9uqgFj3V3wVp
0LHLiCPLe+KjUdT7WUmwfEAF/5V20SUjzTEExkfI0Yhmtt6EYxIUhZ8zeH1+/qox
An7U48oDvSi/QsdGaeuKpTR09tjol/txASJhIhSJqfL+Q9jKG6s+Yc7mTeQe+PZl
JmDDdtiKADcJ9EtIhG6rqt6NBL3NOSA=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----

View File

@@ -1,31 +1,32 @@
-----BEGIN CERTIFICATE-----
MIIE+TCCA+GgAwIBAgISBOmwA+dtHvdNzMHA9pDKORoqMA0GCSqGSIb3DQEBCwUA
MIIFKzCCBBOgAwIBAgISBkq6MpaUzRQ75D6kFDRfobJtMA0GCSqGSIb3DQEBCwUA
MDMxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MQwwCgYDVQQD
EwNSMTAwHhcNMjQxMjI4MjMyNDEyWhcNMjUwMzI4MjMyNDExWjAgMR4wHAYDVQQD
EwNSMTAwHhcNMjUwNDI3MTAyMjM4WhcNMjUwNzI2MTAyMjM3WjAgMR4wHAYDVQQD
ExVwcm9tZXRoZXVzLnBoeC1lcnAuZGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
ggEKAoIBAQCx+/3s+NJEchgLo3GfBIzSBN/q52J46liUx6qISjbswhr7Fxg166rC
K7ybmqs6y+1bsHzYjCQ0bBCp4Gjhrs/kfVSZ/CpjpfNV3XpanJt9AD00qFCbLX9u
X69PRre/URdOfwtxW0ZZxev4/H9us1epSk54vq25ntR9Nl1IFMZ+b7Bk6CRa7Q7i
DkpM/6Cv2A+QhbUozOcQjv1vnpnSxST/QbW03dvvy6OR9OlG5bzrAADW0Mo2BREO
8GfOY8YIX2EZvEYHDda4KztLvMhAN79iKmlzlP+rsKEddvN64ivvKhW9MUgf7NIF
w9dHIJs16lomoxp87kQ9Eav+K8Gw0PNHAgMBAAGjggIYMIICFDAOBgNVHQ8BAf8E
w9dHIJs16lomoxp87kQ9Eav+K8Gw0PNHAgMBAAGjggJKMIICRjAOBgNVHQ8BAf8E
BAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQC
MAAwHQYDVR0OBBYEFCDXhBghaDDRLeSvoxXD7qp/++hrMB8GA1UdIwQYMBaAFLu8
w0el5LypxsOkcgwQjaI14cjoMFcGCCsGAQUFBwEBBEswSTAiBggrBgEFBQcwAYYW
aHR0cDovL3IxMC5vLmxlbmNyLm9yZzAjBggrBgEFBQcwAoYXaHR0cDovL3IxMC5p
LmxlbmNyLm9yZy8wIAYDVR0RBBkwF4IVcHJvbWV0aGV1cy5waHgtZXJwLmRlMBMG
A1UdIAQMMAowCAYGZ4EMAQIBMIIBAwYKKwYBBAHWeQIEAgSB9ASB8QDvAHUAouMK
5EXvva2bfjjtR2d3U9eCW4SU1yteGyzEuVCkR+cAAAGUD8syPAAABAMARjBEAiAA
9m1jW0X1axy3svqRR14c/f+rG3Aqs0hxtFICa0B1JwIgb9EXAXMbLdU8asLkiMUO
FWCPNhsz4BSMmG4hj8a2hmgAdgDM+w9qhXEJZf6Vm1PO6bJ8IumFXA2XjbapflTA
/kwNsAAAAZQPyzJLAAAEAwBHMEUCIQDnaNyvW38vlep1BaPrPp7pB3JHqZFSQvgN
5PpA3F9ZiQIgS06COcu12H6HbgA79457y+3qdyxlhxXdDYLISYwL+swwDQYJKoZI
hvcNAQELBQADggEBAHcxvLgLS8/wTj77T8jC1FLCd/7bUpuBkm7N/XJgi/er7wXv
Y1IE4A1V2hbkcrJT9lH0YbbUYf/UqpNAy1WLD7xAFvuie4jVGtpAgibnaPqNu8x0
V3jPiaUf75Ejb7+5bM39GOaIGheFpciHQ9bLVEG0HpUwV/8W+KYKo37Az8ABuwnJ
Bodmbs73Xcpz2D3p+mSoMu35L3T/ifSsXz/ebzaRiXQRCWLJtOxfBpJbFJFMfyug
mVnSdGsEsLN2WXmSFNxK2o1H61bAgQ5WtadpbGvXOSVrOCOObl4E5QaPQsckqsTj
d/htP0HtBQoWc1kOtZ5Ng+wig5hoojVxTkGH7IA=
A1UdIAQMMAowCAYGZ4EMAQIBMC8GA1UdHwQoMCYwJKAioCCGHmh0dHA6Ly9yMTAu
Yy5sZW5jci5vcmcvMTEwLmNybDCCAQQGCisGAQQB1nkCBAIEgfUEgfIA8AB2AA3h
8jAr0w3BQGISCepVLvxHdHyx1+kw7w5CHrR+Tqo0AAABlnb6yEEAAAQDAEcwRQIh
ALIQgHy9Dm8jEK8pmDdXnTXaW3Hgknamxu4HPvqUZJfFAiBHcNJcEWs3vFmC5RTp
9JEltl7ma7P32N84zgZgWXbGxAB2AMz7D2qFcQll/pWbU87psnwi6YVcDZeNtql+
VMD+TA2wAAABlnb6yFEAAAQDAEcwRQIgLyj1f+tAq4iywH+i/kZV0LjdKXe+S+CU
h/xnuJjdzdYCIQC7FxwVOxzatbYDlHmeRkX4fV0e2HenYUiDgG+/huR1SDANBgkq
hkiG9w0BAQsFAAOCAQEAOqq1KFmNo/lF0bAtFx1YBlGZk8cGQHumUhMlVRR9VKZ+
S43GoJQOg+AD3yCjXA4YNUMpy0n+E8ggWlA2cPwC378crk8Zu/wh2ILcyPDOaik/
vAJ50m+dyZ67Ons9uaBuy7tgiEHmvscRnQI+PjybXrcmdDEMoYu6JW4TEqsVYqi1
Arfr1XbSdC8HDmjNNjJGAxrqsKH25rH1BEXeqwdeDXwXjhL3V66k78MLzRhxH6bf
ngwsfG8DotosCLn4bNICg1x+xrzWMKJUZ7QUe3sf5PMZILPYUYaIdus04hVBN8Hq
CwRvCi7vIJ8PSGvZ7qaYlhr/k/LmPVlz5ik/LF+p2Q==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----

View File

@@ -1,17 +1,2 @@
2025/01/30 03:45:01 [notice] 1650#1650: signal process started
2025/01/30 11:01:20 [notice] 265#265: signal process started
2025/01/30 11:35:11 [notice] 264#264: signal process started
2025/01/30 12:33:22 [notice] 265#265: signal process started
2025/01/30 12:34:04 [notice] 265#265: signal process started
2025/01/30 12:37:19 [notice] 264#264: signal process started
2025/01/30 12:37:41 [notice] 265#265: signal process started
2025/01/30 12:49:57 [notice] 265#265: signal process started
2025/01/30 12:53:15 [notice] 266#266: signal process started
2025/01/30 12:56:58 [notice] 265#265: signal process started
2025/01/30 13:20:47 [notice] 265#265: signal process started
2025/01/30 13:21:52 [notice] 265#265: signal process started
2025/01/30 13:25:46 [notice] 264#264: signal process started
2025/01/30 13:35:24 [notice] 265#265: signal process started
2025/01/30 13:41:55 [notice] 265#265: signal process started
2025/01/30 14:35:34 [notice] 265#265: signal process started
2025/01/30 14:45:52 [notice] 265#265: signal process started
2025/05/13 03:45:01 [notice] 902#902: signal process started
2025/05/13 08:24:10 [notice] 268#268: signal process started