👌 IMPROVE: Update Hetzner DNS mapping and targets configuration by renaming instances, removing obsolete entries, and adding new targets for enhanced monitoring

config/hetzner_dns_mapping.json

@@ -28,15 +28,7 @@
         "ip_address": "142.132.165.231"
     },
     {
-        "dns_name": "teamcity",
-        "ip_address": "23.88.107.109"
-    },
-    {
-        "dns_name": "gitea",
-        "ip_address": "49.13.146.138"
-    },
-    {
-        "dns_name": "phx-beta-rc",
+        "dns_name": "phx-beta",
         "ip_address": "159.69.200.205"
     },
     {
@@ -148,7 +140,47 @@
         "ip_address": "49.13.197.152"
     },
     {
-        "dns_name": "phx-healthcheck-phoenixApp-pgAdmin-Test-2",
-        "ip_address": "116.203.103.215"
+        "dns_name": "modern",
+        "ip_address": "157.90.228.52"
+    },
+    {
+        "dns_name": "burg-itc",
+        "ip_address": "23.88.56.89"
+    },
+    {
+        "dns_name": "capricorn-it",
+        "ip_address": "91.99.109.3"
+    },
+    {
+        "dns_name": "kundw",
+        "ip_address": "91.99.140.34"
+    },
+    {
+        "dns_name": "dieweingoetter",
+        "ip_address": "128.140.33.105"
+    },
+    {
+        "dns_name": "Rustdesk-Relay-Server",
+        "ip_address": "94.130.108.153"
+    },
+    {
+        "dns_name": "rihnet",
+        "ip_address": "91.99.226.250"
+    },
+    {
+        "dns_name": "infoplus",
+        "ip_address": "49.13.205.27"
+    },
+    {
+        "dns_name": "ai-it",
+        "ip_address": "195.201.31.160"
+    },
+    {
+        "dns_name": "guder",
+        "ip_address": "91.99.143.88"
+    },
+    {
+        "dns_name": "testRedisHeathCheck",
+        "ip_address": "91.107.235.238"
     }
 ]

config/hetzner_excluded_servers.json

@@ -1,7 +1,13 @@
 [
     {
-        "server_id": 52530041,
-        "name": "docuvita",
+        "server_id": 41663080,
+        "name": "teamcity",
+        "datacenter": "nbg1",
+        "reason": "Darklisted server"
+    },
+    {
+        "server_id": 41739626,
+        "name": "gitea",
         "datacenter": "nbg1",
         "reason": "Darklisted server"
     }

config/hetzner_heartbeat_targets.json

@@ -0,0 +1,416 @@
+[
+    {
+        "targets": [
+            "157.90.161.42:9091"
+        ],
+        "labels": {
+            "instance": "PHX-DEV-001.Alpha",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "167.235.254.4:9091"
+        ],
+        "labels": {
+            "instance": "ANSIBLE-MASTER",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "116.203.75.215:9091"
+        ],
+        "labels": {
+            "instance": "cts",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "162.55.52.253:9091"
+        ],
+        "labels": {
+            "instance": "Phx-Yuri",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "162.55.54.75:9091"
+        ],
+        "labels": {
+            "instance": "benzinger-demo",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "5.75.183.139:9091"
+        ],
+        "labels": {
+            "instance": "phx-internal",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "142.132.165.231:9091"
+        ],
+        "labels": {
+            "instance": "trachtenmode-schmid",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "159.69.200.205:9091"
+        ],
+        "labels": {
+            "instance": "phx-beta",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "116.203.53.137:9091"
+        ],
+        "labels": {
+            "instance": "lhl",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "5.75.153.161:9091"
+        ],
+        "labels": {
+            "instance": "Grafana-Prometheus",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "159.69.44.39:9091"
+        ],
+        "labels": {
+            "instance": "cooper",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "49.13.165.13:9091"
+        ],
+        "labels": {
+            "instance": "shipxpert",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "188.245.44.219:9091"
+        ],
+        "labels": {
+            "instance": "sartissohn",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "116.203.92.218:9091"
+        ],
+        "labels": {
+            "instance": "guntli",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "94.130.77.57:9091"
+        ],
+        "labels": {
+            "instance": "kolb",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "116.203.151.20:9091"
+        ],
+        "labels": {
+            "instance": "ried",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "116.203.68.120:9091"
+        ],
+        "labels": {
+            "instance": "heba",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "116.203.128.69:9091"
+        ],
+        "labels": {
+            "instance": "eicsoft",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "159.69.93.252:9091"
+        ],
+        "labels": {
+            "instance": "ck-vechta",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "116.203.46.171:9091"
+        ],
+        "labels": {
+            "instance": "eeparts",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "167.235.130.242:9091"
+        ],
+        "labels": {
+            "instance": "big-break-changes",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "188.245.32.214:9091"
+        ],
+        "labels": {
+            "instance": "PHX-License-Server",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "138.199.201.35:9091"
+        ],
+        "labels": {
+            "instance": "PHX-SkyNodus",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "188.245.108.133:9091"
+        ],
+        "labels": {
+            "instance": "bode",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "168.119.174.181:9091"
+        ],
+        "labels": {
+            "instance": "skr04",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "91.99.24.53:9091"
+        ],
+        "labels": {
+            "instance": "artwin",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "168.119.231.96:9091"
+        ],
+        "labels": {
+            "instance": "amplid",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "116.203.143.63:9091"
+        ],
+        "labels": {
+            "instance": "schwarz",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "91.99.53.67:9091"
+        ],
+        "labels": {
+            "instance": "skr03",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "91.99.52.253:9091"
+        ],
+        "labels": {
+            "instance": "moonich",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "91.99.76.156:9091"
+        ],
+        "labels": {
+            "instance": "comp-sys",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "78.46.214.224:9091"
+        ],
+        "labels": {
+            "instance": "skr-ch",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "116.203.18.156:9091"
+        ],
+        "labels": {
+            "instance": "wsoft",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "91.99.71.224:9091"
+        ],
+        "labels": {
+            "instance": "bzp",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "49.13.197.152:9091"
+        ],
+        "labels": {
+            "instance": "dss",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "157.90.228.52:9091"
+        ],
+        "labels": {
+            "instance": "modern",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "23.88.56.89:9091"
+        ],
+        "labels": {
+            "instance": "burg-itc",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "91.99.109.3:9091"
+        ],
+        "labels": {
+            "instance": "capricorn-it",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "91.99.140.34:9091"
+        ],
+        "labels": {
+            "instance": "kundw",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "128.140.33.105:9091"
+        ],
+        "labels": {
+            "instance": "dieweingoetter",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "94.130.108.153:9091"
+        ],
+        "labels": {
+            "instance": "Rustdesk-Relay-Server",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "91.99.226.250:9091"
+        ],
+        "labels": {
+            "instance": "rihnet",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "49.13.205.27:9091"
+        ],
+        "labels": {
+            "instance": "infoplus",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "195.201.31.160:9091"
+        ],
+        "labels": {
+            "instance": "ai-it",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "91.99.143.88:9091"
+        ],
+        "labels": {
+            "instance": "guder",
+            "job": "heartbeat"
+        }
+    },
+    {
+        "targets": [
+            "91.107.235.238:9091"
+        ],
+        "labels": {
+            "instance": "testRedisHeathCheck",
+            "job": "heartbeat"
+        }
+    }
+]

config/hetzner_targets.json

@@ -69,33 +69,13 @@
             "datacenter": "nbg1"
         }
     },
-    {
-        "targets": [
-            "23.88.107.109:9100",
-            "23.88.107.109:9113"
-        ],
-        "labels": {
-            "instance": "teamcity",
-            "datacenter": "nbg1"
-        }
-    },
-    {
-        "targets": [
-            "49.13.146.138:9100",
-            "49.13.146.138:9113"
-        ],
-        "labels": {
-            "instance": "gitea",
-            "datacenter": "nbg1"
-        }
-    },
     {
         "targets": [
             "159.69.200.205:9100",
             "159.69.200.205:9113"
         ],
         "labels": {
-            "instance": "phx-beta-rc",
+            "instance": "phx-beta",
             "datacenter": "nbg1"
         }
     },
@@ -371,11 +351,111 @@
     },
     {
         "targets": [
-            "116.203.103.215:9100",
-            "116.203.103.215:9113"
+            "157.90.228.52:9100",
+            "157.90.228.52:9113"
         ],
         "labels": {
-            "instance": "phx-healthcheck-phoenixApp-pgAdmin-Test-2",
+            "instance": "modern",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "23.88.56.89:9100",
+            "23.88.56.89:9113"
+        ],
+        "labels": {
+            "instance": "burg-itc",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.99.109.3:9100",
+            "91.99.109.3:9113"
+        ],
+        "labels": {
+            "instance": "capricorn-it",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.99.140.34:9100",
+            "91.99.140.34:9113"
+        ],
+        "labels": {
+            "instance": "kundw",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "128.140.33.105:9100",
+            "128.140.33.105:9113"
+        ],
+        "labels": {
+            "instance": "dieweingoetter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "94.130.108.153:9100",
+            "94.130.108.153:9113"
+        ],
+        "labels": {
+            "instance": "Rustdesk-Relay-Server",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.99.226.250:9100",
+            "91.99.226.250:9113"
+        ],
+        "labels": {
+            "instance": "rihnet",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "49.13.205.27:9100",
+            "49.13.205.27:9113"
+        ],
+        "labels": {
+            "instance": "infoplus",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "195.201.31.160:9100",
+            "195.201.31.160:9113"
+        ],
+        "labels": {
+            "instance": "ai-it",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.99.143.88:9100",
+            "91.99.143.88:9113"
+        ],
+        "labels": {
+            "instance": "guder",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.107.235.238:9100",
+            "91.107.235.238:9113"
+        ],
+        "labels": {
+            "instance": "testRedisHeathCheck",
             "datacenter": "nbg1"
         }
     }

config/loki-config.yaml

@@ -1,8 +1,12 @@
----
 auth_enabled: false
 
 server:
   http_listen_port: 3100
+  http_server_read_timeout: 1m
+  http_server_write_timeout: 1m
+  grpc_listen_port: 9095
+  log_level: info
+  log_format: logfmt
 
 common:
   instance_addr: 127.0.0.1
@@ -14,7 +18,15 @@ common:
   replication_factor: 1
   ring:
     kvstore:
-      store: inmemory
+      store: inmemory  # ✅ Use inmemory for single-node setups
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  ingestion_rate_mb: 10
+  ingestion_burst_size_mb: 20
+  max_query_parallelism: 8
+  max_query_lookback: 168h
 
 schema_config:
   configs:
@@ -26,5 +38,12 @@ schema_config:
         prefix: index_
         period: 24h
 
+table_manager:
+  retention_deletes_enabled: true
+  retention_period: 168h
+
 ruler:
   alertmanager_url: http://localhost:9093
+  rule_path: /loki/rules
+  enable_alertmanager_v2: true
+  enable_api: true

config/phoenix_health_exporter_hetzner_targets.json

@@ -0,0 +1,416 @@
+[
+    {
+        "targets": [
+            "157.90.161.42:9800"
+        ],
+        "labels": {
+            "instance": "PHX-DEV-001.Alpha-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "167.235.254.4:9800"
+        ],
+        "labels": {
+            "instance": "ANSIBLE-MASTER-health-exporter",
+            "datacenter": "fsn1"
+        }
+    },
+    {
+        "targets": [
+            "116.203.75.215:9800"
+        ],
+        "labels": {
+            "instance": "cts-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "162.55.52.253:9800"
+        ],
+        "labels": {
+            "instance": "Phx-Yuri-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "162.55.54.75:9800"
+        ],
+        "labels": {
+            "instance": "benzinger-demo-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "5.75.183.139:9800"
+        ],
+        "labels": {
+            "instance": "phx-internal-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "142.132.165.231:9800"
+        ],
+        "labels": {
+            "instance": "trachtenmode-schmid-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "159.69.200.205:9800"
+        ],
+        "labels": {
+            "instance": "phx-beta-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "116.203.53.137:9800"
+        ],
+        "labels": {
+            "instance": "lhl-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "5.75.153.161:9800"
+        ],
+        "labels": {
+            "instance": "Grafana-Prometheus-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "159.69.44.39:9800"
+        ],
+        "labels": {
+            "instance": "cooper-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "49.13.165.13:9800"
+        ],
+        "labels": {
+            "instance": "shipxpert-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "188.245.44.219:9800"
+        ],
+        "labels": {
+            "instance": "sartissohn-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "116.203.92.218:9800"
+        ],
+        "labels": {
+            "instance": "guntli-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "94.130.77.57:9800"
+        ],
+        "labels": {
+            "instance": "kolb-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "116.203.151.20:9800"
+        ],
+        "labels": {
+            "instance": "ried-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "116.203.68.120:9800"
+        ],
+        "labels": {
+            "instance": "heba-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "116.203.128.69:9800"
+        ],
+        "labels": {
+            "instance": "eicsoft-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "159.69.93.252:9800"
+        ],
+        "labels": {
+            "instance": "ck-vechta-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "116.203.46.171:9800"
+        ],
+        "labels": {
+            "instance": "eeparts-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "167.235.130.242:9800"
+        ],
+        "labels": {
+            "instance": "big-break-changes-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "188.245.32.214:9800"
+        ],
+        "labels": {
+            "instance": "PHX-License-Server-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "138.199.201.35:9800"
+        ],
+        "labels": {
+            "instance": "PHX-SkyNodus-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "188.245.108.133:9800"
+        ],
+        "labels": {
+            "instance": "bode-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "168.119.174.181:9800"
+        ],
+        "labels": {
+            "instance": "skr04-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.99.24.53:9800"
+        ],
+        "labels": {
+            "instance": "artwin-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "168.119.231.96:9800"
+        ],
+        "labels": {
+            "instance": "amplid-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "116.203.143.63:9800"
+        ],
+        "labels": {
+            "instance": "schwarz-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.99.53.67:9800"
+        ],
+        "labels": {
+            "instance": "skr03-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.99.52.253:9800"
+        ],
+        "labels": {
+            "instance": "moonich-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.99.76.156:9800"
+        ],
+        "labels": {
+            "instance": "comp-sys-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "78.46.214.224:9800"
+        ],
+        "labels": {
+            "instance": "skr-ch-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "116.203.18.156:9800"
+        ],
+        "labels": {
+            "instance": "wsoft-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.99.71.224:9800"
+        ],
+        "labels": {
+            "instance": "bzp-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "49.13.197.152:9800"
+        ],
+        "labels": {
+            "instance": "dss-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "157.90.228.52:9800"
+        ],
+        "labels": {
+            "instance": "modern-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "23.88.56.89:9800"
+        ],
+        "labels": {
+            "instance": "burg-itc-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.99.109.3:9800"
+        ],
+        "labels": {
+            "instance": "capricorn-it-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.99.140.34:9800"
+        ],
+        "labels": {
+            "instance": "kundw-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "128.140.33.105:9800"
+        ],
+        "labels": {
+            "instance": "dieweingoetter-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "94.130.108.153:9800"
+        ],
+        "labels": {
+            "instance": "Rustdesk-Relay-Server-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.99.226.250:9800"
+        ],
+        "labels": {
+            "instance": "rihnet-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "49.13.205.27:9800"
+        ],
+        "labels": {
+            "instance": "infoplus-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "195.201.31.160:9800"
+        ],
+        "labels": {
+            "instance": "ai-it-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.99.143.88:9800"
+        ],
+        "labels": {
+            "instance": "guder-health-exporter",
+            "datacenter": "nbg1"
+        }
+    },
+    {
+        "targets": [
+            "91.107.235.238:9800"
+        ],
+        "labels": {
+            "instance": "testRedisHeathCheck-health-exporter",
+            "datacenter": "nbg1"
+        }
+    }
+]

config/phoenix_system_hetzner_targets.json

@@ -0,0 +1 @@
+[]

config/phoenix_worker_hetzner_targets.json

@@ -0,0 +1 @@
+[]

config/prometheus.yaml

@@ -1,12 +1,43 @@
 global:
-  scrape_interval: 15s  # Default for all jobs unless overridden
+  scrape_interval: 15s  # ⏱️ Default interval Prometheus uses to scrape metrics from all targets, unless overridden in specific jobs.
 
 scrape_configs:
   # Existing Hetzner Dynamic Configuration
-  - job_name: "hetzner-dynamic"
-    scheme: http
+  - job_name: "hetzner-dynamic"  # 🏷️ Logical name for the scrape job (used in metrics as job="hetzner-dynamic").
+    file_sd_configs:  # 📂 Enable file-based service discovery. Prometheus will look at files to dynamically load targets.
+      - files:
+          - "/opt/phx/main/config/hetzner_targets.json"  # 📁 JSON file containing the list of targets (e.g. IPs, ports, labels).
+        refresh_interval: 30s  # 🔁 How often Prometheus checks the file for changes and updates the target list.
+    scrape_interval: 5s  # ⏱️ Override global default: scrape metrics from each target in this job every x seconds.
+    scheme: http  # 🌐 Use plain HTTP when scraping the targets (not HTTPS).
+  
+  - job_name: "phoenix-health-exporter"
     file_sd_configs:
       - files:
-          - "/opt/phx/main/config/hetzner_targets.json"
+          - "/opt/phx/main/config/phoenix_health_exporter_hetzner_targets.json"
         refresh_interval: 30s
-    scrape_interval: 5s  # Custom interval for Hetzner servers
+    scrape_interval: 2m # Change it carefully, it is a heavy job to keep open and close to check the health of DB connection, there annotation to refactor it and use a new pgDebounce image.
+    scheme: http
+
+  - job_name: "heartbeat"
+    file_sd_configs:
+      - files:
+          - "/opt/phx/main/config/hetzner_heartbeat_targets.json"
+        refresh_interval: 30s
+    honor_labels: true
+
+  # - job_name: "phoenix-system"
+  #   file_sd_configs:
+  #     - files:
+  #         - "/opt/phx/main/config/phoenix_system_hetzner_targets.json"
+  #       refresh_interval: 30s
+  #   scrape_interval: 5s
+  #   scheme: http
+
+  # - job_name: "phoenix-worker"
+  #   file_sd_configs:
+  #     - files:
+  #         - "/opt/phx/main/config/phoenix_worker_hetzner_targets.json"
+  #       refresh_interval: 30s
+  #   scrape_interval: 5s
+  #   scheme: http

docker-compose.yml

@@ -14,24 +14,26 @@ services:
     image: docker.io/grafana/grafana-oss:12.0.0
     container_name: grafana
     environment:
-      - GF_SECURITY_ADMIN_PASSWORD=admin
+      GF_PATHS_PROVISIONING: /var/lib/grafana/provisioning/
+      GF_SECURITY_ADMIN_PASSWORD: admin
       # - GF_SERVER_DOMAIN=%(protocol)s://%(domain)s:%(http_port)s/
       # - GF_SERVER_ROOT_URL=%(protocol)s://%(domain)s/
-      - GF_SERVER_DOMAIN=grafana.phx-erp.de
-      - GF_SERVER_PROTOCOL=https
+      GF_SERVER_DOMAIN: grafana.phx-erp.de
+      GF_SERVER_PROTOCOL: https
       # - GF_SERVER_HTTP_PORT=3000
-      - GF_LOG_LEVEL=debug
-      - GF_SMTP_ENABLED=true
-      - GF_SMTP_FROM_ADDRESS=admin@phx-erp.de
-      - GF_SMTP_FROM_NAME=Grafana
-      - GF_SMTP_HOST=mail.phx-erp.de:465
-      - GF_SMTP_PASSWORD=0rB0@et68
-      - GF_SMTP_USER=yuri.lima@phx-erp.de
-      - GF_SMTP_STARTTLS_POLICY=false
-      - GF_SMTP_SKIP_VERIFY=true
-      - GF_SMTP_ENABLE_TRACING=false
+      GF_LOG_LEVEL: debug
+      GF_SMTP_ENABLED: true
+      GF_SMTP_FROM_ADDRESS: admin@phx-erp.de
+      GF_SMTP_FROM_NAME: Grafana
+      GF_SMTP_HOST: mail.phx-erp.de:465
+      GF_SMTP_PASSWORD: 0rB0@et68
+      GF_SMTP_USER: yuri.lima@phx-erp.de
+      GF_SMTP_STARTTLS_POLICY: false
+      GF_SMTP_SKIP_VERIFY: true
+      GF_SMTP_ENABLE_TRACING: false
     volumes:
       - grafana-data:/var/lib/grafana
+      - ./config/provisioning:/var/lib/grafana/provisioning/
     restart: unless-stopped
     networks:
       - grafana-prometheus
@@ -41,8 +43,8 @@ services:
     image: docker.io/prom/prometheus:v3.1.0
     container_name: prometheus
     user: "65534:65534"
-    ports:
-      - "9090:9090"
+    # ports:
+    #   - "9090:9090"
     command:
       - "--config.file=/etc/prometheus/prometheus.yaml"
       - "--web.config.file=/etc/prometheus/web-config.yaml"
@@ -63,11 +65,11 @@ services:
   loki:
     container_name: loki
     image: docker.io/grafana/loki:3.5.0
-    command: "-config.file=/etc/loki/config.yaml" # Tells Grafana Loki to start with the config file.
+    command: 
+      - "-config.file=/etc/loki/config.yaml" # Tells Grafana Loki to start with the config file.
+      - "-config.expand-env=true"
     ports:
-      # --> (Optional) Remove when using traefik...
       - "3100:3100"
-      # <--
     volumes:
       - ./config/loki-config.yaml:/etc/loki/config.yaml:ro
       - data_loki:/loki:rw
@@ -75,7 +77,7 @@ services:
       - grafana-prometheus
     restart: unless-stopped
     healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3100/ready"]
+      test: ["CMD", "wget", "--spider", "-q", "http://loki:3100/ready"]
       interval: 10s     # check every 10 seconds
       timeout: 5s       # wait max 5 seconds per check
       retries: 5        # mark as unhealthy after 5 failures
@@ -83,8 +85,7 @@ services:
   https_portal:
     container_name: https_portal
     image: "steveltn/https-portal:1.21"
-    restart: unless-stopped
-    user: "root"
+    restart: always
     networks:
       - grafana-prometheus
     ports:
@@ -98,6 +99,7 @@ services:
       CLIENT_MAX_BODY_SIZE: 0
       # FORCE_RENEW: 'true'
       DOMAINS: 'grafana.phx-erp.de -> https://grafana:3000, prometheus.phx-erp.de -> https://prometheus:9090'
+      # loki-logs.phx-erp.de -> http://localhost:3100
     volumes:
       - ./https_portal/data:/var/lib/https-portal # ssl_certs, vhost.d, htdocs
       - ./https_portal/log:/var/log/nginx # nginx logs
@@ -105,3 +107,8 @@ services:
       - prometheus
       - grafana
       - loki
+    deploy:
+      resources:
+        limits:
+          cpus: '0.50'
+          memory: 512M

hetzner_discovery.py

@@ -10,17 +10,29 @@ load_dotenv()
 # 🔑 Load Hetzner API Token & Prometheus URL
 HETZNER_API_TOKEN = os.getenv("HETZNER_API_TOKEN")
 # https://prometheus.io/docs/prometheus/latest/configuration/configuration/
-PROMETHEUS_RELOAD_URL = os.getenv("PROMETHEUS_RELOAD_URL", "https://prometheus.phx-erp.de:9090/-/reload")
+PROMETHEUS_RELOAD_URL = os.getenv("PROMETHEUS_RELOAD_URL", "https://prometheus.phx-erp.de/-/reload")
 
 # Check if token is loaded
 if not HETZNER_API_TOKEN:
     raise ValueError("❌ HETZNER_API_TOKEN is missing! Make sure it's set in the .env file.")
 
 # 📂 Paths to output files
-PROMETHEUS_TARGETS_FILE = "/opt/phx/main/config/hetzner_targets.json"
-ERROR_LOG_FILE = "/opt/phx/main/config/hetzner_error_servers.json"
-EXCLUDED_SERVERS_FILE = "/opt/phx/main/config/hetzner_excluded_servers.json"
-DNS_MAPPING_FILE = "/opt/phx/main/config/hetzner_dns_mapping.json"
+# PROMETHEUS_TARGETS_FILE = "/opt/phx/main/config/hetzner_targets.json"
+# ERROR_LOG_FILE = "/opt/phx/main/config/hetzner_error_servers.json"
+# EXCLUDED_SERVERS_FILE = "/opt/phx/main/config/hetzner_excluded_servers.json"
+# DNS_MAPPING_FILE = "/opt/phx/main/config/hetzner_dns_mapping.json"
+
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+HEARTBEAT_TARGETS_FILE = os.path.join(BASE_DIR, "config/hetzner_heartbeat_targets.json")
+PROMETHEUS_TARGETS_FILE = os.path.join(BASE_DIR, "config/hetzner_targets.json")
+PHX_SYSTEM_FILE = os.path.join(BASE_DIR, "config/phoenix_system_hetzner_targets.json")
+PHX_WORKER_FILE = os.path.join(BASE_DIR, "config/phoenix_worker_hetzner_targets.json")
+PHX_HEALTH_EXPORTER_FILE = os.path.join(BASE_DIR, "config/phoenix_health_exporter_hetzner_targets.json")
+DNS_MAPPING_FILE = os.path.join(BASE_DIR, "config/hetzner_dns_mapping.json")
+ERROR_LOG_FILE = os.path.join(BASE_DIR, "config/hetzner_error_servers.json")
+EXCLUDED_SERVERS_FILE = os.path.join(BASE_DIR, "config/hetzner_excluded_servers.json")
 
 # 📌 Hetzner API URL
 HETZNER_API_URL = os.getenv("HETZNER_API_URL")
@@ -30,7 +42,7 @@ if not HETZNER_API_URL:
     raise ValueError("❌ HETZNER_API_URL is missing! Make sure it's set in the .env file.")
 
 # 🛑 List of server names to exclude (DARKLIST)
-DARKLISTED_SERVERS = ["docuvita"]
+DARKLISTED_SERVERS = ["docuvita", "teamcity", "gitea"]
 
 # 📡 Fetch Hetzner server list with pagination support and Rate Limiting handling
 def get_hetzner_servers():
@@ -78,9 +90,13 @@ def get_hetzner_servers():
 def generate_prometheus_sd_config():
     servers = get_hetzner_servers()
     targets = []
+    phx_system_targets = []
+    phx_worker_targets = []
+    phx_health_exporter_targets = []
     error_servers = []
     excluded_servers = []
     dns_mappings = []  # New list for storing DNS-IP mappings
+    heartbeat_targets = []
 
     for server in servers:
         ipv4 = server.get("public_net", {}).get("ipv4", {}).get("ip")
@@ -97,6 +113,7 @@ def generate_prometheus_sd_config():
             continue  # Skip adding to Prometheus targets
 
         if ipv4:
+            # Add to DNS mapping file
             targets.append({
                 "targets": [f"{ipv4}:9100", f"{ipv4}:9113"],
                 "labels": {
@@ -104,7 +121,45 @@ def generate_prometheus_sd_config():
                     "datacenter": datacenter
                 }
             })
-            # Add to DNS mapping file
+
+            # This is with Python Flask server for health checks
+            phx_health_exporter_targets.append({
+                "targets": [f"{ipv4}:9800"],
+                "labels": {
+                    "instance": f"{server_name}-health-exporter",
+                    "datacenter": datacenter
+                }
+            })
+
+            # Add Pushgateway heartbeat endpoint (default Pushgateway port is 9091)
+            heartbeat_targets.append({
+                "targets": [f"{ipv4}:9091"],  # assuming Pushgateway is running on each server
+                "labels": {
+                    "instance": server_name,
+                    "job": "heartbeat"
+                }
+            })
+
+            # Phoenix System metrics (port 3000)
+            # phx_system_targets.append({
+            #     "targets": [f"{ipv4}:3000"],
+            #     "labels": {
+            #         "instance": f"{server_name}",
+            #         "datacenter": datacenter,
+            #         "__metrics_path__": "/health/metrics"
+            #     }
+            # })
+
+            # # Phoenix Worker metrics (port 3001)
+            # phx_worker_targets.append({
+            #     "targets": [f"{ipv4}:3001"],
+            #     "labels": {
+            #         "instance": f"{server_name}",
+            #         "datacenter": datacenter,
+            #         "__metrics_path__": "/health/metrics"
+            #     }
+            # })
+
             dns_mappings.append({
                 "dns_name": server_name,
                 "ip_address": ipv4
@@ -125,6 +180,22 @@ def generate_prometheus_sd_config():
 
     print(f"✅ Updated Prometheus targets in {PROMETHEUS_TARGETS_FILE}")
 
+    with open(PHX_HEALTH_EXPORTER_FILE, "w") as f:
+        json.dump(phx_health_exporter_targets, f, indent=4)
+        print(f"✅ phoenix-health-exporter targets saved to {PHX_HEALTH_EXPORTER_FILE}")
+
+    with open(HEARTBEAT_TARGETS_FILE, "w") as f:
+        json.dump(heartbeat_targets, f, indent=4)
+        print(f"✅ heartbeat targets saved to {HEARTBEAT_TARGETS_FILE}")
+
+    # with open(PHX_SYSTEM_FILE, "w") as f:
+    #     json.dump(phx_system_targets, f, indent=4)
+    #     print(f"✅ phoenix-system targets saved to {PHX_SYSTEM_FILE}")
+
+    # with open(PHX_WORKER_FILE, "w") as f:
+    #     json.dump(phx_worker_targets, f, indent=4)
+    #     print(f"✅ phoenix-worker targets saved to {PHX_WORKER_FILE}")
+
     # Save DNS Mappings file
     with open(DNS_MAPPING_FILE, "w") as f:
         json.dump(dns_mappings, f, indent=4)