Compare commits

..

19 Commits

Author SHA1 Message Date
f4a6be714b 👌 IMPROVE: Update Hetzner DNS mapping and targets configuration by renaming instances, removing obsolete entries, and adding new targets for enhanced monitoring 2025-08-04 16:58:50 +02:00
ddfdb91401 👌 IMPROVE: Add heartbeat targets configuration for Prometheus monitoring and save to new JSON file 2025-08-04 16:58:39 +02:00
7f9f7b1aa4 👌 IMPROVE: Add new health exporter targets configuration and update Prometheus to include the new health exporter job 2025-05-23 09:30:33 +00:00
945cae3908 👌 IMPROVE: Update Hetzner configuration files and scripts to include new targets, enhance health check metrics, and organize file paths for better maintainability 2025-05-22 10:28:08 +00:00
61d056dcab 👌 IMPROVE: Enhance Loki configuration with additional server settings and limits configuration 2025-05-19 11:13:48 +00:00
011aa2d613 👌 IMPROVE: Update Loki service configuration to expand environment variables and fix healthcheck URL 2025-05-19 11:13:31 +00:00
cab9ccf1af 👌 IMPROVE: Update Hetzner DNS mapping and targets with new instances and IP addresses 2025-05-19 10:18:31 +00:00
50a4670c53 👌 IMPROVE: Refactor Grafana environment variables for consistency and add resource limits for Loki service 2025-05-19 10:18:18 +00:00
38a2439b98 👌 IMPROVE: Fix Prometheus reload URL to use the correct port 2025-05-19 10:18:03 +00:00
43ba5e60b7 👌 IMPROVE: Add additional metrics endpoint for Hetzner server targets in configuration 2025-05-13 14:21:38 +00:00
89519ef1f3 👌 IMPROVE: Update Hetzner server targets in discovery script to include additional metrics endpoint 2025-05-13 14:18:19 +00:00
938eaf2f5f 📦 NEW: Add Loki service and configuration to Docker Compose setup 2025-05-13 10:58:37 +00:00
84db17fa0e 👌 IMPROVE: Update Prometheus and Grafana images, enhance Hetzner DNS mapping, and add Prometheus reload functionality 2025-05-13 10:58:11 +00:00
ebac8ff7e5 👌 IMPROVE: 2025-01-30 14:47:40 +00:00
ebf1ab5985 📦 NEW: DEV-871 Add Hetzner Plugin to Grafana monitor all Server Dynamic 2025-01-30 13:01:50 +00:00
a4d9888bfe 👌 IMPROVE: New Targets 2024-07-28 12:42:08 +00:00
eefc954946 👌 IMPROVE: TLS Connections 2024-07-23 15:43:08 +00:00
ba02008e7c 🐛 FIX: yml 2024-07-23 14:27:01 +00:00
3212a09b69 👌 IMPROVE: Add Grafana Domain 2024-07-23 13:21:13 +00:00
18 changed files with 1942 additions and 146 deletions

1
.gitignore vendored
View File

@@ -8,3 +8,4 @@ e2e
https_portal/log
https_portal/data
assets
.env

6
Dockerfile.prometheus Normal file
View File

@@ -0,0 +1,6 @@
FROM docker.io/prom/prometheus:v3.1.0
# Set the proper permissions for certificates
RUN if [ -f ./https_portal/data/prometheus.phx-erp.de/production/signed.crt ]; then chmod 644 ./https_portal/data/prometheus.phx-erp.de/production/signed.crt; fi && \
if [ -f ./https_portal/data/prometheus.phx-erp.de/production/domain.key ]; then chmod 600 ./https_portal/data/prometheus.phx-erp.de/production/domain.key; fi
CMD ["prometheus", "--config.file=/etc/prometheus/prometheus.yaml", "--web.config.file=/etc/prometheus/web-config.yaml", "--web.external-url=https://prometheus.phx-erp.de"]

View File

@@ -0,0 +1,12 @@
[
{
"targets": [
"https://alpha.phx-erp.de"
],
"labels": {
"job": "blackbox-dynamic",
"instance": "Alpha",
"probe": "http"
}
}
]

View File

@@ -0,0 +1,186 @@
[
{
"dns_name": "PHX-DEV-001.Alpha",
"ip_address": "157.90.161.42"
},
{
"dns_name": "ANSIBLE-MASTER",
"ip_address": "167.235.254.4"
},
{
"dns_name": "cts",
"ip_address": "116.203.75.215"
},
{
"dns_name": "Phx-Yuri",
"ip_address": "162.55.52.253"
},
{
"dns_name": "benzinger-demo",
"ip_address": "162.55.54.75"
},
{
"dns_name": "phx-internal",
"ip_address": "5.75.183.139"
},
{
"dns_name": "trachtenmode-schmid",
"ip_address": "142.132.165.231"
},
{
"dns_name": "phx-beta",
"ip_address": "159.69.200.205"
},
{
"dns_name": "lhl",
"ip_address": "116.203.53.137"
},
{
"dns_name": "Grafana-Prometheus",
"ip_address": "5.75.153.161"
},
{
"dns_name": "cooper",
"ip_address": "159.69.44.39"
},
{
"dns_name": "shipxpert",
"ip_address": "49.13.165.13"
},
{
"dns_name": "sartissohn",
"ip_address": "188.245.44.219"
},
{
"dns_name": "guntli",
"ip_address": "116.203.92.218"
},
{
"dns_name": "kolb",
"ip_address": "94.130.77.57"
},
{
"dns_name": "ried",
"ip_address": "116.203.151.20"
},
{
"dns_name": "heba",
"ip_address": "116.203.68.120"
},
{
"dns_name": "eicsoft",
"ip_address": "116.203.128.69"
},
{
"dns_name": "ck-vechta",
"ip_address": "159.69.93.252"
},
{
"dns_name": "eeparts",
"ip_address": "116.203.46.171"
},
{
"dns_name": "big-break-changes",
"ip_address": "167.235.130.242"
},
{
"dns_name": "PHX-License-Server",
"ip_address": "188.245.32.214"
},
{
"dns_name": "PHX-SkyNodus",
"ip_address": "138.199.201.35"
},
{
"dns_name": "bode",
"ip_address": "188.245.108.133"
},
{
"dns_name": "skr04",
"ip_address": "168.119.174.181"
},
{
"dns_name": "artwin",
"ip_address": "91.99.24.53"
},
{
"dns_name": "amplid",
"ip_address": "168.119.231.96"
},
{
"dns_name": "schwarz",
"ip_address": "116.203.143.63"
},
{
"dns_name": "skr03",
"ip_address": "91.99.53.67"
},
{
"dns_name": "moonich",
"ip_address": "91.99.52.253"
},
{
"dns_name": "comp-sys",
"ip_address": "91.99.76.156"
},
{
"dns_name": "skr-ch",
"ip_address": "78.46.214.224"
},
{
"dns_name": "wsoft",
"ip_address": "116.203.18.156"
},
{
"dns_name": "bzp",
"ip_address": "91.99.71.224"
},
{
"dns_name": "dss",
"ip_address": "49.13.197.152"
},
{
"dns_name": "modern",
"ip_address": "157.90.228.52"
},
{
"dns_name": "burg-itc",
"ip_address": "23.88.56.89"
},
{
"dns_name": "capricorn-it",
"ip_address": "91.99.109.3"
},
{
"dns_name": "kundw",
"ip_address": "91.99.140.34"
},
{
"dns_name": "dieweingoetter",
"ip_address": "128.140.33.105"
},
{
"dns_name": "Rustdesk-Relay-Server",
"ip_address": "94.130.108.153"
},
{
"dns_name": "rihnet",
"ip_address": "91.99.226.250"
},
{
"dns_name": "infoplus",
"ip_address": "49.13.205.27"
},
{
"dns_name": "ai-it",
"ip_address": "195.201.31.160"
},
{
"dns_name": "guder",
"ip_address": "91.99.143.88"
},
{
"dns_name": "testRedisHeathCheck",
"ip_address": "91.107.235.238"
}
]

View File

@@ -0,0 +1,14 @@
[
{
"server_id": 41663080,
"name": "teamcity",
"datacenter": "nbg1",
"reason": "Darklisted server"
},
{
"server_id": 41739626,
"name": "gitea",
"datacenter": "nbg1",
"reason": "Darklisted server"
}
]

View File

@@ -0,0 +1,416 @@
[
{
"targets": [
"157.90.161.42:9091"
],
"labels": {
"instance": "PHX-DEV-001.Alpha",
"job": "heartbeat"
}
},
{
"targets": [
"167.235.254.4:9091"
],
"labels": {
"instance": "ANSIBLE-MASTER",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.75.215:9091"
],
"labels": {
"instance": "cts",
"job": "heartbeat"
}
},
{
"targets": [
"162.55.52.253:9091"
],
"labels": {
"instance": "Phx-Yuri",
"job": "heartbeat"
}
},
{
"targets": [
"162.55.54.75:9091"
],
"labels": {
"instance": "benzinger-demo",
"job": "heartbeat"
}
},
{
"targets": [
"5.75.183.139:9091"
],
"labels": {
"instance": "phx-internal",
"job": "heartbeat"
}
},
{
"targets": [
"142.132.165.231:9091"
],
"labels": {
"instance": "trachtenmode-schmid",
"job": "heartbeat"
}
},
{
"targets": [
"159.69.200.205:9091"
],
"labels": {
"instance": "phx-beta",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.53.137:9091"
],
"labels": {
"instance": "lhl",
"job": "heartbeat"
}
},
{
"targets": [
"5.75.153.161:9091"
],
"labels": {
"instance": "Grafana-Prometheus",
"job": "heartbeat"
}
},
{
"targets": [
"159.69.44.39:9091"
],
"labels": {
"instance": "cooper",
"job": "heartbeat"
}
},
{
"targets": [
"49.13.165.13:9091"
],
"labels": {
"instance": "shipxpert",
"job": "heartbeat"
}
},
{
"targets": [
"188.245.44.219:9091"
],
"labels": {
"instance": "sartissohn",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.92.218:9091"
],
"labels": {
"instance": "guntli",
"job": "heartbeat"
}
},
{
"targets": [
"94.130.77.57:9091"
],
"labels": {
"instance": "kolb",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.151.20:9091"
],
"labels": {
"instance": "ried",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.68.120:9091"
],
"labels": {
"instance": "heba",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.128.69:9091"
],
"labels": {
"instance": "eicsoft",
"job": "heartbeat"
}
},
{
"targets": [
"159.69.93.252:9091"
],
"labels": {
"instance": "ck-vechta",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.46.171:9091"
],
"labels": {
"instance": "eeparts",
"job": "heartbeat"
}
},
{
"targets": [
"167.235.130.242:9091"
],
"labels": {
"instance": "big-break-changes",
"job": "heartbeat"
}
},
{
"targets": [
"188.245.32.214:9091"
],
"labels": {
"instance": "PHX-License-Server",
"job": "heartbeat"
}
},
{
"targets": [
"138.199.201.35:9091"
],
"labels": {
"instance": "PHX-SkyNodus",
"job": "heartbeat"
}
},
{
"targets": [
"188.245.108.133:9091"
],
"labels": {
"instance": "bode",
"job": "heartbeat"
}
},
{
"targets": [
"168.119.174.181:9091"
],
"labels": {
"instance": "skr04",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.24.53:9091"
],
"labels": {
"instance": "artwin",
"job": "heartbeat"
}
},
{
"targets": [
"168.119.231.96:9091"
],
"labels": {
"instance": "amplid",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.143.63:9091"
],
"labels": {
"instance": "schwarz",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.53.67:9091"
],
"labels": {
"instance": "skr03",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.52.253:9091"
],
"labels": {
"instance": "moonich",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.76.156:9091"
],
"labels": {
"instance": "comp-sys",
"job": "heartbeat"
}
},
{
"targets": [
"78.46.214.224:9091"
],
"labels": {
"instance": "skr-ch",
"job": "heartbeat"
}
},
{
"targets": [
"116.203.18.156:9091"
],
"labels": {
"instance": "wsoft",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.71.224:9091"
],
"labels": {
"instance": "bzp",
"job": "heartbeat"
}
},
{
"targets": [
"49.13.197.152:9091"
],
"labels": {
"instance": "dss",
"job": "heartbeat"
}
},
{
"targets": [
"157.90.228.52:9091"
],
"labels": {
"instance": "modern",
"job": "heartbeat"
}
},
{
"targets": [
"23.88.56.89:9091"
],
"labels": {
"instance": "burg-itc",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.109.3:9091"
],
"labels": {
"instance": "capricorn-it",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.140.34:9091"
],
"labels": {
"instance": "kundw",
"job": "heartbeat"
}
},
{
"targets": [
"128.140.33.105:9091"
],
"labels": {
"instance": "dieweingoetter",
"job": "heartbeat"
}
},
{
"targets": [
"94.130.108.153:9091"
],
"labels": {
"instance": "Rustdesk-Relay-Server",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.226.250:9091"
],
"labels": {
"instance": "rihnet",
"job": "heartbeat"
}
},
{
"targets": [
"49.13.205.27:9091"
],
"labels": {
"instance": "infoplus",
"job": "heartbeat"
}
},
{
"targets": [
"195.201.31.160:9091"
],
"labels": {
"instance": "ai-it",
"job": "heartbeat"
}
},
{
"targets": [
"91.99.143.88:9091"
],
"labels": {
"instance": "guder",
"job": "heartbeat"
}
},
{
"targets": [
"91.107.235.238:9091"
],
"labels": {
"instance": "testRedisHeathCheck",
"job": "heartbeat"
}
}
]

462
config/hetzner_targets.json Normal file
View File

@@ -0,0 +1,462 @@
[
{
"targets": [
"157.90.161.42:9100",
"157.90.161.42:9113"
],
"labels": {
"instance": "PHX-DEV-001.Alpha",
"datacenter": "nbg1"
}
},
{
"targets": [
"167.235.254.4:9100",
"167.235.254.4:9113"
],
"labels": {
"instance": "ANSIBLE-MASTER",
"datacenter": "fsn1"
}
},
{
"targets": [
"116.203.75.215:9100",
"116.203.75.215:9113"
],
"labels": {
"instance": "cts",
"datacenter": "nbg1"
}
},
{
"targets": [
"162.55.52.253:9100",
"162.55.52.253:9113"
],
"labels": {
"instance": "Phx-Yuri",
"datacenter": "nbg1"
}
},
{
"targets": [
"162.55.54.75:9100",
"162.55.54.75:9113"
],
"labels": {
"instance": "benzinger-demo",
"datacenter": "nbg1"
}
},
{
"targets": [
"5.75.183.139:9100",
"5.75.183.139:9113"
],
"labels": {
"instance": "phx-internal",
"datacenter": "nbg1"
}
},
{
"targets": [
"142.132.165.231:9100",
"142.132.165.231:9113"
],
"labels": {
"instance": "trachtenmode-schmid",
"datacenter": "nbg1"
}
},
{
"targets": [
"159.69.200.205:9100",
"159.69.200.205:9113"
],
"labels": {
"instance": "phx-beta",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.53.137:9100",
"116.203.53.137:9113"
],
"labels": {
"instance": "lhl",
"datacenter": "nbg1"
}
},
{
"targets": [
"5.75.153.161:9100",
"5.75.153.161:9113"
],
"labels": {
"instance": "Grafana-Prometheus",
"datacenter": "nbg1"
}
},
{
"targets": [
"159.69.44.39:9100",
"159.69.44.39:9113"
],
"labels": {
"instance": "cooper",
"datacenter": "nbg1"
}
},
{
"targets": [
"49.13.165.13:9100",
"49.13.165.13:9113"
],
"labels": {
"instance": "shipxpert",
"datacenter": "nbg1"
}
},
{
"targets": [
"188.245.44.219:9100",
"188.245.44.219:9113"
],
"labels": {
"instance": "sartissohn",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.92.218:9100",
"116.203.92.218:9113"
],
"labels": {
"instance": "guntli",
"datacenter": "nbg1"
}
},
{
"targets": [
"94.130.77.57:9100",
"94.130.77.57:9113"
],
"labels": {
"instance": "kolb",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.151.20:9100",
"116.203.151.20:9113"
],
"labels": {
"instance": "ried",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.68.120:9100",
"116.203.68.120:9113"
],
"labels": {
"instance": "heba",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.128.69:9100",
"116.203.128.69:9113"
],
"labels": {
"instance": "eicsoft",
"datacenter": "nbg1"
}
},
{
"targets": [
"159.69.93.252:9100",
"159.69.93.252:9113"
],
"labels": {
"instance": "ck-vechta",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.46.171:9100",
"116.203.46.171:9113"
],
"labels": {
"instance": "eeparts",
"datacenter": "nbg1"
}
},
{
"targets": [
"167.235.130.242:9100",
"167.235.130.242:9113"
],
"labels": {
"instance": "big-break-changes",
"datacenter": "nbg1"
}
},
{
"targets": [
"188.245.32.214:9100",
"188.245.32.214:9113"
],
"labels": {
"instance": "PHX-License-Server",
"datacenter": "nbg1"
}
},
{
"targets": [
"138.199.201.35:9100",
"138.199.201.35:9113"
],
"labels": {
"instance": "PHX-SkyNodus",
"datacenter": "nbg1"
}
},
{
"targets": [
"188.245.108.133:9100",
"188.245.108.133:9113"
],
"labels": {
"instance": "bode",
"datacenter": "nbg1"
}
},
{
"targets": [
"168.119.174.181:9100",
"168.119.174.181:9113"
],
"labels": {
"instance": "skr04",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.24.53:9100",
"91.99.24.53:9113"
],
"labels": {
"instance": "artwin",
"datacenter": "nbg1"
}
},
{
"targets": [
"168.119.231.96:9100",
"168.119.231.96:9113"
],
"labels": {
"instance": "amplid",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.143.63:9100",
"116.203.143.63:9113"
],
"labels": {
"instance": "schwarz",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.53.67:9100",
"91.99.53.67:9113"
],
"labels": {
"instance": "skr03",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.52.253:9100",
"91.99.52.253:9113"
],
"labels": {
"instance": "moonich",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.76.156:9100",
"91.99.76.156:9113"
],
"labels": {
"instance": "comp-sys",
"datacenter": "nbg1"
}
},
{
"targets": [
"78.46.214.224:9100",
"78.46.214.224:9113"
],
"labels": {
"instance": "skr-ch",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.18.156:9100",
"116.203.18.156:9113"
],
"labels": {
"instance": "wsoft",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.71.224:9100",
"91.99.71.224:9113"
],
"labels": {
"instance": "bzp",
"datacenter": "nbg1"
}
},
{
"targets": [
"49.13.197.152:9100",
"49.13.197.152:9113"
],
"labels": {
"instance": "dss",
"datacenter": "nbg1"
}
},
{
"targets": [
"157.90.228.52:9100",
"157.90.228.52:9113"
],
"labels": {
"instance": "modern",
"datacenter": "nbg1"
}
},
{
"targets": [
"23.88.56.89:9100",
"23.88.56.89:9113"
],
"labels": {
"instance": "burg-itc",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.109.3:9100",
"91.99.109.3:9113"
],
"labels": {
"instance": "capricorn-it",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.140.34:9100",
"91.99.140.34:9113"
],
"labels": {
"instance": "kundw",
"datacenter": "nbg1"
}
},
{
"targets": [
"128.140.33.105:9100",
"128.140.33.105:9113"
],
"labels": {
"instance": "dieweingoetter",
"datacenter": "nbg1"
}
},
{
"targets": [
"94.130.108.153:9100",
"94.130.108.153:9113"
],
"labels": {
"instance": "Rustdesk-Relay-Server",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.226.250:9100",
"91.99.226.250:9113"
],
"labels": {
"instance": "rihnet",
"datacenter": "nbg1"
}
},
{
"targets": [
"49.13.205.27:9100",
"49.13.205.27:9113"
],
"labels": {
"instance": "infoplus",
"datacenter": "nbg1"
}
},
{
"targets": [
"195.201.31.160:9100",
"195.201.31.160:9113"
],
"labels": {
"instance": "ai-it",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.143.88:9100",
"91.99.143.88:9113"
],
"labels": {
"instance": "guder",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.107.235.238:9100",
"91.107.235.238:9113"
],
"labels": {
"instance": "testRedisHeathCheck",
"datacenter": "nbg1"
}
}
]

49
config/loki-config.yaml Normal file
View File

@@ -0,0 +1,49 @@
auth_enabled: false
server:
http_listen_port: 3100
http_server_read_timeout: 1m
http_server_write_timeout: 1m
grpc_listen_port: 9095
log_level: info
log_format: logfmt
common:
instance_addr: 127.0.0.1
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory # ✅ Use inmemory for single-node setups
limits_config:
reject_old_samples: true
reject_old_samples_max_age: 168h
ingestion_rate_mb: 10
ingestion_burst_size_mb: 20
max_query_parallelism: 8
max_query_lookback: 168h
schema_config:
configs:
- from: 2020-10-24
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
table_manager:
retention_deletes_enabled: true
retention_period: 168h
ruler:
alertmanager_url: http://localhost:9093
rule_path: /loki/rules
enable_alertmanager_v2: true
enable_api: true

View File

@@ -0,0 +1,416 @@
[
{
"targets": [
"157.90.161.42:9800"
],
"labels": {
"instance": "PHX-DEV-001.Alpha-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"167.235.254.4:9800"
],
"labels": {
"instance": "ANSIBLE-MASTER-health-exporter",
"datacenter": "fsn1"
}
},
{
"targets": [
"116.203.75.215:9800"
],
"labels": {
"instance": "cts-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"162.55.52.253:9800"
],
"labels": {
"instance": "Phx-Yuri-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"162.55.54.75:9800"
],
"labels": {
"instance": "benzinger-demo-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"5.75.183.139:9800"
],
"labels": {
"instance": "phx-internal-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"142.132.165.231:9800"
],
"labels": {
"instance": "trachtenmode-schmid-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"159.69.200.205:9800"
],
"labels": {
"instance": "phx-beta-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.53.137:9800"
],
"labels": {
"instance": "lhl-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"5.75.153.161:9800"
],
"labels": {
"instance": "Grafana-Prometheus-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"159.69.44.39:9800"
],
"labels": {
"instance": "cooper-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"49.13.165.13:9800"
],
"labels": {
"instance": "shipxpert-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"188.245.44.219:9800"
],
"labels": {
"instance": "sartissohn-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.92.218:9800"
],
"labels": {
"instance": "guntli-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"94.130.77.57:9800"
],
"labels": {
"instance": "kolb-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.151.20:9800"
],
"labels": {
"instance": "ried-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.68.120:9800"
],
"labels": {
"instance": "heba-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.128.69:9800"
],
"labels": {
"instance": "eicsoft-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"159.69.93.252:9800"
],
"labels": {
"instance": "ck-vechta-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.46.171:9800"
],
"labels": {
"instance": "eeparts-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"167.235.130.242:9800"
],
"labels": {
"instance": "big-break-changes-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"188.245.32.214:9800"
],
"labels": {
"instance": "PHX-License-Server-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"138.199.201.35:9800"
],
"labels": {
"instance": "PHX-SkyNodus-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"188.245.108.133:9800"
],
"labels": {
"instance": "bode-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"168.119.174.181:9800"
],
"labels": {
"instance": "skr04-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.24.53:9800"
],
"labels": {
"instance": "artwin-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"168.119.231.96:9800"
],
"labels": {
"instance": "amplid-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.143.63:9800"
],
"labels": {
"instance": "schwarz-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.53.67:9800"
],
"labels": {
"instance": "skr03-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.52.253:9800"
],
"labels": {
"instance": "moonich-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.76.156:9800"
],
"labels": {
"instance": "comp-sys-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"78.46.214.224:9800"
],
"labels": {
"instance": "skr-ch-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"116.203.18.156:9800"
],
"labels": {
"instance": "wsoft-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.71.224:9800"
],
"labels": {
"instance": "bzp-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"49.13.197.152:9800"
],
"labels": {
"instance": "dss-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"157.90.228.52:9800"
],
"labels": {
"instance": "modern-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"23.88.56.89:9800"
],
"labels": {
"instance": "burg-itc-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.109.3:9800"
],
"labels": {
"instance": "capricorn-it-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.140.34:9800"
],
"labels": {
"instance": "kundw-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"128.140.33.105:9800"
],
"labels": {
"instance": "dieweingoetter-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"94.130.108.153:9800"
],
"labels": {
"instance": "Rustdesk-Relay-Server-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.226.250:9800"
],
"labels": {
"instance": "rihnet-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"49.13.205.27:9800"
],
"labels": {
"instance": "infoplus-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"195.201.31.160:9800"
],
"labels": {
"instance": "ai-it-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.99.143.88:9800"
],
"labels": {
"instance": "guder-health-exporter",
"datacenter": "nbg1"
}
},
{
"targets": [
"91.107.235.238:9800"
],
"labels": {
"instance": "testRedisHeathCheck-health-exporter",
"datacenter": "nbg1"
}
}
]

View File

@@ -0,0 +1 @@
[]

View File

@@ -0,0 +1 @@
[]

View File

@@ -1,80 +1,43 @@
global:
scrape_interval: 15s # By default, scrape targets every 15 seconds.
scrape_interval: 15s # ⏱️ Default interval Prometheus uses to scrape metrics from all targets, unless overridden in specific jobs.
# Attach these labels to any time series or alerts when communicating with
# external systems (federation, remote storage, Alertmanager).
# external_labels:
# monitor: 'codelab-monitor'
# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
- job_name: "Prometheus Server"
scrape_interval: 5s
static_configs:
- targets: ["5.75.153.161:9090"]
# Existing Hetzner Dynamic Configuration
- job_name: "hetzner-dynamic" # 🏷️ Logical name for the scrape job (used in metrics as job="hetzner-dynamic").
file_sd_configs: # 📂 Enable file-based service discovery. Prometheus will look at files to dynamically load targets.
- files:
- "/opt/phx/main/config/hetzner_targets.json" # 📁 JSON file containing the list of targets (e.g. IPs, ports, labels).
refresh_interval: 30s # 🔁 How often Prometheus checks the file for changes and updates the target list.
scrape_interval: 5s # ⏱️ Override global default: scrape metrics from each target in this job every x seconds.
scheme: http # 🌐 Use plain HTTP when scraping the targets (not HTTPS).
- job_name: 'node'
scrape_interval: 5s
static_configs:
- targets: ['5.75.153.161:9100']
- job_name: "phoenix-health-exporter"
file_sd_configs:
- files:
- "/opt/phx/main/config/phoenix_health_exporter_hetzner_targets.json"
refresh_interval: 30s
scrape_interval: 2m # Change it carefully, it is a heavy job to keep open and close to check the health of DB connection, there annotation to refactor it and use a new pgDebounce image.
scheme: http
- job_name: "Phx Yuri"
scrape_interval: 5s
static_configs:
- targets: ["162.55.52.253:9100"]
- job_name: "heartbeat"
file_sd_configs:
- files:
- "/opt/phx/main/config/hetzner_heartbeat_targets.json"
refresh_interval: 30s
honor_labels: true
- job_name: "lhl"
scrape_interval: 5s
static_configs:
- targets: ["116.203.53.137:9100"]
- job_name: "Phx Beta RC"
scrape_interval: 5s
static_configs:
- targets: ["116.203.53.137:9100"]
- job_name: "Gitea"
scrape_interval: 5s
static_configs:
- targets: ["49.13.146.138:9100"]
- job_name: "Team City"
scrape_interval: 5s
static_configs:
- targets: ["23.88.107.109:9100"]
# - job_name: "Ambio"
# - job_name: "phoenix-system"
# file_sd_configs:
# - files:
# - "/opt/phx/main/config/phoenix_system_hetzner_targets.json"
# refresh_interval: 30s
# scrape_interval: 5s
# static_configs:
# - targets: ["116.203.28.202:9100"]
# scheme: http
- job_name: "Trachtenmode Schmid"
scrape_interval: 5s
static_configs:
- targets: ["142.132.165.231:9100"]
- job_name: "Phx Internal"
scrape_interval: 5s
static_configs:
- targets: ["5.75.183.139:9100"]
- job_name: "Benzinger Demo"
scrape_interval: 5s
static_configs:
- targets: ["162.55.54.75:9100"]
- job_name: "CTS"
scrape_interval: 5s
static_configs:
- targets: ["116.203.75.215:9100"]
- job_name: "Ansible Master"
scrape_interval: 5s
static_configs:
- targets: ["167.235.254.4:9100"]
- job_name: "Alpha"
scrape_interval: 5s
static_configs:
- targets: ["157.90.161.42:9100"]
# - job_name: "phoenix-worker"
# file_sd_configs:
# - files:
# - "/opt/phx/main/config/phoenix_worker_hetzner_targets.json"
# refresh_interval: 30s
# scrape_interval: 5s
# scheme: http

3
config/web-config.yaml Normal file
View File

@@ -0,0 +1,3 @@
tls_server_config:
cert_file: /etc/prometheus/certs/signed.crt
key_file: /etc/prometheus/certs/domain.key

View File

@@ -4,68 +4,90 @@ volumes:
driver: local
prometheus-data:
driver: local
data_loki:
driver: local
networks:
default:
external:
name: grafana-prometheus
grafana-prometheus:
external: true
services:
grafana:
image: docker.io/grafana/grafana-oss:10.4.2
image: docker.io/grafana/grafana-oss:12.0.0
container_name: grafana
environment:
- GF_SECURITY_ADMIN_PASSWORD=admin
- GF_LOG_LEVEL=debug
- GF_SMTP_ENABLED=true
- GF_STMP_FROM_ADDRESS=admin@phx-erp.de
- GF_SMTP_FROM_NAME=Grafana
- GF_SMTP_HOST=mail.phx-erp.de
- GF_SMTP_PASSWORD=0rB0@et68
- GF_SMTP_USER=yuri.lima@phx-erp.de
- GF_SMTP_STARTTLS_POLICY=false
- GF_SMTP_SKIP_VERIFY=false
- GF_SMTP_ENABLE_TRACING=false
ports:
- "3000:3000"
GF_PATHS_PROVISIONING: /var/lib/grafana/provisioning/
GF_SECURITY_ADMIN_PASSWORD: admin
# - GF_SERVER_DOMAIN=%(protocol)s://%(domain)s:%(http_port)s/
# - GF_SERVER_ROOT_URL=%(protocol)s://%(domain)s/
GF_SERVER_DOMAIN: grafana.phx-erp.de
GF_SERVER_PROTOCOL: https
# - GF_SERVER_HTTP_PORT=3000
GF_LOG_LEVEL: debug
GF_SMTP_ENABLED: true
GF_SMTP_FROM_ADDRESS: admin@phx-erp.de
GF_SMTP_FROM_NAME: Grafana
GF_SMTP_HOST: mail.phx-erp.de:465
GF_SMTP_PASSWORD: 0rB0@et68
GF_SMTP_USER: yuri.lima@phx-erp.de
GF_SMTP_STARTTLS_POLICY: false
GF_SMTP_SKIP_VERIFY: true
GF_SMTP_ENABLE_TRACING: false
volumes:
- grafana-data:/var/lib/grafana
- ./config/provisioning:/var/lib/grafana/provisioning/
restart: unless-stopped
networks:
- default
- grafana-prometheus
depends_on:
- prometheus
prometheus:
image: docker.io/prom/prometheus:v2.51.2
image: docker.io/prom/prometheus:v3.1.0
container_name: prometheus
ports:
- 9090:9090
command: "--config.file=/etc/prometheus/prometheus.yaml"
user: "65534:65534"
# ports:
# - "9090:9090"
command:
- "--config.file=/etc/prometheus/prometheus.yaml"
- "--web.config.file=/etc/prometheus/web-config.yaml"
- "--web.external-url=https://prometheus.phx-erp.de"
- "--web.enable-lifecycle" # 🔹 Enable dynamic config reload
volumes:
# - '--web.enable-lifecycle' # Enable reload of configuration automatically without restart !!!
- ./config/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro # RO Read-only
- ./config/web-config.yaml:/etc/prometheus/web-config.yaml:ro # RO Read-only
- ./https_portal/data/prometheus.phx-erp.de/production:/etc/prometheus/certs
- ./config:/opt/phx/main/config # 🔹 Add this to mount the config folder
- prometheus-data:/prometheus
- ./https_portal/data/prometheus.phx-erp.de/production/signed.crt:/etc/prometheus/certs/signed.crt:ro
- ./https_portal/data/prometheus.phx-erp.de/production/domain.key:/etc/prometheus/certs/domain.key:ro
restart: unless-stopped
networks:
- default
node_exporter:
image: quay.io/prometheus/node-exporter:latest
container_name: node_exporter
network_mode: host
pid: host
restart: unless-stopped
- grafana-prometheus
loki:
container_name: loki
image: docker.io/grafana/loki:3.5.0
command:
- "--path.procfs=/host/proc"
- "--path.sysfs=/host/sys"
- "--path.rootfs=/host"
- "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev)($$|/)"
- "-config.file=/etc/loki/config.yaml" # Tells Grafana Loki to start with the config file.
- "-config.expand-env=true"
ports:
- "3100:3100"
volumes:
- "/proc:/host/proc:ro"
- "/sys:/host/sys:ro"
- "/:/host:ro,rslave"
- ./config/loki-config.yaml:/etc/loki/config.yaml:ro
- data_loki:/loki:rw
networks:
- grafana-prometheus
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "--spider", "-q", "http://loki:3100/ready"]
interval: 10s # check every 10 seconds
timeout: 5s # wait max 5 seconds per check
retries: 5 # mark as unhealthy after 5 failures
start_period: 20s # wait 20s after container starts before first check
https_portal:
container_name: https_portal
image: "steveltn/https-portal:1.21"
restart: unless-stopped
user: "root"
restart: always
networks:
- default
- grafana-prometheus
ports:
- "80:80"
- "443:443"
@@ -76,7 +98,17 @@ services:
RENEW_MARGIN_DAYS: 30
CLIENT_MAX_BODY_SIZE: 0
# FORCE_RENEW: 'true'
DOMAINS: 'grafana.phx-erp.de -> 5.75.153.161:3000, prometheus.phx-erp.de -> 5.75.153.161:9090'
DOMAINS: 'grafana.phx-erp.de -> https://grafana:3000, prometheus.phx-erp.de -> https://prometheus:9090'
# loki-logs.phx-erp.de -> http://localhost:3100
volumes:
- ./https_portal/data:/var/lib/https-portal # ssl_certs, vhost.d, htdocs
- ./https_portal/log:/var/log/nginx # nginx logs
depends_on:
- prometheus
- grafana
- loki
deploy:
resources:
limits:
cpus: '0.50'
memory: 512M

231
hetzner_discovery.py Normal file
View File

@@ -0,0 +1,231 @@
import requests
import json
import os
import time
from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
# 🔑 Load Hetzner API Token & Prometheus URL
HETZNER_API_TOKEN = os.getenv("HETZNER_API_TOKEN")
# https://prometheus.io/docs/prometheus/latest/configuration/configuration/
PROMETHEUS_RELOAD_URL = os.getenv("PROMETHEUS_RELOAD_URL", "https://prometheus.phx-erp.de/-/reload")
# Check if token is loaded
if not HETZNER_API_TOKEN:
raise ValueError("❌ HETZNER_API_TOKEN is missing! Make sure it's set in the .env file.")
# 📂 Paths to output files
# PROMETHEUS_TARGETS_FILE = "/opt/phx/main/config/hetzner_targets.json"
# ERROR_LOG_FILE = "/opt/phx/main/config/hetzner_error_servers.json"
# EXCLUDED_SERVERS_FILE = "/opt/phx/main/config/hetzner_excluded_servers.json"
# DNS_MAPPING_FILE = "/opt/phx/main/config/hetzner_dns_mapping.json"
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
HEARTBEAT_TARGETS_FILE = os.path.join(BASE_DIR, "config/hetzner_heartbeat_targets.json")
PROMETHEUS_TARGETS_FILE = os.path.join(BASE_DIR, "config/hetzner_targets.json")
PHX_SYSTEM_FILE = os.path.join(BASE_DIR, "config/phoenix_system_hetzner_targets.json")
PHX_WORKER_FILE = os.path.join(BASE_DIR, "config/phoenix_worker_hetzner_targets.json")
PHX_HEALTH_EXPORTER_FILE = os.path.join(BASE_DIR, "config/phoenix_health_exporter_hetzner_targets.json")
DNS_MAPPING_FILE = os.path.join(BASE_DIR, "config/hetzner_dns_mapping.json")
ERROR_LOG_FILE = os.path.join(BASE_DIR, "config/hetzner_error_servers.json")
EXCLUDED_SERVERS_FILE = os.path.join(BASE_DIR, "config/hetzner_excluded_servers.json")
# 📌 Hetzner API URL
HETZNER_API_URL = os.getenv("HETZNER_API_URL")
# Check if API URL is loaded
if not HETZNER_API_URL:
raise ValueError("❌ HETZNER_API_URL is missing! Make sure it's set in the .env file.")
# 🛑 List of server names to exclude (DARKLIST)
DARKLISTED_SERVERS = ["docuvita", "teamcity", "gitea"]
# 📡 Fetch Hetzner server list with pagination support and Rate Limiting handling
def get_hetzner_servers():
headers = {"Authorization": f"Bearer {HETZNER_API_TOKEN}"}
all_servers = []
page = 1
per_page = 50 # Max per request
while True:
response = requests.get(
f"{HETZNER_API_URL}?page={page}&per_page={per_page}", headers=headers
)
# Handle Rate Limiting (429 Too Many Requests)
if response.status_code == 429:
reset_time = int(response.headers.get("RateLimit-Reset", time.time() + 60))
wait_time = reset_time - int(time.time())
print(f"⏳ Rate limit exceeded! Waiting {wait_time} seconds until reset...")
time.sleep(wait_time)
continue # Retry after waiting
if response.status_code != 200:
print(f"❌ Error fetching servers: {response.text}")
break # Stop fetching if error
# Read rate limit headers
remaining = response.headers.get("RateLimit-Remaining", "unknown")
reset_time = int(response.headers.get("RateLimit-Reset", time.time() + 60))
print(f"📊 API Rate Limit: {remaining} requests remaining. Next reset at {reset_time}.")
data = response.json()
servers = data.get("servers", [])
all_servers.extend(servers)
# Check if there's a next page
pagination = data.get("meta", {}).get("pagination", {})
if not pagination.get("next_page"):
break # Exit if no more pages
page = pagination["next_page"] # Move to the next page
return all_servers
# 🏗️ Generate Prometheus JSON file and DNS Mapping
def generate_prometheus_sd_config():
servers = get_hetzner_servers()
targets = []
phx_system_targets = []
phx_worker_targets = []
phx_health_exporter_targets = []
error_servers = []
excluded_servers = []
dns_mappings = [] # New list for storing DNS-IP mappings
heartbeat_targets = []
for server in servers:
ipv4 = server.get("public_net", {}).get("ipv4", {}).get("ip")
server_name = server["name"]
datacenter = server["datacenter"]["location"]["name"]
if server_name in DARKLISTED_SERVERS:
excluded_servers.append({
"server_id": server["id"],
"name": server_name,
"datacenter": datacenter,
"reason": "Darklisted server"
})
continue # Skip adding to Prometheus targets
if ipv4:
# Add to DNS mapping file
targets.append({
"targets": [f"{ipv4}:9100", f"{ipv4}:9113"],
"labels": {
"instance": server_name,
"datacenter": datacenter
}
})
# This is with Python Flask server for health checks
phx_health_exporter_targets.append({
"targets": [f"{ipv4}:9800"],
"labels": {
"instance": f"{server_name}-health-exporter",
"datacenter": datacenter
}
})
# Add Pushgateway heartbeat endpoint (default Pushgateway port is 9091)
heartbeat_targets.append({
"targets": [f"{ipv4}:9091"], # assuming Pushgateway is running on each server
"labels": {
"instance": server_name,
"job": "heartbeat"
}
})
# Phoenix System metrics (port 3000)
# phx_system_targets.append({
# "targets": [f"{ipv4}:3000"],
# "labels": {
# "instance": f"{server_name}",
# "datacenter": datacenter,
# "__metrics_path__": "/health/metrics"
# }
# })
# # Phoenix Worker metrics (port 3001)
# phx_worker_targets.append({
# "targets": [f"{ipv4}:3001"],
# "labels": {
# "instance": f"{server_name}",
# "datacenter": datacenter,
# "__metrics_path__": "/health/metrics"
# }
# })
dns_mappings.append({
"dns_name": server_name,
"ip_address": ipv4
})
else:
# Log the server that couldn't be added
error_servers.append({
"server_id": server["id"],
"name": server_name,
"status": server["status"],
"datacenter": datacenter,
"reason": "Missing public_net or IPv4"
})
# Save JSON to Prometheus target file
with open(PROMETHEUS_TARGETS_FILE, "w") as f:
json.dump(targets, f, indent=4)
print(f"✅ Updated Prometheus targets in {PROMETHEUS_TARGETS_FILE}")
with open(PHX_HEALTH_EXPORTER_FILE, "w") as f:
json.dump(phx_health_exporter_targets, f, indent=4)
print(f"✅ phoenix-health-exporter targets saved to {PHX_HEALTH_EXPORTER_FILE}")
with open(HEARTBEAT_TARGETS_FILE, "w") as f:
json.dump(heartbeat_targets, f, indent=4)
print(f"✅ heartbeat targets saved to {HEARTBEAT_TARGETS_FILE}")
# with open(PHX_SYSTEM_FILE, "w") as f:
# json.dump(phx_system_targets, f, indent=4)
# print(f"✅ phoenix-system targets saved to {PHX_SYSTEM_FILE}")
# with open(PHX_WORKER_FILE, "w") as f:
# json.dump(phx_worker_targets, f, indent=4)
# print(f"✅ phoenix-worker targets saved to {PHX_WORKER_FILE}")
# Save DNS Mappings file
with open(DNS_MAPPING_FILE, "w") as f:
json.dump(dns_mappings, f, indent=4)
print(f"📡 Created DNS Mapping file: {DNS_MAPPING_FILE}")
# Save error logs if any servers were skipped due to missing data
if error_servers:
with open(ERROR_LOG_FILE, "w") as f:
json.dump(error_servers, f, indent=4)
print(f"⚠️ Some servers could not be added. Check {ERROR_LOG_FILE} for details.")
# Save excluded servers log
if excluded_servers:
with open(EXCLUDED_SERVERS_FILE, "w") as f:
json.dump(excluded_servers, f, indent=4)
print(f"🚫 Darklisted servers were skipped. See {EXCLUDED_SERVERS_FILE} for details.")
else:
print("✅ No servers were excluded due to the darklist.")
# 🔄 **Trigger Prometheus Reload**
try:
response = requests.post(PROMETHEUS_RELOAD_URL)
if response.status_code == 200:
print("🔄 ✅ Prometheus configuration reloaded successfully!")
else:
print(f"⚠️ Warning: Prometheus reload failed with status {response.status_code}: {response.text}")
except requests.exceptions.RequestException as e:
print(f"❌ Error reloading Prometheus: {e}")
# 🔄 Run the script
if __name__ == "__main__":
generate_prometheus_sd_config()

View File

@@ -1,31 +1,32 @@
-----BEGIN CERTIFICATE-----
MIIE9DCCA9ygAwIBAgISBDTvkeGcJN7cgStYhZRIi215MA0GCSqGSIb3DQEBCwUA
MIIFIzCCBAugAwIBAgISBVolsu7Gt+CzF7nMdCREzqXYMA0GCSqGSIb3DQEBCwUA
MDMxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MQwwCgYDVQQD
EwNSMTEwHhcNMjQwNzAyMDUxMjAzWhcNMjQwOTMwMDUxMjAyWjAdMRswGQYDVQQD
EwNSMTEwHhcNMjUwNDI3MTAyMjI3WhcNMjUwNzI2MTAyMjI2WjAdMRswGQYDVQQD
ExJncmFmYW5hLnBoeC1lcnAuZGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
AoIBAQDBdjepETkdm9xE+6NG4FvXAhEE16t0aBGkIBgs/DUUyMcx8f8b4TMDqEtl
AMXtkRAC4pM/eeAMNq3yfeuLX9vG+E6Jra9vrcF6ThikARlnrEmNHD5qq7FfGiWM
vAsRrxeOW22TFQu7AlUK3mxQodDQkvKRhwHflo/f6dL6Z7jPTgVteZb5U0cEWxba
4kW/voO5aI/vm4kABoUdyzSy1cwza5fO0OQRLnAnV9wdtbx/47BLuacHSK4L8n30
KLhqrnnftT9XQiT18L9L4N8laBPuIGD+9Xix74izRs6kJCc/fm21hFTA+NuT5URM
UizyBp17yeX/vXH7d1v9TYL59UjdAgMBAAGjggIWMIICEjAOBgNVHQ8BAf8EBAMC
UizyBp17yeX/vXH7d1v9TYL59UjdAgMBAAGjggJFMIICQTAOBgNVHQ8BAf8EBAMC
BaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAw
HQYDVR0OBBYEFLuJi7SLpiYjNoDoBnLeTIb1Hh50MB8GA1UdIwQYMBaAFMXPRqTq
9MPAemyVxC2wXpIvJuO5MFcGCCsGAQUFBwEBBEswSTAiBggrBgEFBQcwAYYWaHR0
cDovL3IxMS5vLmxlbmNyLm9yZzAjBggrBgEFBQcwAoYXaHR0cDovL3IxMS5pLmxl
bmNyLm9yZy8wHQYDVR0RBBYwFIISZ3JhZmFuYS5waHgtZXJwLmRlMBMGA1UdIAQM
MAowCAYGZ4EMAQIBMIIBBAYKKwYBBAHWeQIEAgSB9QSB8gDwAHYASLDja9qmRzQP
5WoC+p0w6xxSActW3SyB2bu/qznYhHMAAAGQchJbcAAABAMARzBFAiEAu0E/v48I
rq3HBHWKTh6o80DulrsGxHWCosqePtHOx+YCIFUg7+MOTzGSq/DiAz6ISLgtalhw
1bhvPtHN+/qB5zL9AHYAPxdLT9ciR1iUHWUchL4NEu2QN38fhWrrwb8ohez4ZG4A
AAGQchJbZQAABAMARzBFAiASWbFzTz35eEulTzgKvehuAhbbAuIhXNuSKUDMtFom
DAIhAKMQtJgBjln1nKe8Vc5r8vWor0c5IfBXhHxzoCvok8anMA0GCSqGSIb3DQEB
CwUAA4IBAQAmtvQyWxnq8gh17CRYC9UenmC0mpoLo5UiGcmV5SR4mIVjnQRdp1fr
mJFpcGVTpVDAd22aUIS1QcOejEICC5r0ykrB7Gj66D0nYXrm4CfX71XhHViba7kv
ouB6j2C0cNIYd3NmrAZWWPH8osbR1AFpJWwWSX8KojKTZ3K8OY7/SkM+uZ2enVHE
HRD0s2R0rHUvTYkmYX6yMAKD9uceN7MzdkzLdbIh+zKiUBVYK37L23jSGw75PI8U
zN8/bE91XKb4/hH/YsCMO/1u68D2p6vmgmxZRj8H8hEBBhFpQaC2hCYRoWsbuNVu
1POM3rYNrHkNLzoZBvOFL8WgJ6MtfRVy
MAowCAYGZ4EMAQIBMC4GA1UdHwQnMCUwI6AhoB+GHWh0dHA6Ly9yMTEuYy5sZW5j
ci5vcmcvODkuY3JsMIIBAwYKKwYBBAHWeQIEAgSB9ASB8QDvAHYAzPsPaoVxCWX+
lZtTzumyfCLphVwNl422qX5UwP5MDbAAAAGWdvqaYwAABAMARzBFAiEA0e6Cw5IB
RmotsjC9qcc1X9Ab0NODHRs5JOLEp7CB/J8CIG4b6L+EH2qhYed9TSjln8Nt1Tym
Yq4PI29PmNnbaCnzAHUAfVkeEuF4KnscYWd8Xv340IdcFKBOlZ65Ay/ZDowuebgA
AAGWdvqaUAAABAMARjBEAiBcksSA/FONdr4vvl6Y6b2AaAh+wHzByinHdn6Jc09q
AgIgJfM1XdTgmY/QJb29OGpH8kgLkTZn+MnwHEoHYI2nobMwDQYJKoZIhvcNAQEL
BQADggEBALfI55bDe7jZLZiD+VdpPDhGyx6Rf7rAkHZ4xtZL4v6lhd2pWZeyV3bJ
DNj7IQ/iVIxe388G9MhfmyYjn4fPmDvczGrJzoA2z45bCAS7x/chYjeej4MPq/Ud
VwCrU2OUFH9dMB0q+3qHr6aGnKRuaG4DS0qIkTEa4ou2KfSbaDOR9uqgFj3V3wVp
0LHLiCPLe+KjUdT7WUmwfEAF/5V20SUjzTEExkfI0Yhmtt6EYxIUhZ8zeH1+/qox
An7U48oDvSi/QsdGaeuKpTR09tjol/txASJhIhSJqfL+Q9jKG6s+Yc7mTeQe+PZl
JmDDdtiKADcJ9EtIhG6rqt6NBL3NOSA=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----

View File

@@ -1,31 +1,32 @@
-----BEGIN CERTIFICATE-----
MIIE+DCCA+CgAwIBAgISBH95HJK8b1S9cOWH1ErrGG7vMA0GCSqGSIb3DQEBCwUA
MIIFKzCCBBOgAwIBAgISBkq6MpaUzRQ75D6kFDRfobJtMA0GCSqGSIb3DQEBCwUA
MDMxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MQwwCgYDVQQD
EwNSMTAwHhcNMjQwNzAyMDUxMjE1WhcNMjQwOTMwMDUxMjE0WjAgMR4wHAYDVQQD
EwNSMTAwHhcNMjUwNDI3MTAyMjM4WhcNMjUwNzI2MTAyMjM3WjAgMR4wHAYDVQQD
ExVwcm9tZXRoZXVzLnBoeC1lcnAuZGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw
ggEKAoIBAQCx+/3s+NJEchgLo3GfBIzSBN/q52J46liUx6qISjbswhr7Fxg166rC
K7ybmqs6y+1bsHzYjCQ0bBCp4Gjhrs/kfVSZ/CpjpfNV3XpanJt9AD00qFCbLX9u
X69PRre/URdOfwtxW0ZZxev4/H9us1epSk54vq25ntR9Nl1IFMZ+b7Bk6CRa7Q7i
DkpM/6Cv2A+QhbUozOcQjv1vnpnSxST/QbW03dvvy6OR9OlG5bzrAADW0Mo2BREO
8GfOY8YIX2EZvEYHDda4KztLvMhAN79iKmlzlP+rsKEddvN64ivvKhW9MUgf7NIF
w9dHIJs16lomoxp87kQ9Eav+K8Gw0PNHAgMBAAGjggIXMIICEzAOBgNVHQ8BAf8E
w9dHIJs16lomoxp87kQ9Eav+K8Gw0PNHAgMBAAGjggJKMIICRjAOBgNVHQ8BAf8E
BAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQC
MAAwHQYDVR0OBBYEFCDXhBghaDDRLeSvoxXD7qp/++hrMB8GA1UdIwQYMBaAFLu8
w0el5LypxsOkcgwQjaI14cjoMFcGCCsGAQUFBwEBBEswSTAiBggrBgEFBQcwAYYW
aHR0cDovL3IxMC5vLmxlbmNyLm9yZzAjBggrBgEFBQcwAoYXaHR0cDovL3IxMC5p
LmxlbmNyLm9yZy8wIAYDVR0RBBkwF4IVcHJvbWV0aGV1cy5waHgtZXJwLmRlMBMG
A1UdIAQMMAowCAYGZ4EMAQIBMIIBAgYKKwYBBAHWeQIEAgSB8wSB8ADuAHUASLDj
a9qmRzQP5WoC+p0w6xxSActW3SyB2bu/qznYhHMAAAGQchKHvAAABAMARjBEAiAL
mEILf1FPzNkLwSWQ5rxxu7kqojeclLB19m5x3PHgTgIgaCIfNOPqHgiiD7d1yTHo
cL46GqtXSg5rEOS8vswpZLYAdQDuzdBk1dsazsVct520zROiModGfLzs3sNRSFlG
cR+1mwAAAZByEofDAAAEAwBGMEQCIBjBmMC0yaiWf0P8NjLRTepuEcN9ydKyu6nW
pjNIragwAiBP8WFdGHVotFkaoK1qD5oQj75pOATfOOOlZ5ZRyanS9DANBgkqhkiG
9w0BAQsFAAOCAQEAdHTTmispo3jCdWGByTaTD7w+W9HMdg6v61c5ZTNgn4Kb2cxS
PqZWE6b/KSxxx/VZb+4VS7aF89kHpNv2jFZmIlub7huqox9ey7SPSnYY4jMtAVdu
xtWslYuUH4Nv0Knzszja/yKxFk3XMZ0Ah/MdqT+JAClLt0K6qiWZf0tyOCXyUZxh
BHYVrc9V+xZkcQ1dd+UMAFrvfZSyRXy8wYJzz9Kuvk1S4wsQtWp+1YkIoFfiSIJN
GisX3WMjdx5EsAJjjjqT8ZlJ3TdwdHmKPkudSc4VmrYjt6pNRda+rztKxldX2bsU
J/qMB51HMuz9I+wfrj8ZNtXbSuSqdMaFGIcUDg==
A1UdIAQMMAowCAYGZ4EMAQIBMC8GA1UdHwQoMCYwJKAioCCGHmh0dHA6Ly9yMTAu
Yy5sZW5jci5vcmcvMTEwLmNybDCCAQQGCisGAQQB1nkCBAIEgfUEgfIA8AB2AA3h
8jAr0w3BQGISCepVLvxHdHyx1+kw7w5CHrR+Tqo0AAABlnb6yEEAAAQDAEcwRQIh
ALIQgHy9Dm8jEK8pmDdXnTXaW3Hgknamxu4HPvqUZJfFAiBHcNJcEWs3vFmC5RTp
9JEltl7ma7P32N84zgZgWXbGxAB2AMz7D2qFcQll/pWbU87psnwi6YVcDZeNtql+
VMD+TA2wAAABlnb6yFEAAAQDAEcwRQIgLyj1f+tAq4iywH+i/kZV0LjdKXe+S+CU
h/xnuJjdzdYCIQC7FxwVOxzatbYDlHmeRkX4fV0e2HenYUiDgG+/huR1SDANBgkq
hkiG9w0BAQsFAAOCAQEAOqq1KFmNo/lF0bAtFx1YBlGZk8cGQHumUhMlVRR9VKZ+
S43GoJQOg+AD3yCjXA4YNUMpy0n+E8ggWlA2cPwC378crk8Zu/wh2ILcyPDOaik/
vAJ50m+dyZ67Ons9uaBuy7tgiEHmvscRnQI+PjybXrcmdDEMoYu6JW4TEqsVYqi1
Arfr1XbSdC8HDmjNNjJGAxrqsKH25rH1BEXeqwdeDXwXjhL3V66k78MLzRhxH6bf
ngwsfG8DotosCLn4bNICg1x+xrzWMKJUZ7QUe3sf5PMZILPYUYaIdus04hVBN8Hq
CwRvCi7vIJ8PSGvZ7qaYlhr/k/LmPVlz5ik/LF+p2Q==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----

View File

@@ -1 +1,2 @@
2024/07/23 03:45:01 [notice] 499#499: signal process started
2025/05/13 03:45:01 [notice] 902#902: signal process started
2025/05/13 08:24:10 [notice] 268#268: signal process started