👌 IMPROVE:
This commit is contained in:
12
config/blackbox_targets.json
Normal file
12
config/blackbox_targets.json
Normal file
@@ -0,0 +1,12 @@
|
||||
[
|
||||
{
|
||||
"targets": [
|
||||
"https://alpha.phx-erp.de"
|
||||
],
|
||||
"labels": {
|
||||
"job": "blackbox-dynamic",
|
||||
"instance": "Alpha",
|
||||
"probe": "http"
|
||||
}
|
||||
}
|
||||
]
|
||||
102
config/hetzner_dns_mapping.json
Normal file
102
config/hetzner_dns_mapping.json
Normal file
@@ -0,0 +1,102 @@
|
||||
[
|
||||
{
|
||||
"dns_name": "PHX-DEV-001.Alpha",
|
||||
"ip_address": "157.90.161.42"
|
||||
},
|
||||
{
|
||||
"dns_name": "ANSIBLE-MASTER",
|
||||
"ip_address": "167.235.254.4"
|
||||
},
|
||||
{
|
||||
"dns_name": "cts",
|
||||
"ip_address": "116.203.75.215"
|
||||
},
|
||||
{
|
||||
"dns_name": "Phx-Yuri",
|
||||
"ip_address": "162.55.52.253"
|
||||
},
|
||||
{
|
||||
"dns_name": "benzinger-demo",
|
||||
"ip_address": "162.55.54.75"
|
||||
},
|
||||
{
|
||||
"dns_name": "phx-internal",
|
||||
"ip_address": "5.75.183.139"
|
||||
},
|
||||
{
|
||||
"dns_name": "trachtenmode-schmid",
|
||||
"ip_address": "142.132.165.231"
|
||||
},
|
||||
{
|
||||
"dns_name": "teamcity",
|
||||
"ip_address": "23.88.107.109"
|
||||
},
|
||||
{
|
||||
"dns_name": "gitea",
|
||||
"ip_address": "49.13.146.138"
|
||||
},
|
||||
{
|
||||
"dns_name": "phx-beta-rc",
|
||||
"ip_address": "159.69.200.205"
|
||||
},
|
||||
{
|
||||
"dns_name": "lhl",
|
||||
"ip_address": "116.203.53.137"
|
||||
},
|
||||
{
|
||||
"dns_name": "Grafana-Prometheus",
|
||||
"ip_address": "5.75.153.161"
|
||||
},
|
||||
{
|
||||
"dns_name": "cooper",
|
||||
"ip_address": "159.69.44.39"
|
||||
},
|
||||
{
|
||||
"dns_name": "shipxpert",
|
||||
"ip_address": "49.13.165.13"
|
||||
},
|
||||
{
|
||||
"dns_name": "dss",
|
||||
"ip_address": "49.13.197.152"
|
||||
},
|
||||
{
|
||||
"dns_name": "sartissohn",
|
||||
"ip_address": "188.245.44.219"
|
||||
},
|
||||
{
|
||||
"dns_name": "guntli",
|
||||
"ip_address": "116.203.92.218"
|
||||
},
|
||||
{
|
||||
"dns_name": "kolb",
|
||||
"ip_address": "94.130.77.57"
|
||||
},
|
||||
{
|
||||
"dns_name": "ried",
|
||||
"ip_address": "116.203.151.20"
|
||||
},
|
||||
{
|
||||
"dns_name": "heba",
|
||||
"ip_address": "116.203.68.120"
|
||||
},
|
||||
{
|
||||
"dns_name": "eicsoft",
|
||||
"ip_address": "116.203.128.69"
|
||||
},
|
||||
{
|
||||
"dns_name": "ck-vechta",
|
||||
"ip_address": "159.69.93.252"
|
||||
},
|
||||
{
|
||||
"dns_name": "eeparts",
|
||||
"ip_address": "116.203.46.171"
|
||||
},
|
||||
{
|
||||
"dns_name": "big-break-changes",
|
||||
"ip_address": "167.235.130.242"
|
||||
},
|
||||
{
|
||||
"dns_name": "bug",
|
||||
"ip_address": "138.199.196.189"
|
||||
}
|
||||
]
|
||||
8
config/hetzner_excluded_servers.json
Normal file
8
config/hetzner_excluded_servers.json
Normal file
@@ -0,0 +1,8 @@
|
||||
[
|
||||
{
|
||||
"server_id": 52530041,
|
||||
"name": "docuvita",
|
||||
"datacenter": "nbg1",
|
||||
"reason": "Darklisted server"
|
||||
}
|
||||
]
|
||||
@@ -161,15 +161,6 @@
|
||||
"datacenter": "nbg1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"targets": [
|
||||
"128.140.15.177:9100"
|
||||
],
|
||||
"labels": {
|
||||
"instance": "docuvita",
|
||||
"datacenter": "nbg1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"targets": [
|
||||
"116.203.151.20:9100"
|
||||
@@ -223,5 +214,14 @@
|
||||
"instance": "big-break-changes",
|
||||
"datacenter": "nbg1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"targets": [
|
||||
"138.199.196.189:9100"
|
||||
],
|
||||
"labels": {
|
||||
"instance": "bug",
|
||||
"datacenter": "nbg1"
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -2,9 +2,28 @@ global:
|
||||
scrape_interval: 15s # Default for all jobs unless overridden
|
||||
|
||||
scrape_configs:
|
||||
# Existing Hetzner Dynamic Configuration
|
||||
- job_name: "hetzner-dynamic"
|
||||
scheme: http
|
||||
file_sd_configs:
|
||||
- files:
|
||||
- "/hetzner_targets.json"
|
||||
- "/opt/phx/main/config/hetzner_targets.json"
|
||||
refresh_interval: 30s
|
||||
scrape_interval: 5s # Custom interval for Hetzner servers
|
||||
|
||||
# Blackbox Exporter with Dynamic File SD
|
||||
# - job_name: 'blackbox-dynamic'
|
||||
# metrics_path: /probe
|
||||
# params:
|
||||
# module: [http_2xx] # HTTP check to see if target is up
|
||||
# file_sd_configs:
|
||||
# - files:
|
||||
# - "/opt/phx/main/config/blackbox_targets.json" # Dynamic file like hetzner_targets.json
|
||||
# refresh_interval: 30s
|
||||
# relabel_configs:
|
||||
# - source_labels: [__address__]
|
||||
# target_label: __param_target
|
||||
# - source_labels: [__param_target]
|
||||
# target_label: instance
|
||||
# - target_label: __address__
|
||||
# replacement: blackbox_exporter:9115 # Blackbox Exporter container/service
|
||||
@@ -73,6 +73,14 @@ services:
|
||||
- "/proc:/host/proc:ro"
|
||||
- "/sys:/host/sys:ro"
|
||||
- "/:/host:ro,rslave"
|
||||
# blackbox_exporter:
|
||||
# image: prom/blackbox-exporter
|
||||
# container_name: blackbox_exporter
|
||||
# ports:
|
||||
# - "9115:9115"
|
||||
# restart: unless-stopped
|
||||
# networks:
|
||||
# - default
|
||||
https_portal:
|
||||
container_name: https_portal
|
||||
image: "steveltn/https-portal:1.21"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import requests
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables from .env file
|
||||
@@ -13,9 +14,11 @@ HETZNER_API_TOKEN = os.getenv("HETZNER_API_TOKEN")
|
||||
if not HETZNER_API_TOKEN:
|
||||
raise ValueError("❌ HETZNER_API_TOKEN is missing! Make sure it's set in the .env file.")
|
||||
|
||||
# 📂 Path to the Prometheus service discovery file
|
||||
# 📂 Paths to output files
|
||||
PROMETHEUS_TARGETS_FILE = "/opt/phx/main/config/hetzner_targets.json"
|
||||
ERROR_LOG_FILE = "/opt/phx/main/config/hetzner_error_servers.json"
|
||||
EXCLUDED_SERVERS_FILE = "/opt/phx/main/config/hetzner_excluded_servers.json"
|
||||
DNS_MAPPING_FILE = "/opt/phx/main/config/hetzner_dns_mapping.json"
|
||||
|
||||
# 📌 Hetzner API URL
|
||||
HETZNER_API_URL = os.getenv("HETZNER_API_URL")
|
||||
@@ -24,31 +27,74 @@ HETZNER_API_URL = os.getenv("HETZNER_API_URL")
|
||||
if not HETZNER_API_URL:
|
||||
raise ValueError("❌ HETZNER_API_URL is missing! Make sure it's set in the .env file.")
|
||||
|
||||
# 🛑 List of server names to exclude (DARKLIST)
|
||||
DARKLISTED_SERVERS = ["docuvita"]
|
||||
|
||||
# 📡 Fetch Hetzner server list
|
||||
# 📡 Fetch Hetzner server list with pagination support and Rate Limiting handling
|
||||
def get_hetzner_servers():
|
||||
headers = {"Authorization": f"Bearer {HETZNER_API_TOKEN}"}
|
||||
response = requests.get(HETZNER_API_URL, headers=headers)
|
||||
all_servers = []
|
||||
page = 1
|
||||
per_page = 50 # Max per request
|
||||
|
||||
while True:
|
||||
response = requests.get(
|
||||
f"{HETZNER_API_URL}?page={page}&per_page={per_page}", headers=headers
|
||||
)
|
||||
|
||||
# Handle Rate Limiting (429 Too Many Requests)
|
||||
if response.status_code == 429:
|
||||
reset_time = int(response.headers.get("RateLimit-Reset", time.time() + 60))
|
||||
wait_time = reset_time - int(time.time())
|
||||
print(f"⏳ Rate limit exceeded! Waiting {wait_time} seconds until reset...")
|
||||
time.sleep(wait_time)
|
||||
continue # Retry after waiting
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"❌ Error fetching servers: {response.text}")
|
||||
return []
|
||||
break # Stop fetching if error
|
||||
|
||||
return response.json().get("servers", [])
|
||||
# Read rate limit headers
|
||||
remaining = response.headers.get("RateLimit-Remaining", "unknown")
|
||||
reset_time = int(response.headers.get("RateLimit-Reset", time.time() + 60))
|
||||
print(f"📊 API Rate Limit: {remaining} requests remaining. Next reset at {reset_time}.")
|
||||
|
||||
data = response.json()
|
||||
servers = data.get("servers", [])
|
||||
all_servers.extend(servers)
|
||||
|
||||
# 🏗️ Generate Prometheus JSON file
|
||||
# Check if there's a next page
|
||||
pagination = data.get("meta", {}).get("pagination", {})
|
||||
if not pagination.get("next_page"):
|
||||
break # Exit if no more pages
|
||||
|
||||
page = pagination["next_page"] # Move to the next page
|
||||
|
||||
return all_servers
|
||||
|
||||
# 🏗️ Generate Prometheus JSON file and DNS Mapping
|
||||
def generate_prometheus_sd_config():
|
||||
servers = get_hetzner_servers()
|
||||
targets = []
|
||||
error_servers = []
|
||||
excluded_servers = []
|
||||
dns_mappings = [] # New list for storing DNS-IP mappings
|
||||
|
||||
for server in servers:
|
||||
if "public_net" in server and "ipv4" in server["public_net"]:
|
||||
ipv4 = server["public_net"]["ipv4"]["ip"]
|
||||
ipv4 = server.get("public_net", {}).get("ipv4", {}).get("ip")
|
||||
server_name = server["name"]
|
||||
datacenter = server["datacenter"]["location"]["name"]
|
||||
|
||||
if server_name in DARKLISTED_SERVERS:
|
||||
excluded_servers.append({
|
||||
"server_id": server["id"],
|
||||
"name": server_name,
|
||||
"datacenter": datacenter,
|
||||
"reason": "Darklisted server"
|
||||
})
|
||||
continue # Skip adding to Prometheus targets
|
||||
|
||||
if ipv4:
|
||||
targets.append({
|
||||
"targets": [f"{ipv4}:9100"],
|
||||
"labels": {
|
||||
@@ -56,13 +102,18 @@ def generate_prometheus_sd_config():
|
||||
"datacenter": datacenter
|
||||
}
|
||||
})
|
||||
# Add to DNS mapping file
|
||||
dns_mappings.append({
|
||||
"dns_name": server_name,
|
||||
"ip_address": ipv4
|
||||
})
|
||||
else:
|
||||
# Log the server that couldn't be added
|
||||
error_servers.append({
|
||||
"server_id": server["id"],
|
||||
"name": server["name"],
|
||||
"name": server_name,
|
||||
"status": server["status"],
|
||||
"datacenter": server["datacenter"]["location"]["name"],
|
||||
"datacenter": datacenter,
|
||||
"reason": "Missing public_net or IPv4"
|
||||
})
|
||||
|
||||
@@ -72,14 +123,25 @@ def generate_prometheus_sd_config():
|
||||
|
||||
print(f"✅ Updated Prometheus targets in {PROMETHEUS_TARGETS_FILE}")
|
||||
|
||||
# Save error logs if any servers were skipped
|
||||
# Save DNS Mappings file
|
||||
with open(DNS_MAPPING_FILE, "w") as f:
|
||||
json.dump(dns_mappings, f, indent=4)
|
||||
|
||||
print(f"📡 Created DNS Mapping file: {DNS_MAPPING_FILE}")
|
||||
|
||||
# Save error logs if any servers were skipped due to missing data
|
||||
if error_servers:
|
||||
with open(ERROR_LOG_FILE, "w") as f:
|
||||
json.dump(error_servers, f, indent=4)
|
||||
print(f"⚠️ Some servers could not be added. Check {ERROR_LOG_FILE} for details.")
|
||||
else:
|
||||
print("✅ All servers were added successfully, no errors detected.")
|
||||
|
||||
# Save excluded servers log
|
||||
if excluded_servers:
|
||||
with open(EXCLUDED_SERVERS_FILE, "w") as f:
|
||||
json.dump(excluded_servers, f, indent=4)
|
||||
print(f"🚫 Darklisted servers were skipped. See {EXCLUDED_SERVERS_FILE} for details.")
|
||||
else:
|
||||
print("✅ No servers were excluded due to the darklist.")
|
||||
|
||||
# 🔄 Run the script
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -8,3 +8,10 @@
|
||||
2025/01/30 12:49:57 [notice] 265#265: signal process started
|
||||
2025/01/30 12:53:15 [notice] 266#266: signal process started
|
||||
2025/01/30 12:56:58 [notice] 265#265: signal process started
|
||||
2025/01/30 13:20:47 [notice] 265#265: signal process started
|
||||
2025/01/30 13:21:52 [notice] 265#265: signal process started
|
||||
2025/01/30 13:25:46 [notice] 264#264: signal process started
|
||||
2025/01/30 13:35:24 [notice] 265#265: signal process started
|
||||
2025/01/30 13:41:55 [notice] 265#265: signal process started
|
||||
2025/01/30 14:35:34 [notice] 265#265: signal process started
|
||||
2025/01/30 14:45:52 [notice] 265#265: signal process started
|
||||
|
||||
Reference in New Issue
Block a user