From 191b044ff3a3704af1d114d0f42d0e022672b31b Mon Sep 17 00:00:00 2001 From: Yuri-Lima Date: Mon, 28 Jul 2025 18:37:53 +0200 Subject: [PATCH] first commit --- .gitignore | 3 + Dockerfile.semaphore | 14 +++ docker-compose.yml | 127 ++++++++++++++++++++++++++++ docker_cleanup.sh | 68 +++++++++++++++ resource_monitor.sh | 58 +++++++++++++ semaphore-config/config/config.json | 73 ++++++++++++++++ system_cleanup.sh | 103 ++++++++++++++++++++++ 7 files changed, 446 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile.semaphore create mode 100644 docker-compose.yml create mode 100755 docker_cleanup.sh create mode 100755 resource_monitor.sh create mode 100644 semaphore-config/config/config.json create mode 100755 system_cleanup.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d9364f5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +./ansible-dev +/https_portal +/log diff --git a/Dockerfile.semaphore b/Dockerfile.semaphore new file mode 100644 index 0000000..0494558 --- /dev/null +++ b/Dockerfile.semaphore @@ -0,0 +1,14 @@ +FROM semaphoreui/semaphore:v2.15.4 + +USER root +LABEL maintainer="y.m.lima19@gmail.com" + +# Update apk and install Python and necessary Python packages +# RUN apk update && \ +# apk add --no-cache python3 py3-pip \ +# py3-requests py3-python-dateutil + +# # Clean up APK cache +# RUN rm -rf /var/cache/apk/* + +RUN echo "Python3, requests, and python-dateutil installed via apk" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..76f74fc --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,127 @@ +# https://github.com/ChristianLempa/videos/blob/main/ansiblesemaphore-tutorial/docker-compose.yaml +services: + postgres: + container_name: ansible-postgres + restart: unless-stopped + image: postgres:14 # postgres:15.1-alpine + hostname: postgres + volumes: + - semaphore-postgres:/var/lib/postgresql/data + environment: + POSTGRES_USER: semaphore + POSTGRES_PASSWORD: semaphore + POSTGRES_DB: semaphore + networks: + - ansible + + pgadmin: + container_name: ansible-pgadmin + image: dpage/pgadmin4:7.6 + restart: unless-stopped + environment: + PGADMIN_DEFAULT_EMAIL: y.m.lima19@gmail.com + PGADMIN_DEFAULT_PASSWORD: semaphore + PGADMIN_CONFIG_SERVER_MODE: 'False' + depends_on: + - postgres + networks: + - ansible + volumes: + - pgadmin-data:/var/lib/pgadmin + + semaphore: + container_name: ansible-semaphore + build: + context: . # Directory containing the Dockerfile + dockerfile: Dockerfile.semaphore # Name of the Dockerfile + restart: always + # user: "${UID}:${GID}" + # Use root is not the best approach, we need soon create a user and group to run this container. + user: "root" # change to your user id and group id, this is to avoid permission issues + # image: semaphoreui/semaphore:v2.9.75 # v2.9.28 + environment: + TZ: Europe/Berlin + SEMAPHORE_SCHEDULE_TIMEZONE: Europe/Berlin + SEMAPHORE_DB_USER: semaphore + SEMAPHORE_DB_PASS: semaphore + SEMAPHORE_DB_HOST: postgres + SEMAPHORE_DB_PORT: 5432 + SEMAPHORE_DB_DIALECT: postgres + SEMAPHORE_DB: semaphore + SEMAPHORE_PLAYBOOK_PATH: /tmp/semaphore/ + SEMAPHORE_ADMIN_PASSWORD: 12345678 + SEMAPHORE_ADMIN_NAME: admin + SEMAPHORE_ADMIN_EMAIL: y.m.lima19@gmail.com + SEMAPHORE_ADMIN: admin #OLD from access_key_encryption in config file => eMsZ2Zk3C/1DORYCLRwS/pO/NKsthZhuckTRRxIn+PM= + SEMAPHORE_ACCESS_KEY_ENCRYPTION: BvmQlUSZ9BoUK7x2U7LoyKh7uyxLNeaPwvryzhjs2pY= # DONT CHANGE UNLESS YOU KNOW WHAT YOU ARE DOING!!!!!!!!!! + ANSIBLE_HOST_KEY_CHECKING: false # (optional) change to true if you want to enable host key checking, fingerprint will be saved in /etc/semaphore/known_hosts + # ANSIBLE_USER: ansible + #ssh_args -o + # ANSIBLE_SSH_ARGS: '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' + # ForwardAgent + # ANSIBLE_SSH_ARGS: '-o ForwardAgent=yes' + # scp_if_ssh + # ANSIBLE_SCP_IF_SSH: 'True' + # SEMAPHORE_LDAP_ACTIVATED: 'no' # if you wish to use ldap, set to: 'yes' + # SEMAPHORE_LDAP_HOST: dc01.local.example.com + # SEMAPHORE_LDAP_PORT: '636' + # SEMAPHORE_LDAP_NEEDTLS: 'yes' + # SEMAPHORE_LDAP_DN_BIND: 'uid=bind_user,cn=users,cn=accounts,dc=local,dc=shiftsystems,dc=net' + # SEMAPHORE_LDAP_PASSWORD: 'ldap_bind_account_password' + # SEMAPHORE_LDAP_DN_SEARCH: 'dc=local,dc=example,dc=com' + # SEMAPHORE_LDAP_SEARCH_FILTER: "(&(uid=%s)(memberOf=cn=ipausers,cn=groups,cn=accounts,dc=local,dc=example,dc=com))" + volumes: + - ./semaphore-config/config/:/etc/semaphore:rw # This is working fine!!! + depends_on: + - postgres + networks: + - ansible + node_exporter: + image: quay.io/prometheus/node-exporter:latest + container_name: node_exporter + network_mode: host + pid: host + restart: unless-stopped + command: + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--path.rootfs=/host" + - "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev)($$|/)" + volumes: + - "/proc:/host/proc:ro" + - "/sys:/host/sys:ro" + - "/:/host:ro,rslave" + https_portal: + container_name: https_portal + image: "steveltn/https-portal:1.21" + restart: unless-stopped + user: "root" + networks: + - ansible # internal network + ports: + - "80:80" + - "443:443" + environment: + STAGE: "production" # Use Let's Encrypt production server + WEBSOCKET: "true" # Enable websocket support + DEBUG: "true" + RENEW_MARGIN_DAYS: 30 + CLIENT_MAX_BODY_SIZE: 0 + # FORCE_RENEW: 'true' + DOMAINS: > + ansible.phx-erp.de -> semaphore:3000, pgadmin-ansible.phx-erp.de -> pgadmin:80 + volumes: + - ./https_portal/data:/var/lib/https-portal # ssl_certs, vhost.d, htdocs + - ./https_portal/log:/var/log/nginx # nginx logs + depends_on: + - semaphore + - pgadmin + +volumes: + semaphore-postgres: + pgadmin-data: + +networks: + ansible: + name: ansible + driver: bridge diff --git a/docker_cleanup.sh b/docker_cleanup.sh new file mode 100755 index 0000000..5ee0b73 --- /dev/null +++ b/docker_cleanup.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Log file path +LOG_FILE="/opt/phx/log/docker_cleanup.log" + +# Function to log informational messages with timestamp +log_info() { + echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] $1" | tee -a "$LOG_FILE" +} + +# Function to log error messages with timestamp +log_error() { + echo "$(date '+%Y-%m-%d %H:%M:%S') [ERROR] $1" | tee -a "$LOG_FILE" >&2 +} + +# Start Docker Cleanup +log_info "========== Docker Cleanup Started ==========" + +# Remove stopped containers +log_info "Removing all stopped Docker containers..." +if docker container prune -f >> "$LOG_FILE" 2>&1; then + log_info "Stopped Docker containers removed successfully." +else + log_error "Failed to remove stopped Docker containers." +fi + +# Remove unused images +log_info "Removing all unused Docker images..." +if docker image prune -af >> "$LOG_FILE" 2>&1; then + log_info "Unused Docker images removed successfully." +else + log_error "Failed to remove unused Docker images." +fi + +# Remove unused volumes +log_info "Removing all unused Docker volumes..." +if docker volume prune -f >> "$LOG_FILE" 2>&1; then + log_info "Unused Docker volumes removed successfully." +else + log_error "Failed to remove unused Docker volumes." +fi + +# Remove unused networks +log_info "Removing all unused Docker networks..." +if docker network prune -f >> "$LOG_FILE" 2>&1; then + log_info "Unused Docker networks removed successfully." +else + log_error "Failed to remove unused Docker networks." +fi + +# Optional: Remove dangling build cache +log_info "Removing dangling Docker build cache..." +if docker builder prune -f >> "$LOG_FILE" 2>&1; then + log_info "Dangling Docker build cache removed successfully." +else + log_error "Failed to remove dangling Docker build cache." +fi + +# Optional: Restart Docker Service +log_info "Restarting Docker service to apply changes..." +if systemctl restart docker; then + log_info "Docker service restarted successfully." +else + log_error "Failed to restart Docker service." +fi + +# Completion log +log_info "========== Docker Cleanup Completed ==========" \ No newline at end of file diff --git a/resource_monitor.sh b/resource_monitor.sh new file mode 100755 index 0000000..69ca0cc --- /dev/null +++ b/resource_monitor.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# Log file path +LOG_FILE="/opt/phx/log/resource_monitor.log" + +# Thresholds +CPU_THRESHOLD=85 +MEM_THRESHOLD=90 +DISK_THRESHOLD=90 + +# Get the hostname +HOSTNAME=$(hostname) + +# Function to log messages with timestamp +log_info() { + echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] $1" | tee -a "$LOG_FILE" +} + +log_warning() { + echo "$(date '+%Y-%m-%d %H:%M:%S') [WARNING] $1" | tee -a "$LOG_FILE" +} + +log_error() { + echo "$(date '+%Y-%m-%d %H:%M:%S') [ERROR] $1" | tee -a "$LOG_FILE" >&2 +} + +# Start monitoring +log_info "========== Resource Monitoring Started ==========" + +# Check CPU usage (Fixed) +CPU_USAGE=$(top -bn1 | grep "Cpu(s)" | awk '{print 100 - $8}') +CPU_USAGE=${CPU_USAGE%.*} + +if (( CPU_USAGE > CPU_THRESHOLD )); then + log_warning "⚠️ CPU usage is high: ${CPU_USAGE}% (Threshold: ${CPU_THRESHOLD}%) on $HOSTNAME." +else + log_info "✅ CPU usage is normal: ${CPU_USAGE}%." +fi + +# Check Memory usage +MEM_USAGE=$(free | awk '/Mem/ {printf "%.0f", ($3/$2)*100}') + +if (( MEM_USAGE > MEM_THRESHOLD )); then + log_warning "⚠️ Memory usage is high: ${MEM_USAGE}% (Threshold: ${MEM_THRESHOLD}%) on $HOSTNAME." +else + log_info "✅ Memory usage is normal: ${MEM_USAGE}%." +fi + +# Check Disk usage +DISK_USAGE=$(df / | awk 'END {print $5}' | sed 's/%//') + +if (( DISK_USAGE > DISK_THRESHOLD )); then + log_warning "⚠️ Disk usage is high: ${DISK_USAGE}% (Threshold: ${DISK_THRESHOLD}%) on $HOSTNAME." +else + log_info "✅ Disk usage is normal: ${DISK_USAGE}%." +fi + +log_info "========== Resource Monitoring Completed ==========" \ No newline at end of file diff --git a/semaphore-config/config/config.json b/semaphore-config/config/config.json new file mode 100644 index 0000000..61acb75 --- /dev/null +++ b/semaphore-config/config/config.json @@ -0,0 +1,73 @@ +{ + "mysql": { + "host": "", + "user": "", + "pass": "", + "name": "", + "options": null + }, + "bolt": { + "host": "", + "user": "", + "pass": "", + "name": "", + "options": null + }, + "postgres": { + "host": "postgres:5432", + "user": "semaphore", + "pass": "semaphore", + "name": "semaphore", + "options": { + "sslmode": "disable" + } + }, + "dialect": "postgres", + "port": "", + "interface": "", + "tmp_path": "/tmp/semaphore", + "ssh_config_path": "", + "git_client": "", + "web_host": "", + "cookie_hash": "I8IwsODrcyEL0472ycEiZCDu4F5VF0MA/J7DVe6/V9s=", + "cookie_encryption": "2tv+2qHbRY5v0XjNr1vp1dSR/xD0BJLngVxnxpeXLvk=", + "access_key_encryption": "BvmQlUSZ9BoUK7x2U7LoyKh7uyxLNeaPwvryzhjs2pY=", + "email_alert": false, + "email_sender": "info@phx-erp.de", + "email_host": "mail.phx-erp.de", + "email_port": "465", + "email_username": "yuri.lima@phx-erp.de", + "email_password": "0rB0@et68", + "email_secure": true, + "ldap_enable": false, + "ldap_binddn": "", + "ldap_bindpassword": "", + "ldap_server": "", + "ldap_searchdn": "", + "ldap_searchfilter": "", + "ldap_mappings": { + "dn": "", + "mail": "", + "uid": "", + "cn": "" + }, + "ldap_needtls": false, + "telegram_alert": false, + "telegram_chat": "", + "telegram_token": "", + "slack_alert": true, + "slack_url": "https://hooks.slack.com/services/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", + "oidc_providers": null, + "max_parallel_tasks": 1, + "runner_registration_token": "", + "password_login_disable": false, + "non_admin_can_create_project": true, + "runner": { + "api_url": "", + "registration_token": "", + "config_file": "" + }, + "schedule": { + "timezone": "Europe/Berlin" + } +} \ No newline at end of file diff --git a/system_cleanup.sh b/system_cleanup.sh new file mode 100755 index 0000000..eb8402a --- /dev/null +++ b/system_cleanup.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +# Log file path +LOG_FILE="/opt/phx/log/system_cleanup.log" +EMAIL="y.m.lima19@gmail.com" + +# Function to send email notifications +send_email() { + SUBJECT="$1" + MESSAGE="$2" + echo "$MESSAGE" | mail -s "$SUBJECT" "$EMAIL" +} + +# Function to log informational messages +log_info() { + echo "$(date '+%Y-%m-%d %H:%M:%S') [INFO] $1" | tee -a "$LOG_FILE" +} + +# Function to log errors and send email notifications +log_error() { + echo "$(date '+%Y-%m-%d %H:%M:%S') [ERROR] $1" | tee -a "$LOG_FILE" >&2 + send_email "⚠️ System Cleanup Error" "$1" +} + +# Start Cleanup +log_info "========== System Cleanup Started ==========" + +# Clear system cache +log_info "Clearing system cache..." +if sync && echo 3 > /proc/sys/vm/drop_caches; then + log_info "System cache cleared successfully." +else + log_error "Failed to clear system cache." +fi + +# Clean up packages +log_info "Cleaning up unused packages..." +if apt-get autoremove -y >> "$LOG_FILE" 2>&1; then + log_info "Unused packages removed." +else + log_error "Failed to remove unused packages." +fi + +log_info "Running apt autoclean..." +if apt-get autoclean -y >> "$LOG_FILE" 2>&1; then + log_info "Autoclean completed." +else + log_error "Autoclean failed." +fi + +log_info "Running apt clean..." +if apt-get clean >> "$LOG_FILE" 2>&1; then + log_info "Clean completed." +else + log_error "Clean failed." +fi + +# Clean up system logs +log_info "Cleaning system logs older than 7 days..." +if journalctl --vacuum-time=7d >> "$LOG_FILE" 2>&1; then + log_info "Old logs cleaned." +else + log_error "Failed to clean old logs." +fi + +# Restart services +log_info "Restarting Docker service..." +if systemctl restart docker; then + log_info "Docker restarted successfully." +else + log_error "Failed to restart Docker." +fi + +# Kill zombie processes +log_info "Checking for zombie processes..." +ZOMBIES=$(ps aux | grep 'defunct' | grep -v grep) +if [[ -n "$ZOMBIES" ]]; then + log_info "Found zombie processes. Attempting to kill..." + if pkill -9 -f 'defunct'; then + log_info "Zombie processes killed." + else + log_error "Failed to kill zombie processes." + fi +else + log_info "No zombie processes found." +fi + +# Swap check and creation +log_info "Checking swap space..." +if swapon --show | grep -q '/swapfile'; then + log_info "Swap space is active." +else + log_info "No swap detected. Creating 2GB swap file..." + if fallocate -l 2G /swapfile && chmod 600 /swapfile && mkswap /swapfile && swapon /swapfile; then + log_info "Swap file created and activated." + else + log_error "Failed to create swap file." + fi +fi + +# Cleanup completed +log_info "========== System Cleanup Completed ==========" +send_email "✅ System Cleanup Successful" "The system cleanup script has completed successfully." \ No newline at end of file