#!/bin/bash # ============================================================================================== # ================================= Certificate Monitor ======================================== # ============================================================================================== # Monitors SSL certificate expiry for all configured domains by connecting directly # via openssl — no dependency on NPM or any other service. Reads the actual certificate # the server is presenting to the outside world. # # ── WHY DIRECT OPENSSL ──────────────────────────────────────────────────────────────────────── # Catches real-world cert issues that API-based checks miss: # - Cert renewed in NPM but server not reloaded (old cert still serving) # - Wrong cert being served to external clients # - Cert chain issues not visible from the internal network # - NPM reporting healthy while the world sees an expired cert # # ── BEHAVIOUR ───────────────────────────────────────────────────────────────────────────────── # Each domain is checked independently — they have independent certs. # Results per domain: # HEALTHY — > CERT_WARN_DAYS remaining — silent ✅ # WARNING — <= CERT_WARN_DAYS remaining — notifies # CRITICAL — <= CERT_CRIT_DAYS remaining — notifies with urgency # FAILED — could not connect or parse cert — notifies # # Notifications batched per severity — one message per severity level, not per domain. # This is a monitor script — SILENT_MODE=false — output is the point. # # ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── # detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS → CERT_MONITOR_DOMAINS. # Each server monitors its own domains — HOST1 monitors Gmer4Lfe.com etc. # # ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # acquire_lock — prevents concurrent runs # detect_hosts() — correct domain list per host via MY_ID aliases # Empty array guard — warns and exits cleanly if no domains configured # CERT_TIMEOUT — openssl connects are time-limited per domain # validate_unraid_cmd — openssl and notify validated before use # Silent healthy certs — only problems produce visible output # # ── CONFIGURATION (master_host*.conf) ───────────────────────────────────────────────────────── # HOST*_CERT_MONITOR_DOMAINS — domains checked by this host # Aliased by detect_hosts() — script uses CERT_MONITOR_DOMAINS # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # CERT_WARN_DAYS — warn when cert expires within this many days (default 30) # CERT_CRIT_DAYS — critical alert within this many days (default 7) # CERT_TIMEOUT — seconds per domain before giving up (default 10) # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # cert_monitor.sh — normal run # cert_monitor.sh --dry-run — check certs and show results, no notifications # cert_monitor.sh --log — verbose output # cert_monitor.sh --status — show config and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" # Monitor script — output is the point SILENT_MODE=false parse_args "$@" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi # Validate openssl — required for all cert checks validate_unraid_cmd \ "$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \ "version" "OpenSSL" \ "openssl" || { error "openssl not found — required for certificate checks"; exit 1; } validate_unraid_cmd \ "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ "" "" \ "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" acquire_lock # detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS detect_hosts # Empty array guard if [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then warn "CERT_MONITOR_DOMAINS is empty for $MY_ID" warn "Check HOST*_CERT_MONITOR_DOMAINS in master_host*.conf" exit 0 fi log "Domains to check: ${#CERT_MONITOR_DOMAINS[@]}" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent" # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}" echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining" echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining" echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain" echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ── CERT CHECK FUNCTION ─────────────────────────────────────────────────────────────────────── # ============================================================================================== # Connects to domain:443 via openssl, extracts expiry date, calculates days remaining. # Returns: # 0 = healthy (> CERT_WARN_DAYS) # 1 = warning (<= CERT_WARN_DAYS) # 2 = critical (<= CERT_CRIT_DAYS) # 3 = failed (could not connect or parse) check_cert() { local domain="$1" local port="${2:-443}" local expiry_str expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \ -connect "${domain}:${port}" \ -servername "$domain" \ 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) if [[ -z "$expiry_str" ]]; then error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)" return 3 fi local expiry_epoch expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null) if [[ -z "$expiry_epoch" ]]; then error "$ICON_CERT $domain — could not parse expiry date: $expiry_str" return 3 fi local now days_remaining expiry_display now=$(date +%s) days_remaining=$(( (expiry_epoch - now) / 86400 )) expiry_display=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null) if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then error "$ICON_CERT $domain — CRITICAL: ${days_remaining} days remaining (expires $expiry_display)" return 2 elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then warn "$ICON_CERT $domain — WARNING: ${days_remaining} days remaining (expires $expiry_display)" return 1 else log "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)" return 0 fi } # ============================================================================================== # ━━━ Certificate Monitor ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_CERT Certificate Monitor — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" log "Warn threshold: ${CERT_WARN_DAYS} days" log "Crit threshold: ${CERT_CRIT_DAYS} days" echo "" START=$(date +%s) HEALTHY=() WARNING=() CRITICAL=() FAILED=() declare -A DOMAIN_STATUS for domain in "${CERT_MONITOR_DOMAINS[@]}"; do [[ -z "$domain" ]] && continue check_cert "$domain" result=$? case $result in 0) HEALTHY+=("$domain"); DOMAIN_STATUS["$domain"]="OK" ;; 1) WARNING+=("$domain"); DOMAIN_STATUS["$domain"]="WARN" ;; 2) CRITICAL+=("$domain"); DOMAIN_STATUS["$domain"]="CRIT" ;; 3) FAILED+=("$domain"); DOMAIN_STATUS["$domain"]="FAIL" ;; esac done END=$(date +%s) # ── Send notifications — batched per severity ───────────────────────────────────────────────── if [[ "$DRY_RUN" == false ]]; then [[ ${#CRITICAL[@]} -gt 0 ]] && \ notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" \ "Certificate Monitor" "warning" [[ ${#WARNING[@]} -gt 0 ]] && \ notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" \ "Certificate Monitor" "warning" [[ ${#FAILED[@]} -gt 0 ]] && \ notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" \ "Certificate Monitor" "warning" fi # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY CERTIFICATE MONITOR SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]}" [[ ${#WARNING[@]} -gt 0 ]] && warn "Warning: ${#WARNING[@]} — renewal recommended" [[ ${#CRITICAL[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#CRITICAL[@]} — ACTION REQUIRED" [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#FAILED[@]} — unreachable" echo "" # Per-domain results — only show problems, healthy ones stay in log() for domain in "${CERT_MONITOR_DOMAINS[@]}"; do [[ -z "$domain" ]] && continue case "${DOMAIN_STATUS[$domain]:-UNKN}" in OK) log " $ICON_SUCCESS $domain — healthy" ;; WARN) warn " $ICON_WARN $domain — warning" ;; CRIT) echo " $ICON_ERROR $domain — CRITICAL" ;; FAIL) echo " $ICON_ERROR $domain — unreachable" ;; esac done echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no notifications sent" elif [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]]; then echo "$ICON_ERROR Status: ACTION REQUIRED" elif [[ ${#WARNING[@]} -gt 0 ]]; then warn "Status: WARNINGS — renewal recommended" else log "$ICON_DONE Status: all ${#HEALTHY[@]} certs healthy ✅" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]] && exit 1 exit 0