#!/bin/bash # ============================================================================================== # ============================= Certificate Monitor ============================================ # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # SSL certificate expiry monitoring for all configured domains. Scheduled weekly # (Sunday 9am). Connects via openssl directly to each domain — not to NPM's API, # not to any internal check, but to the actual TLS handshake the outside world sees. # # Per domain: HEALTHY (> CERT_WARN_DAYS remaining, silent) | WARNING (≤ CERT_WARN_DAYS) # | CRITICAL (≤ CERT_CRIT_DAYS) | FAILED (could not connect or parse cert). # Notifications batched by severity — one message lists all WARNING domains, a # separate message lists all CRITICAL domains. Not one notification per domain. # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Direct openssl, Not an API # API-based cert checks ask the certificate manager whether the cert is valid. # openssl checks ask the server what cert it is actually serving. These are not # the same question and the answers can differ. Catches: cert renewed in NPM but # server not reloaded (old cert still serving), wrong cert being served to external # clients, chain issues visible externally but not internally, NPM reporting healthy # while the outside world sees an expired cert. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Single Instance Lock # acquire_lock prevents concurrent runs producing duplicate notifications. # # Per-Host Domain List # detect_hosts() aliases HOST*_CERT_MONITOR_DOMAINS → CERT_MONITOR_DOMAINS. # Each server monitors its own domains only. # # Empty Array Guard # Warns and exits cleanly if CERT_MONITOR_DOMAINS is empty — no silent no-op. # # Connection Timeout # CERT_TIMEOUT caps each openssl connection attempt. One unreachable domain # does not block the remaining domains. # # Notification Validated # validate_unraid_cmd confirms openssl and notify script are present before use. # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # master_host*.conf # # HOST*_CERT_MONITOR_DOMAINS # Domains this host monitors. Each domain and subdomain is a separate entry — # they have independent certs. Aliased by detect_hosts() → CERT_MONITOR_DOMAINS. # # master.conf # # CERT_WARN_DAYS # Days before expiry at which to send a warning notification. (default: 30) # # CERT_CRIT_DAYS # Days before expiry at which to send a critical notification. (default: 7) # # CERT_TIMEOUT # Seconds to wait per domain before declaring FAILED. (default: 10) # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # cert_monitor.sh # Check all configured domains and notify on WARNING, CRITICAL, or FAILED. # Silent when all domains are healthy. # # cert_monitor.sh --dry-run # Check all domains and show results. No notifications sent regardless of result. # # cert_monitor.sh --status # Show domain list, warning thresholds, and timeout. Then exit. # # cert_monitor.sh --log # Verbose per-domain output during the run. # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" # Monitor script — output is the point SILENT_MODE=false parse_args "$@" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi # Validate openssl — required for all cert checks validate_unraid_cmd \ "$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \ "version" "OpenSSL" \ "openssl" || { error "openssl not found — required for certificate checks"; exit 1; } validate_unraid_cmd \ "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ "" "" \ "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" acquire_lock # detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS detect_hosts # Empty array guard if [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then warn "CERT_MONITOR_DOMAINS is empty for $MY_ID" warn "Check HOST*_CERT_MONITOR_DOMAINS in master_host*.conf" exit 0 fi log "Domains to check: ${#CERT_MONITOR_DOMAINS[@]}" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent" # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}" echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining" echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining" echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain" echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ── CERT CHECK FUNCTION ─────────────────────────────────────────────────────────────────────── # ============================================================================================== # Connects to domain:443 via openssl, extracts expiry date, calculates days remaining. # Returns: # 0 = healthy (> CERT_WARN_DAYS) # 1 = warning (<= CERT_WARN_DAYS) # 2 = critical (<= CERT_CRIT_DAYS) # 3 = failed (could not connect or parse) check_cert() { local domain="$1" local port="${2:-443}" local expiry_str expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \ -connect "${domain}:${port}" \ -servername "$domain" \ 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) if [[ -z "$expiry_str" ]]; then error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)" return 3 fi local expiry_epoch expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null) if [[ -z "$expiry_epoch" ]]; then error "$ICON_CERT $domain — could not parse expiry date: $expiry_str" return 3 fi local now days_remaining expiry_display now=$(date +%s) days_remaining=$(( (expiry_epoch - now) / 86400 )) expiry_display=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null) if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then error "$ICON_CERT $domain — CRITICAL: ${days_remaining} days remaining (expires $expiry_display)" return 2 elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then warn "$ICON_CERT $domain — WARNING: ${days_remaining} days remaining (expires $expiry_display)" return 1 else log "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)" return 0 fi } # ============================================================================================== # ━━━ Certificate Monitor ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_CERT Certificate Monitor — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" log "Warn threshold: ${CERT_WARN_DAYS} days" log "Crit threshold: ${CERT_CRIT_DAYS} days" echo "" START=$(date +%s) HEALTHY=() WARNING=() CRITICAL=() FAILED=() declare -A DOMAIN_STATUS for domain in "${CERT_MONITOR_DOMAINS[@]}"; do [[ -z "$domain" ]] && continue check_cert "$domain" result=$? case $result in 0) HEALTHY+=("$domain"); DOMAIN_STATUS["$domain"]="OK" ;; 1) WARNING+=("$domain"); DOMAIN_STATUS["$domain"]="WARN" ;; 2) CRITICAL+=("$domain"); DOMAIN_STATUS["$domain"]="CRIT" ;; 3) FAILED+=("$domain"); DOMAIN_STATUS["$domain"]="FAIL" ;; esac done END=$(date +%s) # ── Send notifications — batched per severity ───────────────────────────────────────────────── if [[ "$DRY_RUN" == false ]]; then [[ ${#CRITICAL[@]} -gt 0 ]] && \ notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" \ "Certificate Monitor" "warning" [[ ${#WARNING[@]} -gt 0 ]] && \ notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" \ "Certificate Monitor" "warning" [[ ${#FAILED[@]} -gt 0 ]] && \ notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" \ "Certificate Monitor" "warning" fi # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY CERTIFICATE MONITOR SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]}" [[ ${#WARNING[@]} -gt 0 ]] && warn "Warning: ${#WARNING[@]} — renewal recommended" [[ ${#CRITICAL[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#CRITICAL[@]} — ACTION REQUIRED" [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#FAILED[@]} — unreachable" echo "" # Per-domain results — only show problems, healthy ones stay in log() for domain in "${CERT_MONITOR_DOMAINS[@]}"; do [[ -z "$domain" ]] && continue case "${DOMAIN_STATUS[$domain]:-UNKN}" in OK) log " $ICON_SUCCESS $domain — healthy" ;; WARN) warn " $ICON_WARN $domain — warning" ;; CRIT) echo " $ICON_ERROR $domain — CRITICAL" ;; FAIL) echo " $ICON_ERROR $domain — unreachable" ;; esac done echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no notifications sent" elif [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]]; then echo "$ICON_ERROR Status: ACTION REQUIRED" elif [[ ${#WARNING[@]} -gt 0 ]]; then warn "Status: WARNINGS — renewal recommended" else log "$ICON_DONE Status: all ${#HEALTHY[@]} certs healthy ✅" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]] && exit 1 exit 0