#!/bin/bash # ============================================================================================== # ============================= Certificate Monitor ============================================ # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # SSL certificate expiry monitoring for all configured domains. Scheduled weekly # (Sunday 9am). Connects via openssl directly to each domain — not to NPM's API, # not to any internal check, but to the actual TLS handshake the outside world sees. # # Per domain: HEALTHY (> CERT_WARN_DAYS remaining, silent) | WARNING (≤ CERT_WARN_DAYS) # | CRITICAL (≤ CERT_CRIT_DAYS) | FAILED (could not connect or parse cert). # Notifications batched by severity — one message lists all WARNING domains, a # separate message lists all CRITICAL domains. Not one notification per domain. # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # 1. Validate openssl is present (platform_require_cmd) — without it nothing can be checked # 2. Per domain in CERT_MONITOR_DOMAINS: # a. Open a real TLS connection with openssl s_client # b. Parse notAfter from the served certificate # c. Compute days remaining # d. Classify: HEALTHY (silent) | WARNING (≤ CERT_WARN_DAYS) # | CRITICAL (≤ CERT_CRIT_DAYS) | FAILED (no connect / no parse) # 3. Batch by severity — one notification listing all WARNING domains, a separate # one listing all CRITICAL domains # # A domain that fails to connect is reported as FAILED rather than assumed healthy or # assumed expired — an unreachable host and an expiring cert are different problems. # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Direct openssl, Not an API # API-based cert checks ask the certificate manager whether the cert is valid. # openssl checks ask the server what cert it is actually serving. These are not # the same question and the answers can differ. Catches: cert renewed in NPM but # server not reloaded (old cert still serving), wrong cert being served to external # clients, chain issues visible externally but not internally, NPM reporting healthy # while the outside world sees an expired cert. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Single Instance Lock # acquire_lock prevents concurrent runs producing duplicate notifications. # # Per-Host Domain List # detect_hosts() aliases HOST*_CERT_MONITOR_DOMAINS → CERT_MONITOR_DOMAINS. # Each server monitors its own domains only. # # Empty Array Guard # Warns and exits cleanly if CERT_MONITOR_DOMAINS is empty — no silent no-op. # # Connection Timeout # CERT_TIMEOUT caps each openssl connection attempt. One unreachable domain # does not block the remaining domains. # # openssl Validated # platform_require_cmd confirms openssl is present before any domain is checked — every # check depends on it, so a missing binary is reported as itself rather than as every # domain failing. The notify script is validated separately by the platform adapter. # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # host*.conf # # HOST*_CERT_MONITOR_DOMAINS # Domains this host monitors. Each domain and subdomain is a separate entry — # they have independent certs. Aliased by detect_hosts() → CERT_MONITOR_DOMAINS. # # master.conf # # CERT_WARN_DAYS # Days before expiry at which to send a warning notification. (default: 30) # # CERT_CRIT_DAYS # Days before expiry at which to send a critical notification. (default: 7) # # CERT_TIMEOUT # Seconds to wait per domain before declaring FAILED. (default: 10) # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # cert_monitor.sh # Check all configured domains and notify on WARNING, CRITICAL, or FAILED. # Silent when all domains are healthy. # # cert_monitor.sh --dry-run # Check all domains and show results. No notifications sent regardless of result. # # cert_monitor.sh --status # Show domain list, warning thresholds, and timeout. Then exit. # # cert_monitor.sh --log # Include healthy domains in per-domain output with expiry date and days remaining. # Problems (WARN/CRIT/FAIL) always show with their details regardless of this flag. # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_GEAR Setup ━━━" if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi # Validate openssl — required for all cert checks platform_require_cmd \ "$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \ "version" "OpenSSL" \ "openssl" || { error "openssl not found — required for certificate checks"; exit 1; } acquire_lock # detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS detect_hosts # Empty array guard if [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then warn "CERT_MONITOR_DOMAINS is empty for $MY_ID" warn "Check HOST*_CERT_MONITOR_DOMAINS in host*.conf" exit 0 fi info "Domains to check: ${#CERT_MONITOR_DOMAINS[@]}" log "$ICON_GEAR Config: warn=${CERT_WARN_DAYS}d crit=${CERT_CRIT_DAYS}d timeout=${CERT_TIMEOUT}s" log "$ICON_GEAR Domains: ${CERT_MONITOR_DOMAINS[*]}" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent" # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}" echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining" echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining" echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain" echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ── CERT CHECK FUNCTION ─────────────────────────────────────────────────────────────────────── # ============================================================================================== # Connects to domain:443 via openssl, extracts expiry date, calculates days remaining. # Returns: # 0 = healthy (> CERT_WARN_DAYS) # 1 = warning (<= CERT_WARN_DAYS) # 2 = critical (<= CERT_CRIT_DAYS) # 3 = failed (could not connect or parse) check_cert() { local domain="$1" local port="${2:-443}" if ! check_cert_expiry "$domain" "$port" "$CERT_TIMEOUT"; then if [[ -n "$_CERT_EXPIRY_RAW" ]]; then error "$ICON_CERT $domain — could not parse expiry date: $_CERT_EXPIRY_RAW" else error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)" fi return 3 fi if [[ "$_CERT_DAYS" -le "$CERT_CRIT_DAYS" ]]; then error "$ICON_CERT $domain — CRITICAL: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)" return 2 elif [[ "$_CERT_DAYS" -le "$CERT_WARN_DAYS" ]]; then warn "$ICON_CERT $domain — WARNING: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)" return 1 else log "$ICON_CERT $domain — OK: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)" return 0 fi } # ============================================================================================== # ━━━ Certificate Monitor ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_CERT Certificate Monitor — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" info "Warn threshold: ${CERT_WARN_DAYS} days" info "Crit threshold: ${CERT_CRIT_DAYS} days" echo "" START=$(date +%s) HEALTHY=() WARNING=() CRITICAL=() FAILED=() declare -A DOMAIN_STATUS DOMAIN_DAYS DOMAIN_EXPIRY for domain in "${CERT_MONITOR_DOMAINS[@]}"; do [[ -z "$domain" ]] && continue check_cert "$domain" result=$? DOMAIN_DAYS["$domain"]="${_CERT_DAYS:-}" DOMAIN_EXPIRY["$domain"]="${_CERT_EXPIRY:-}" case $result in 0) HEALTHY+=("$domain"); DOMAIN_STATUS["$domain"]="OK" ;; 1) WARNING+=("$domain"); DOMAIN_STATUS["$domain"]="WARN" ;; 2) CRITICAL+=("$domain"); DOMAIN_STATUS["$domain"]="CRIT" ;; 3) FAILED+=("$domain"); DOMAIN_STATUS["$domain"]="FAIL" ;; esac done END=$(date +%s) # ── Send notifications — batched per severity ───────────────────────────────────────────────── if [[ "$DRY_RUN" == false ]]; then [[ ${#CRITICAL[@]} -gt 0 ]] && \ notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" \ "Certificate Monitor" "warning" [[ ${#WARNING[@]} -gt 0 ]] && \ notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" \ "Certificate Monitor" "warning" [[ ${#FAILED[@]} -gt 0 ]] && \ notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" \ "Certificate Monitor" "warning" fi # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY CERTIFICATE MONITOR SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]}" [[ ${#WARNING[@]} -gt 0 ]] && warn "Warning: ${#WARNING[@]} — renewal recommended" [[ ${#CRITICAL[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#CRITICAL[@]} — ACTION REQUIRED" [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#FAILED[@]} — unreachable" echo "" # Per-domain results — problems always shown with days remaining; healthy only with --log for domain in "${CERT_MONITOR_DOMAINS[@]}"; do [[ -z "$domain" ]] && continue _days="${DOMAIN_DAYS[$domain]:-?}" _exp="${DOMAIN_EXPIRY[$domain]:-unknown}" case "${DOMAIN_STATUS[$domain]:-UNKN}" in OK) log " $ICON_SUCCESS $domain — healthy (${_days}d, expires ${_exp})" ;; WARN) warn " $ICON_WARN $domain — warning (${_days}d, expires ${_exp})" ;; CRIT) echo " $ICON_ERROR $domain — CRITICAL (${_days}d, expires ${_exp})" ;; FAIL) echo " $ICON_ERROR $domain — unreachable" ;; esac done echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no notifications sent" elif [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]]; then echo "$ICON_ERROR Status: ACTION REQUIRED" elif [[ ${#WARNING[@]} -gt 0 ]]; then warn "Status: WARNINGS — renewal recommended" else echo "$ICON_DONE Status: all ${#HEALTHY[@]} certs healthy ✅" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # ── Write JSON status cache ─────────────────────────────────────────────────── _CERT_CACHE_FILE="$STATE_DIR/cert_status.json" { printf '{"checked_at":%d,"host":"%s","warn_days":%d,"crit_days":%d,"dry_run":%s,"domains":[\n' \ "$(date +%s)" "$MY_ID" "$CERT_WARN_DAYS" "$CERT_CRIT_DAYS" \ "$([[ $DRY_RUN == true ]] && echo true || echo false)" _first=true for _d in "${CERT_MONITOR_DOMAINS[@]}"; do [[ -z "$_d" ]] && continue [[ "$_first" != true ]] && printf ',' _first=false _days="${DOMAIN_DAYS[$_d]:-null}" _exp="${DOMAIN_EXPIRY[$_d]:-}" printf '{"domain":"%s","status":"%s","days":%s,"expires":"%s"}\n' \ "$_d" "${DOMAIN_STATUS[$_d]:-UNKN}" "$_days" "$_exp" done printf ']}\n' } > "$_CERT_CACHE_FILE" 2>/dev/null [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]] && exit 1 exit 0