250 lines
12 KiB
Bash
250 lines
12 KiB
Bash
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= Certificate Monitor ========================================
|
|
# ==============================================================================================
|
|
# Monitors SSL certificate expiry for all configured domains by connecting directly
|
|
# via openssl — no dependency on NPM or any other service. Reads the actual certificate
|
|
# the server is presenting to the outside world.
|
|
#
|
|
# ── WHY DIRECT OPENSSL ────────────────────────────────────────────────────────────────────────
|
|
# Catches real-world cert issues that API-based checks miss:
|
|
# - Cert renewed in NPM but server not reloaded (old cert still serving)
|
|
# - Wrong cert being served to external clients
|
|
# - Cert chain issues not visible from the internal network
|
|
# - NPM reporting healthy while the world sees an expired cert
|
|
#
|
|
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
|
|
# Each domain is checked independently — they have independent certs.
|
|
# Results per domain:
|
|
# HEALTHY — > CERT_WARN_DAYS remaining — silent ✅
|
|
# WARNING — <= CERT_WARN_DAYS remaining — notifies
|
|
# CRITICAL — <= CERT_CRIT_DAYS remaining — notifies with urgency
|
|
# FAILED — could not connect or parse cert — notifies
|
|
#
|
|
# Notifications batched per severity — one message per severity level, not per domain.
|
|
# This is a monitor script — SILENT_MODE=false — output is the point.
|
|
#
|
|
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
|
# detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS → CERT_MONITOR_DOMAINS.
|
|
# Each server monitors its own domains — HOST1 monitors Gmer4Lfe.com etc.
|
|
#
|
|
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
|
# acquire_lock — prevents concurrent runs
|
|
# detect_hosts() — correct domain list per host via MY_ID aliases
|
|
# Empty array guard — warns and exits cleanly if no domains configured
|
|
# CERT_TIMEOUT — openssl connects are time-limited per domain
|
|
# validate_unraid_cmd — openssl and notify validated before use
|
|
# Silent healthy certs — only problems produce visible output
|
|
#
|
|
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
|
# HOST*_CERT_MONITOR_DOMAINS — domains checked by this host
|
|
# Aliased by detect_hosts() — script uses CERT_MONITOR_DOMAINS
|
|
#
|
|
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
|
# CERT_WARN_DAYS — warn when cert expires within this many days (default 30)
|
|
# CERT_CRIT_DAYS — critical alert within this many days (default 7)
|
|
# CERT_TIMEOUT — seconds per domain before giving up (default 10)
|
|
#
|
|
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
|
# cert_monitor.sh — normal run
|
|
# cert_monitor.sh --dry-run — check certs and show results, no notifications
|
|
# cert_monitor.sh --log — verbose output
|
|
# cert_monitor.sh --status — show config and exit
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
# Monitor script — output is the point
|
|
SILENT_MODE=false
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
# Validate openssl — required for all cert checks
|
|
validate_unraid_cmd \
|
|
"$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \
|
|
"version" "OpenSSL" \
|
|
"openssl" || { error "openssl not found — required for certificate checks"; exit 1; }
|
|
|
|
validate_unraid_cmd \
|
|
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
|
"" "" \
|
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
|
|
|
acquire_lock
|
|
|
|
# detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS
|
|
detect_hosts
|
|
|
|
# Empty array guard
|
|
if [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then
|
|
warn "CERT_MONITOR_DOMAINS is empty for $MY_ID"
|
|
warn "Check HOST*_CERT_MONITOR_DOMAINS in master_host*.conf"
|
|
exit 0
|
|
fi
|
|
|
|
log "Domains to check: ${#CERT_MONITOR_DOMAINS[@]}"
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}"
|
|
echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining"
|
|
echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining"
|
|
echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ── CERT CHECK FUNCTION ───────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
# Connects to domain:443 via openssl, extracts expiry date, calculates days remaining.
|
|
# Returns:
|
|
# 0 = healthy (> CERT_WARN_DAYS)
|
|
# 1 = warning (<= CERT_WARN_DAYS)
|
|
# 2 = critical (<= CERT_CRIT_DAYS)
|
|
# 3 = failed (could not connect or parse)
|
|
|
|
check_cert() {
|
|
local domain="$1"
|
|
local port="${2:-443}"
|
|
|
|
local expiry_str
|
|
expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \
|
|
-connect "${domain}:${port}" \
|
|
-servername "$domain" \
|
|
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
|
|
|
if [[ -z "$expiry_str" ]]; then
|
|
error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)"
|
|
return 3
|
|
fi
|
|
|
|
local expiry_epoch
|
|
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
|
|
|
|
if [[ -z "$expiry_epoch" ]]; then
|
|
error "$ICON_CERT $domain — could not parse expiry date: $expiry_str"
|
|
return 3
|
|
fi
|
|
|
|
local now days_remaining expiry_display
|
|
now=$(date +%s)
|
|
days_remaining=$(( (expiry_epoch - now) / 86400 ))
|
|
expiry_display=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null)
|
|
|
|
if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then
|
|
error "$ICON_CERT $domain — CRITICAL: ${days_remaining} days remaining (expires $expiry_display)"
|
|
return 2
|
|
elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then
|
|
warn "$ICON_CERT $domain — WARNING: ${days_remaining} days remaining (expires $expiry_display)"
|
|
return 1
|
|
else
|
|
log "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)"
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Certificate Monitor ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_CERT Certificate Monitor — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
|
log "Warn threshold: ${CERT_WARN_DAYS} days"
|
|
log "Crit threshold: ${CERT_CRIT_DAYS} days"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
HEALTHY=()
|
|
WARNING=()
|
|
CRITICAL=()
|
|
FAILED=()
|
|
declare -A DOMAIN_STATUS
|
|
|
|
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
|
[[ -z "$domain" ]] && continue
|
|
check_cert "$domain"
|
|
result=$?
|
|
case $result in
|
|
0) HEALTHY+=("$domain"); DOMAIN_STATUS["$domain"]="OK" ;;
|
|
1) WARNING+=("$domain"); DOMAIN_STATUS["$domain"]="WARN" ;;
|
|
2) CRITICAL+=("$domain"); DOMAIN_STATUS["$domain"]="CRIT" ;;
|
|
3) FAILED+=("$domain"); DOMAIN_STATUS["$domain"]="FAIL" ;;
|
|
esac
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ── Send notifications — batched per severity ─────────────────────────────────────────────────
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
[[ ${#CRITICAL[@]} -gt 0 ]] && \
|
|
notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" \
|
|
"Certificate Monitor" "warning"
|
|
[[ ${#WARNING[@]} -gt 0 ]] && \
|
|
notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" \
|
|
"Certificate Monitor" "warning"
|
|
[[ ${#FAILED[@]} -gt 0 ]] && \
|
|
notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" \
|
|
"Certificate Monitor" "warning"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY CERTIFICATE MONITOR SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]}"
|
|
[[ ${#WARNING[@]} -gt 0 ]] && warn "Warning: ${#WARNING[@]} — renewal recommended"
|
|
[[ ${#CRITICAL[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#CRITICAL[@]} — ACTION REQUIRED"
|
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#FAILED[@]} — unreachable"
|
|
echo ""
|
|
|
|
# Per-domain results — only show problems, healthy ones stay in log()
|
|
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
|
[[ -z "$domain" ]] && continue
|
|
case "${DOMAIN_STATUS[$domain]:-UNKN}" in
|
|
OK) log " $ICON_SUCCESS $domain — healthy" ;;
|
|
WARN) warn " $ICON_WARN $domain — warning" ;;
|
|
CRIT) echo " $ICON_ERROR $domain — CRITICAL" ;;
|
|
FAIL) echo " $ICON_ERROR $domain — unreachable" ;;
|
|
esac
|
|
done
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no notifications sent"
|
|
elif [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]]; then
|
|
echo "$ICON_ERROR Status: ACTION REQUIRED"
|
|
elif [[ ${#WARNING[@]} -gt 0 ]]; then
|
|
warn "Status: WARNINGS — renewal recommended"
|
|
else
|
|
log "$ICON_DONE Status: all ${#HEALTHY[@]} certs healthy ✅"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]] && exit 1
|
|
exit 0 |