Gmer4Lfe
6623d1e776
Codebase-wide audit pass: fixed real bugs (SSH hangs missing BatchMode, local-outside-function no-ops, variable name collisions, a truncated ratio calc, wrong state-dir path, DARK vs NO_INTERNET drift, and more), then pulled logic that was duplicated across multiple scripts — arr cleanup safety gates, docker restart ordering, container maintenance stop/restart, watchdog state-file helpers, partnership role resolution, cert expiry checks, remote node discovery, and TMDB discovery scoring — into common.sh so each now has a single implementation.
288 lines
12 KiB
Bash
Executable File
288 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Certificate Monitor ============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# SSL certificate expiry monitoring for all configured domains. Scheduled weekly
|
|
# (Sunday 9am). Connects via openssl directly to each domain — not to NPM's API,
|
|
# not to any internal check, but to the actual TLS handshake the outside world sees.
|
|
#
|
|
# Per domain: HEALTHY (> CERT_WARN_DAYS remaining, silent) | WARNING (≤ CERT_WARN_DAYS)
|
|
# | CRITICAL (≤ CERT_CRIT_DAYS) | FAILED (could not connect or parse cert).
|
|
# Notifications batched by severity — one message lists all WARNING domains, a
|
|
# separate message lists all CRITICAL domains. Not one notification per domain.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Direct openssl, Not an API
|
|
# API-based cert checks ask the certificate manager whether the cert is valid.
|
|
# openssl checks ask the server what cert it is actually serving. These are not
|
|
# the same question and the answers can differ. Catches: cert renewed in NPM but
|
|
# server not reloaded (old cert still serving), wrong cert being served to external
|
|
# clients, chain issues visible externally but not internally, NPM reporting healthy
|
|
# while the outside world sees an expired cert.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Single Instance Lock
|
|
# acquire_lock prevents concurrent runs producing duplicate notifications.
|
|
#
|
|
# Per-Host Domain List
|
|
# detect_hosts() aliases HOST*_CERT_MONITOR_DOMAINS → CERT_MONITOR_DOMAINS.
|
|
# Each server monitors its own domains only.
|
|
#
|
|
# Empty Array Guard
|
|
# Warns and exits cleanly if CERT_MONITOR_DOMAINS is empty — no silent no-op.
|
|
#
|
|
# Connection Timeout
|
|
# CERT_TIMEOUT caps each openssl connection attempt. One unreachable domain
|
|
# does not block the remaining domains.
|
|
#
|
|
# Notification Validated
|
|
# platform_require_cmd confirms openssl and notify script are present before use.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_CERT_MONITOR_DOMAINS
|
|
# Domains this host monitors. Each domain and subdomain is a separate entry —
|
|
# they have independent certs. Aliased by detect_hosts() → CERT_MONITOR_DOMAINS.
|
|
#
|
|
# master.conf
|
|
#
|
|
# CERT_WARN_DAYS
|
|
# Days before expiry at which to send a warning notification. (default: 30)
|
|
#
|
|
# CERT_CRIT_DAYS
|
|
# Days before expiry at which to send a critical notification. (default: 7)
|
|
#
|
|
# CERT_TIMEOUT
|
|
# Seconds to wait per domain before declaring FAILED. (default: 10)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# cert_monitor.sh
|
|
# Check all configured domains and notify on WARNING, CRITICAL, or FAILED.
|
|
# Silent when all domains are healthy.
|
|
#
|
|
# cert_monitor.sh --dry-run
|
|
# Check all domains and show results. No notifications sent regardless of result.
|
|
#
|
|
# cert_monitor.sh --status
|
|
# Show domain list, warning thresholds, and timeout. Then exit.
|
|
#
|
|
# cert_monitor.sh --log
|
|
# Include healthy domains in per-domain output with expiry date and days remaining.
|
|
# Problems (WARN/CRIT/FAIL) always show with their details regardless of this flag.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
# Validate openssl — required for all cert checks
|
|
platform_require_cmd \
|
|
"$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \
|
|
"version" "OpenSSL" \
|
|
"openssl" || { error "openssl not found — required for certificate checks"; exit 1; }
|
|
|
|
|
|
acquire_lock
|
|
|
|
# detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS
|
|
detect_hosts
|
|
|
|
# Empty array guard
|
|
if [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then
|
|
warn "CERT_MONITOR_DOMAINS is empty for $MY_ID"
|
|
warn "Check HOST*_CERT_MONITOR_DOMAINS in host*.conf"
|
|
exit 0
|
|
fi
|
|
|
|
info "Domains to check: ${#CERT_MONITOR_DOMAINS[@]}"
|
|
log "$ICON_GEAR Config: warn=${CERT_WARN_DAYS}d crit=${CERT_CRIT_DAYS}d timeout=${CERT_TIMEOUT}s"
|
|
log "$ICON_GEAR Domains: ${CERT_MONITOR_DOMAINS[*]}"
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}"
|
|
echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining"
|
|
echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining"
|
|
echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ── CERT CHECK FUNCTION ───────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
# Connects to domain:443 via openssl, extracts expiry date, calculates days remaining.
|
|
# Returns:
|
|
# 0 = healthy (> CERT_WARN_DAYS)
|
|
# 1 = warning (<= CERT_WARN_DAYS)
|
|
# 2 = critical (<= CERT_CRIT_DAYS)
|
|
# 3 = failed (could not connect or parse)
|
|
|
|
check_cert() {
|
|
local domain="$1"
|
|
local port="${2:-443}"
|
|
|
|
if ! check_cert_expiry "$domain" "$port" "$CERT_TIMEOUT"; then
|
|
if [[ -n "$_CERT_EXPIRY_RAW" ]]; then
|
|
error "$ICON_CERT $domain — could not parse expiry date: $_CERT_EXPIRY_RAW"
|
|
else
|
|
error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)"
|
|
fi
|
|
return 3
|
|
fi
|
|
|
|
if [[ "$_CERT_DAYS" -le "$CERT_CRIT_DAYS" ]]; then
|
|
error "$ICON_CERT $domain — CRITICAL: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)"
|
|
return 2
|
|
elif [[ "$_CERT_DAYS" -le "$CERT_WARN_DAYS" ]]; then
|
|
warn "$ICON_CERT $domain — WARNING: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)"
|
|
return 1
|
|
else
|
|
log "$ICON_CERT $domain — OK: ${_CERT_DAYS} days remaining (expires $_CERT_EXPIRY)"
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Certificate Monitor ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_CERT Certificate Monitor — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
|
info "Warn threshold: ${CERT_WARN_DAYS} days"
|
|
info "Crit threshold: ${CERT_CRIT_DAYS} days"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
HEALTHY=()
|
|
WARNING=()
|
|
CRITICAL=()
|
|
FAILED=()
|
|
declare -A DOMAIN_STATUS DOMAIN_DAYS DOMAIN_EXPIRY
|
|
|
|
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
|
[[ -z "$domain" ]] && continue
|
|
check_cert "$domain"
|
|
result=$?
|
|
DOMAIN_DAYS["$domain"]="${_CERT_DAYS:-}"
|
|
DOMAIN_EXPIRY["$domain"]="${_CERT_EXPIRY:-}"
|
|
case $result in
|
|
0) HEALTHY+=("$domain"); DOMAIN_STATUS["$domain"]="OK" ;;
|
|
1) WARNING+=("$domain"); DOMAIN_STATUS["$domain"]="WARN" ;;
|
|
2) CRITICAL+=("$domain"); DOMAIN_STATUS["$domain"]="CRIT" ;;
|
|
3) FAILED+=("$domain"); DOMAIN_STATUS["$domain"]="FAIL" ;;
|
|
esac
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ── Send notifications — batched per severity ─────────────────────────────────────────────────
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
[[ ${#CRITICAL[@]} -gt 0 ]] && \
|
|
notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" \
|
|
"Certificate Monitor" "warning"
|
|
[[ ${#WARNING[@]} -gt 0 ]] && \
|
|
notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" \
|
|
"Certificate Monitor" "warning"
|
|
[[ ${#FAILED[@]} -gt 0 ]] && \
|
|
notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" \
|
|
"Certificate Monitor" "warning"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY CERTIFICATE MONITOR SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]}"
|
|
[[ ${#WARNING[@]} -gt 0 ]] && warn "Warning: ${#WARNING[@]} — renewal recommended"
|
|
[[ ${#CRITICAL[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#CRITICAL[@]} — ACTION REQUIRED"
|
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#FAILED[@]} — unreachable"
|
|
echo ""
|
|
|
|
# Per-domain results — problems always shown with days remaining; healthy only with --log
|
|
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
|
[[ -z "$domain" ]] && continue
|
|
_days="${DOMAIN_DAYS[$domain]:-?}" _exp="${DOMAIN_EXPIRY[$domain]:-unknown}"
|
|
case "${DOMAIN_STATUS[$domain]:-UNKN}" in
|
|
OK) log " $ICON_SUCCESS $domain — healthy (${_days}d, expires ${_exp})" ;;
|
|
WARN) warn " $ICON_WARN $domain — warning (${_days}d, expires ${_exp})" ;;
|
|
CRIT) echo " $ICON_ERROR $domain — CRITICAL (${_days}d, expires ${_exp})" ;;
|
|
FAIL) echo " $ICON_ERROR $domain — unreachable" ;;
|
|
esac
|
|
done
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no notifications sent"
|
|
elif [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]]; then
|
|
echo "$ICON_ERROR Status: ACTION REQUIRED"
|
|
elif [[ ${#WARNING[@]} -gt 0 ]]; then
|
|
warn "Status: WARNINGS — renewal recommended"
|
|
else
|
|
echo "$ICON_DONE Status: all ${#HEALTHY[@]} certs healthy ✅"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
# ── Write JSON status cache ───────────────────────────────────────────────────
|
|
_CERT_CACHE_FILE="$STATE_DIR/cert_status.json"
|
|
{
|
|
printf '{"checked_at":%d,"host":"%s","warn_days":%d,"crit_days":%d,"dry_run":%s,"domains":[\n' \
|
|
"$(date +%s)" "$MY_ID" "$CERT_WARN_DAYS" "$CERT_CRIT_DAYS" \
|
|
"$([[ $DRY_RUN == true ]] && echo true || echo false)"
|
|
_first=true
|
|
for _d in "${CERT_MONITOR_DOMAINS[@]}"; do
|
|
[[ -z "$_d" ]] && continue
|
|
[[ "$_first" != true ]] && printf ','
|
|
_first=false
|
|
_days="${DOMAIN_DAYS[$_d]:-null}"
|
|
_exp="${DOMAIN_EXPIRY[$_d]:-}"
|
|
printf '{"domain":"%s","status":"%s","days":%s,"expires":"%s"}\n' \
|
|
"$_d" "${DOMAIN_STATUS[$_d]:-UNKN}" "$_days" "$_exp"
|
|
done
|
|
printf ']}\n'
|
|
} > "$_CERT_CACHE_FILE" 2>/dev/null
|
|
|
|
[[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]] && exit 1
|
|
exit 0 |