audit echo vs log across all scripts — outcomes always visible, verbose for per-item loops
This commit is contained in:
Gmer4Lfe
+305
@@ -0,0 +1,305 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Certificate Monitor ============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# SSL certificate expiry monitoring for all configured domains. Scheduled weekly
|
||||
# (Sunday 9am). Connects via openssl directly to each domain — not to NPM's API,
|
||||
# not to any internal check, but to the actual TLS handshake the outside world sees.
|
||||
#
|
||||
# Per domain: HEALTHY (> CERT_WARN_DAYS remaining, silent) | WARNING (≤ CERT_WARN_DAYS)
|
||||
# | CRITICAL (≤ CERT_CRIT_DAYS) | FAILED (could not connect or parse cert).
|
||||
# Notifications batched by severity — one message lists all WARNING domains, a
|
||||
# separate message lists all CRITICAL domains. Not one notification per domain.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Direct openssl, Not an API
|
||||
# API-based cert checks ask the certificate manager whether the cert is valid.
|
||||
# openssl checks ask the server what cert it is actually serving. These are not
|
||||
# the same question and the answers can differ. Catches: cert renewed in NPM but
|
||||
# server not reloaded (old cert still serving), wrong cert being served to external
|
||||
# clients, chain issues visible externally but not internally, NPM reporting healthy
|
||||
# while the outside world sees an expired cert.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Single Instance Lock
|
||||
# acquire_lock prevents concurrent runs producing duplicate notifications.
|
||||
#
|
||||
# Per-Host Domain List
|
||||
# detect_hosts() aliases HOST*_CERT_MONITOR_DOMAINS → CERT_MONITOR_DOMAINS.
|
||||
# Each server monitors its own domains only.
|
||||
#
|
||||
# Empty Array Guard
|
||||
# Warns and exits cleanly if CERT_MONITOR_DOMAINS is empty — no silent no-op.
|
||||
#
|
||||
# Connection Timeout
|
||||
# CERT_TIMEOUT caps each openssl connection attempt. One unreachable domain
|
||||
# does not block the remaining domains.
|
||||
#
|
||||
# Notification Validated
|
||||
# platform_require_cmd confirms openssl and notify script are present before use.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_CERT_MONITOR_DOMAINS
|
||||
# Domains this host monitors. Each domain and subdomain is a separate entry —
|
||||
# they have independent certs. Aliased by detect_hosts() → CERT_MONITOR_DOMAINS.
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# CERT_WARN_DAYS
|
||||
# Days before expiry at which to send a warning notification. (default: 30)
|
||||
#
|
||||
# CERT_CRIT_DAYS
|
||||
# Days before expiry at which to send a critical notification. (default: 7)
|
||||
#
|
||||
# CERT_TIMEOUT
|
||||
# Seconds to wait per domain before declaring FAILED. (default: 10)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# cert_monitor.sh
|
||||
# Check all configured domains and notify on WARNING, CRITICAL, or FAILED.
|
||||
# Silent when all domains are healthy.
|
||||
#
|
||||
# cert_monitor.sh --dry-run
|
||||
# Check all domains and show results. No notifications sent regardless of result.
|
||||
#
|
||||
# cert_monitor.sh --status
|
||||
# Show domain list, warning thresholds, and timeout. Then exit.
|
||||
#
|
||||
# cert_monitor.sh --log
|
||||
# Verbose per-domain output during the run.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Validate openssl — required for all cert checks
|
||||
platform_require_cmd \
|
||||
"$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \
|
||||
"version" "OpenSSL" \
|
||||
"openssl" || { error "openssl not found — required for certificate checks"; exit 1; }
|
||||
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS
|
||||
detect_hosts
|
||||
|
||||
# Empty array guard
|
||||
if [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then
|
||||
warn "CERT_MONITOR_DOMAINS is empty for $MY_ID"
|
||||
warn "Check HOST*_CERT_MONITOR_DOMAINS in host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
info "Domains to check: ${#CERT_MONITOR_DOMAINS[@]}"
|
||||
log "$ICON_GEAR Config: warn=${CERT_WARN_DAYS}d crit=${CERT_CRIT_DAYS}d timeout=${CERT_TIMEOUT}s"
|
||||
log "$ICON_GEAR Domains: ${CERT_MONITOR_DOMAINS[*]}"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}"
|
||||
echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining"
|
||||
echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining"
|
||||
echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── CERT CHECK FUNCTION ───────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Connects to domain:443 via openssl, extracts expiry date, calculates days remaining.
|
||||
# Returns:
|
||||
# 0 = healthy (> CERT_WARN_DAYS)
|
||||
# 1 = warning (<= CERT_WARN_DAYS)
|
||||
# 2 = critical (<= CERT_CRIT_DAYS)
|
||||
# 3 = failed (could not connect or parse)
|
||||
|
||||
check_cert() {
|
||||
local domain="$1"
|
||||
local port="${2:-443}"
|
||||
_CERT_DAYS=""
|
||||
_CERT_EXPIRY=""
|
||||
|
||||
local expiry_str
|
||||
expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \
|
||||
-connect "${domain}:${port}" \
|
||||
-servername "$domain" \
|
||||
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||||
|
||||
if [[ -z "$expiry_str" ]]; then
|
||||
error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)"
|
||||
return 3
|
||||
fi
|
||||
|
||||
local expiry_epoch
|
||||
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
|
||||
|
||||
if [[ -z "$expiry_epoch" ]]; then
|
||||
error "$ICON_CERT $domain — could not parse expiry date: $expiry_str"
|
||||
return 3
|
||||
fi
|
||||
|
||||
local now days_remaining expiry_display
|
||||
now=$(date +%s)
|
||||
days_remaining=$(( (expiry_epoch - now) / 86400 ))
|
||||
expiry_display=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null)
|
||||
_CERT_DAYS=$days_remaining
|
||||
_CERT_EXPIRY=$expiry_display
|
||||
|
||||
if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then
|
||||
error "$ICON_CERT $domain — CRITICAL: ${days_remaining} days remaining (expires $expiry_display)"
|
||||
return 2
|
||||
elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then
|
||||
warn "$ICON_CERT $domain — WARNING: ${days_remaining} days remaining (expires $expiry_display)"
|
||||
return 1
|
||||
else
|
||||
log "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CERT Certificate Monitor — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
info "Warn threshold: ${CERT_WARN_DAYS} days"
|
||||
info "Crit threshold: ${CERT_CRIT_DAYS} days"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
HEALTHY=()
|
||||
WARNING=()
|
||||
CRITICAL=()
|
||||
FAILED=()
|
||||
declare -A DOMAIN_STATUS DOMAIN_DAYS DOMAIN_EXPIRY
|
||||
|
||||
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
||||
[[ -z "$domain" ]] && continue
|
||||
check_cert "$domain"
|
||||
result=$?
|
||||
DOMAIN_DAYS["$domain"]="${_CERT_DAYS:-}"
|
||||
DOMAIN_EXPIRY["$domain"]="${_CERT_EXPIRY:-}"
|
||||
case $result in
|
||||
0) HEALTHY+=("$domain"); DOMAIN_STATUS["$domain"]="OK" ;;
|
||||
1) WARNING+=("$domain"); DOMAIN_STATUS["$domain"]="WARN" ;;
|
||||
2) CRITICAL+=("$domain"); DOMAIN_STATUS["$domain"]="CRIT" ;;
|
||||
3) FAILED+=("$domain"); DOMAIN_STATUS["$domain"]="FAIL" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ── Send notifications — batched per severity ─────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
[[ ${#CRITICAL[@]} -gt 0 ]] && \
|
||||
notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" \
|
||||
"Certificate Monitor" "warning"
|
||||
[[ ${#WARNING[@]} -gt 0 ]] && \
|
||||
notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" \
|
||||
"Certificate Monitor" "warning"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && \
|
||||
notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" \
|
||||
"Certificate Monitor" "warning"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CERTIFICATE MONITOR SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]}"
|
||||
[[ ${#WARNING[@]} -gt 0 ]] && warn "Warning: ${#WARNING[@]} — renewal recommended"
|
||||
[[ ${#CRITICAL[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#CRITICAL[@]} — ACTION REQUIRED"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#FAILED[@]} — unreachable"
|
||||
echo ""
|
||||
|
||||
# Per-domain results — only show problems, healthy ones stay in log()
|
||||
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
||||
[[ -z "$domain" ]] && continue
|
||||
case "${DOMAIN_STATUS[$domain]:-UNKN}" in
|
||||
OK) log " $ICON_SUCCESS $domain — healthy" ;;
|
||||
WARN) warn " $ICON_WARN $domain — warning" ;;
|
||||
CRIT) echo " $ICON_ERROR $domain — CRITICAL" ;;
|
||||
FAIL) echo " $ICON_ERROR $domain — unreachable" ;;
|
||||
esac
|
||||
done
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no notifications sent"
|
||||
elif [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ACTION REQUIRED"
|
||||
elif [[ ${#WARNING[@]} -gt 0 ]]; then
|
||||
warn "Status: WARNINGS — renewal recommended"
|
||||
else
|
||||
echo "$ICON_DONE Status: all ${#HEALTHY[@]} certs healthy ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# ── Write JSON status cache ───────────────────────────────────────────────────
|
||||
_CERT_CACHE_FILE="$SCRIPTS_DIR/State_Files/cert_status.json"
|
||||
{
|
||||
printf '{"checked_at":%d,"host":"%s","warn_days":%d,"crit_days":%d,"dry_run":%s,"domains":[\n' \
|
||||
"$(date +%s)" "$MY_ID" "$CERT_WARN_DAYS" "$CERT_CRIT_DAYS" \
|
||||
"$([[ $DRY_RUN == true ]] && echo true || echo false)"
|
||||
_first=true
|
||||
for _d in "${CERT_MONITOR_DOMAINS[@]}"; do
|
||||
[[ -z "$_d" ]] && continue
|
||||
[[ "$_first" != true ]] && printf ','
|
||||
_first=false
|
||||
_days="${DOMAIN_DAYS[$_d]:-null}"
|
||||
_exp="${DOMAIN_EXPIRY[$_d]:-}"
|
||||
printf '{"domain":"%s","status":"%s","days":%s,"expires":"%s"}\n' \
|
||||
"$_d" "${DOMAIN_STATUS[$_d]:-UNKN}" "$_days" "$_exp"
|
||||
done
|
||||
printf ']}\n'
|
||||
} > "$_CERT_CACHE_FILE" 2>/dev/null
|
||||
|
||||
[[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user