Gmer4Lfe
369a9e6c19
- Rename unRAID_Essentials/ → System_Essentials/ (git detects as rename) - Add Plugin/unraid/adapter.sh: 13 platform_*() functions providing OS-agnostic API for storage health, service management, mover, user scripts, notifications, disk temps, and platform command validation - Update load_config.sh: detect PLATFORM (unraid/truenas/unknown), export SCRIPTS_DIR, auto-source Plugin/$PLATFORM/adapter.sh after common.sh - Wire all call sites: replace direct rc.d, pgrep/pkill, var.ini, dynamix.cfg, disks.ini, and validate_unraid_cmd calls with platform_*() functions across watchdogs, orchestrators, and System_Essentials scripts - Update all documentation: rename refs, update webgui escalation logic, add platform adapter section to Plugin README, update main README with portability vision and corrected self-healing stack description
282 lines
12 KiB
Bash
Executable File
282 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Certificate Monitor ============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# SSL certificate expiry monitoring for all configured domains. Scheduled weekly
|
|
# (Sunday 9am). Connects via openssl directly to each domain — not to NPM's API,
|
|
# not to any internal check, but to the actual TLS handshake the outside world sees.
|
|
#
|
|
# Per domain: HEALTHY (> CERT_WARN_DAYS remaining, silent) | WARNING (≤ CERT_WARN_DAYS)
|
|
# | CRITICAL (≤ CERT_CRIT_DAYS) | FAILED (could not connect or parse cert).
|
|
# Notifications batched by severity — one message lists all WARNING domains, a
|
|
# separate message lists all CRITICAL domains. Not one notification per domain.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Direct openssl, Not an API
|
|
# API-based cert checks ask the certificate manager whether the cert is valid.
|
|
# openssl checks ask the server what cert it is actually serving. These are not
|
|
# the same question and the answers can differ. Catches: cert renewed in NPM but
|
|
# server not reloaded (old cert still serving), wrong cert being served to external
|
|
# clients, chain issues visible externally but not internally, NPM reporting healthy
|
|
# while the outside world sees an expired cert.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Single Instance Lock
|
|
# acquire_lock prevents concurrent runs producing duplicate notifications.
|
|
#
|
|
# Per-Host Domain List
|
|
# detect_hosts() aliases HOST*_CERT_MONITOR_DOMAINS → CERT_MONITOR_DOMAINS.
|
|
# Each server monitors its own domains only.
|
|
#
|
|
# Empty Array Guard
|
|
# Warns and exits cleanly if CERT_MONITOR_DOMAINS is empty — no silent no-op.
|
|
#
|
|
# Connection Timeout
|
|
# CERT_TIMEOUT caps each openssl connection attempt. One unreachable domain
|
|
# does not block the remaining domains.
|
|
#
|
|
# Notification Validated
|
|
# platform_require_cmd confirms openssl and notify script are present before use.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_CERT_MONITOR_DOMAINS
|
|
# Domains this host monitors. Each domain and subdomain is a separate entry —
|
|
# they have independent certs. Aliased by detect_hosts() → CERT_MONITOR_DOMAINS.
|
|
#
|
|
# master.conf
|
|
#
|
|
# CERT_WARN_DAYS
|
|
# Days before expiry at which to send a warning notification. (default: 30)
|
|
#
|
|
# CERT_CRIT_DAYS
|
|
# Days before expiry at which to send a critical notification. (default: 7)
|
|
#
|
|
# CERT_TIMEOUT
|
|
# Seconds to wait per domain before declaring FAILED. (default: 10)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# cert_monitor.sh
|
|
# Check all configured domains and notify on WARNING, CRITICAL, or FAILED.
|
|
# Silent when all domains are healthy.
|
|
#
|
|
# cert_monitor.sh --dry-run
|
|
# Check all domains and show results. No notifications sent regardless of result.
|
|
#
|
|
# cert_monitor.sh --status
|
|
# Show domain list, warning thresholds, and timeout. Then exit.
|
|
#
|
|
# cert_monitor.sh --log
|
|
# Verbose per-domain output during the run.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
# Validate openssl — required for all cert checks
|
|
platform_require_cmd \
|
|
"$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \
|
|
"version" "OpenSSL" \
|
|
"openssl" || { error "openssl not found — required for certificate checks"; exit 1; }
|
|
|
|
platform_require_cmd \
|
|
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
|
"" "" \
|
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
|
|
|
acquire_lock
|
|
|
|
# detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS
|
|
detect_hosts
|
|
|
|
# Empty array guard
|
|
if [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then
|
|
warn "CERT_MONITOR_DOMAINS is empty for $MY_ID"
|
|
warn "Check HOST*_CERT_MONITOR_DOMAINS in host*.conf"
|
|
exit 0
|
|
fi
|
|
|
|
info "Domains to check: ${#CERT_MONITOR_DOMAINS[@]}"
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}"
|
|
echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining"
|
|
echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining"
|
|
echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ── CERT CHECK FUNCTION ───────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
# Connects to domain:443 via openssl, extracts expiry date, calculates days remaining.
|
|
# Returns:
|
|
# 0 = healthy (> CERT_WARN_DAYS)
|
|
# 1 = warning (<= CERT_WARN_DAYS)
|
|
# 2 = critical (<= CERT_CRIT_DAYS)
|
|
# 3 = failed (could not connect or parse)
|
|
|
|
check_cert() {
|
|
local domain="$1"
|
|
local port="${2:-443}"
|
|
|
|
local expiry_str
|
|
expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \
|
|
-connect "${domain}:${port}" \
|
|
-servername "$domain" \
|
|
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
|
|
|
if [[ -z "$expiry_str" ]]; then
|
|
error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)"
|
|
return 3
|
|
fi
|
|
|
|
local expiry_epoch
|
|
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
|
|
|
|
if [[ -z "$expiry_epoch" ]]; then
|
|
error "$ICON_CERT $domain — could not parse expiry date: $expiry_str"
|
|
return 3
|
|
fi
|
|
|
|
local now days_remaining expiry_display
|
|
now=$(date +%s)
|
|
days_remaining=$(( (expiry_epoch - now) / 86400 ))
|
|
expiry_display=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null)
|
|
|
|
if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then
|
|
error "$ICON_CERT $domain — CRITICAL: ${days_remaining} days remaining (expires $expiry_display)"
|
|
return 2
|
|
elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then
|
|
warn "$ICON_CERT $domain — WARNING: ${days_remaining} days remaining (expires $expiry_display)"
|
|
return 1
|
|
else
|
|
log "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)"
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Certificate Monitor ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_CERT Certificate Monitor — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
|
info "Warn threshold: ${CERT_WARN_DAYS} days"
|
|
info "Crit threshold: ${CERT_CRIT_DAYS} days"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
HEALTHY=()
|
|
WARNING=()
|
|
CRITICAL=()
|
|
FAILED=()
|
|
declare -A DOMAIN_STATUS
|
|
|
|
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
|
[[ -z "$domain" ]] && continue
|
|
check_cert "$domain"
|
|
result=$?
|
|
case $result in
|
|
0) HEALTHY+=("$domain"); DOMAIN_STATUS["$domain"]="OK" ;;
|
|
1) WARNING+=("$domain"); DOMAIN_STATUS["$domain"]="WARN" ;;
|
|
2) CRITICAL+=("$domain"); DOMAIN_STATUS["$domain"]="CRIT" ;;
|
|
3) FAILED+=("$domain"); DOMAIN_STATUS["$domain"]="FAIL" ;;
|
|
esac
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ── Send notifications — batched per severity ─────────────────────────────────────────────────
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
[[ ${#CRITICAL[@]} -gt 0 ]] && \
|
|
notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" \
|
|
"Certificate Monitor" "warning"
|
|
[[ ${#WARNING[@]} -gt 0 ]] && \
|
|
notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" \
|
|
"Certificate Monitor" "warning"
|
|
[[ ${#FAILED[@]} -gt 0 ]] && \
|
|
notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" \
|
|
"Certificate Monitor" "warning"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY CERTIFICATE MONITOR SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]}"
|
|
[[ ${#WARNING[@]} -gt 0 ]] && warn "Warning: ${#WARNING[@]} — renewal recommended"
|
|
[[ ${#CRITICAL[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#CRITICAL[@]} — ACTION REQUIRED"
|
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#FAILED[@]} — unreachable"
|
|
echo ""
|
|
|
|
# Per-domain results — only show problems, healthy ones stay in log()
|
|
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
|
[[ -z "$domain" ]] && continue
|
|
case "${DOMAIN_STATUS[$domain]:-UNKN}" in
|
|
OK) log " $ICON_SUCCESS $domain — healthy" ;;
|
|
WARN) warn " $ICON_WARN $domain — warning" ;;
|
|
CRIT) echo " $ICON_ERROR $domain — CRITICAL" ;;
|
|
FAIL) echo " $ICON_ERROR $domain — unreachable" ;;
|
|
esac
|
|
done
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no notifications sent"
|
|
elif [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]]; then
|
|
echo "$ICON_ERROR Status: ACTION REQUIRED"
|
|
elif [[ ${#WARNING[@]} -gt 0 ]]; then
|
|
warn "Status: WARNINGS — renewal recommended"
|
|
else
|
|
echo "$ICON_DONE Status: all ${#HEALTHY[@]} certs healthy ✅"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]] && exit 1
|
|
exit 0 |