massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
+205
-161
@@ -1,50 +1,117 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Health Digest ----------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= Health Digest ==============================================
|
||||
# ==============================================================================================
|
||||
# Aggregates system health data from across the ecosystem into a single digest report.
|
||||
# Reads existing state files — no new writes to flash drive.
|
||||
# Reads existing state files — no new writes.
|
||||
#
|
||||
# ── THREE PROFILES ────────────────────────────────────────────────────────────────────────────
|
||||
# always — sends every run regardless of findings
|
||||
# schedule daily for a daily digest
|
||||
#
|
||||
# smart — sends only if something worth reporting was found
|
||||
# runs every run but stays silent when all healthy
|
||||
# DIGEST_SMART_ON_* toggles control what triggers a send
|
||||
#
|
||||
# Three profiles controlled by DIGEST_PROFILE in Master.conf:
|
||||
# always — sends every run regardless of findings (schedule daily for daily digest)
|
||||
# smart — sends only if something worth reporting was found (intelligent filtering)
|
||||
# weekly — sends once per week on DIGEST_DAY regardless of schedule frequency
|
||||
# run daily, digest only fires on DIGEST_DAY (default Sunday)
|
||||
#
|
||||
# The cron schedule stays the same regardless of profile — just change DIGEST_PROFILE
|
||||
# in Master.conf to switch behavior. Run daily, profile controls when it actually notifies.
|
||||
# The cron schedule stays the same regardless of profile — change DIGEST_PROFILE in
|
||||
# master.conf to switch behaviour. No cron changes needed.
|
||||
#
|
||||
# Data sources (reads only — no writes):
|
||||
# /tmp/transcode_state.db — ramdisk symlink and usage
|
||||
# /tmp/container_watchdog_state.db — active container strikes
|
||||
# /tmp/system_watchdog_state.db — active system strikes
|
||||
# /boot/config/failover_state.db — current failover state
|
||||
# /boot/config/system_watchdog_failed.db — container skip list
|
||||
# /boot/config/bandwidth_history.db — recent transfer totals
|
||||
# SSL certs via openssl (live check) — days remaining per domain
|
||||
# ── DATA SOURCES (reads only) ─────────────────────────────────────────────────────────────────
|
||||
# FAILOVER_STATE_FILE — current failover state
|
||||
# SYS_WATCHDOG_FAILED_FILE — container skip list (manual intervention needed)
|
||||
# WATCHDOG_STATE_FILE — active container watchdog strikes
|
||||
# SYS_WATCHDOG_STATE_FILE — active system watchdog strikes
|
||||
# BANDWIDTH_LOG — yesterday's transfer totals
|
||||
# TRANSCODE_DAILY_LOG — weekly transcode statistics
|
||||
# CERT_MONITOR_DOMAINS — live SSL cert check via openssl
|
||||
# RAMDISK_PATH / TRANSCODE_LINK — current transcode location and usage
|
||||
#
|
||||
# All configuration in Master.conf under Health Digest section.
|
||||
# Supports --dry-run to generate report without sending notification.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases CERT_MONITOR_DOMAINS, RAMDISK_WARN_GB,
|
||||
# RAMDISK_SIZE, RAMDISK_LOW_GB and all other host-specific vars used in this report.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — report takes time, prevent duplicate runs
|
||||
# detect_hosts() — correct vars per host
|
||||
# validate_unraid_cmd — notify and openssl validated before use
|
||||
# Per-section guards — missing state file skipped cleanly
|
||||
# Silent smart profile — completely silent when nothing to report
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# DIGEST_PROFILE — always | smart | weekly
|
||||
# DIGEST_DAY — day name for weekly profile (e.g. Sunday)
|
||||
# DIGEST_SMART_ON_WATCHDOG — send on active watchdog strikes
|
||||
# DIGEST_SMART_ON_FAILOVER — send on non-NORMAL failover state
|
||||
# DIGEST_SMART_ON_CERT_WARN — send on cert warning
|
||||
# DIGEST_SMART_ON_BANDWIDTH — send on high bandwidth day
|
||||
# CERT_WARN_DAYS / CERT_CRIT_DAYS / CERT_TIMEOUT
|
||||
# BANDWIDTH_WARN_GB
|
||||
# TRANSCODE_DAILY_LOG
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# weekly_health_digest.sh — normal run
|
||||
# weekly_health_digest.sh --dry-run — generate report, no notification
|
||||
# weekly_health_digest.sh --log — verbose output
|
||||
# weekly_health_digest.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Report/monitor script — output is the point when sending
|
||||
SILENT_MODE=false
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
validate_unraid_cmd \
|
||||
"$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \
|
||||
"version" "OpenSSL" \
|
||||
"openssl" || warn "openssl not found — SSL cert checks will be skipped"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases all host-specific vars used in this report
|
||||
detect_hosts
|
||||
|
||||
success "Health Digest — profile: $DIGEST_PROFILE"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — report generated but no notification sent"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Profile check — should we send today?
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_DIGEST Profile: $DIGEST_PROFILE"
|
||||
echo "$ICON_DIGEST Digest day: $DIGEST_DAY"
|
||||
echo "$ICON_DIGEST Smart triggers: watchdog=$DIGEST_SMART_ON_WATCHDOG failover=$DIGEST_SMART_ON_FAILOVER cert=$DIGEST_SMART_ON_CERT_WARN bandwidth=$DIGEST_SMART_ON_BANDWIDTH"
|
||||
echo "$ICON_CERT Cert domains: ${CERT_MONITOR_DOMAINS[*]:-none}"
|
||||
echo "$ICON_BANDWIDTH Bandwidth warn: ${BANDWIDTH_WARN_GB}GB"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── Profile gate — should we send today? ──────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
SHOULD_SEND=false
|
||||
|
||||
case "$DIGEST_PROFILE" in
|
||||
@@ -56,229 +123,206 @@ case "$DIGEST_PROFILE" in
|
||||
TODAY_NAME=$(date '+%A')
|
||||
if [[ "$TODAY_NAME" == "$DIGEST_DAY" ]]; then
|
||||
SHOULD_SEND=true
|
||||
log "Profile: weekly — today is $DIGEST_DAY, will send"
|
||||
log "Profile: weekly — today is $DIGEST_DAY — will send"
|
||||
else
|
||||
info "Profile: weekly — today is $TODAY_NAME, digest day is $DIGEST_DAY — skipping"
|
||||
log "Profile: weekly — today is $TODAY_NAME, digest day is $DIGEST_DAY — skipping"
|
||||
exit 0
|
||||
fi
|
||||
;;
|
||||
smart)
|
||||
log "Profile: smart — will evaluate findings before deciding"
|
||||
SHOULD_SEND=false # determined after gathering data
|
||||
log "Profile: smart — evaluating findings before deciding"
|
||||
SHOULD_SEND=false
|
||||
;;
|
||||
*)
|
||||
warn "Unknown DIGEST_PROFILE: $DIGEST_PROFILE — defaulting to weekly behavior"
|
||||
warn "Unknown DIGEST_PROFILE: $DIGEST_PROFILE — defaulting to weekly"
|
||||
TODAY_NAME=$(date '+%A')
|
||||
[[ "$TODAY_NAME" == "$DIGEST_DAY" ]] && SHOULD_SEND=true
|
||||
;;
|
||||
esac
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# DATA GATHERING
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_DIGEST Gathering System Data ━━━"
|
||||
# ==============================================================================================
|
||||
# ── Data Gathering ────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
FINDINGS=() # notable but not critical
|
||||
ISSUES=() # need attention
|
||||
DIGEST_LINES=() # full report lines
|
||||
|
||||
FINDINGS=() # things worth noting
|
||||
ISSUES=() # things that need attention
|
||||
DIGEST_LINES=() # full report lines
|
||||
|
||||
# ── Failover State ──────────────────────────────────────────────────────────────────────────
|
||||
# ── Failover State ────────────────────────────────────────────────────────────────────────────
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
FAILOVER_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
DIGEST_LINES+=("$ICON_FAILOVER Failover: $FAILOVER_STATE (last change: ${FAILOVER_CHANGE:-unknown})")
|
||||
|
||||
if [[ "$FAILOVER_STATE" != "NORMAL" && -n "$FAILOVER_STATE" ]]; then
|
||||
ISSUES+=("Failover state: $FAILOVER_STATE")
|
||||
[[ "$DIGEST_SMART_ON_FAILOVER" == true ]] && SHOULD_SEND=true
|
||||
if [[ -n "$FAILOVER_STATE" ]]; then
|
||||
DIGEST_LINES+=("$ICON_FAILOVER Failover: $FAILOVER_STATE")
|
||||
if [[ "$FAILOVER_STATE" != "NORMAL" ]]; then
|
||||
ISSUES+=("Failover state: $FAILOVER_STATE")
|
||||
[[ "$DIGEST_SMART_ON_FAILOVER" == true ]] && SHOULD_SEND=true
|
||||
fi
|
||||
fi
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_FAILOVER Failover: state file not found")
|
||||
fi
|
||||
|
||||
# ── Transcode Ramdisk ───────────────────────────────────────────────────────────────────────
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
|
||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
|
||||
RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail | tail -1 | tr -d ' ')
|
||||
RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}")
|
||||
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
||||
DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET")
|
||||
|
||||
# Read weekly transcode stats from daily log if available
|
||||
TRANSCODE_DAILY_LOG="/boot/config/transcode_daily.db"
|
||||
if [[ -f "$TRANSCODE_DAILY_LOG" ]]; then
|
||||
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d')
|
||||
|
||||
# Peak ramdisk usage this week
|
||||
WEEK_PEAK=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||||
'$1 >= cutoff {if ($2 > max) max=$2} END {printf "%.2f", max+0}' \
|
||||
"$TRANSCODE_DAILY_LOG")
|
||||
|
||||
# Total flips this week
|
||||
WEEK_FLIPS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||||
'$1 >= cutoff {sum+=$3} END {print sum+0}' \
|
||||
"$TRANSCODE_DAILY_LOG")
|
||||
|
||||
# Total files cleaned this week
|
||||
WEEK_FILES=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||||
'$1 >= cutoff {sum+=$6} END {print sum+0}' \
|
||||
"$TRANSCODE_DAILY_LOG")
|
||||
|
||||
# Ram vs SSD session ratio
|
||||
WEEK_RAM=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||||
'$1 >= cutoff {sum+=$4} END {print sum+0}' \
|
||||
"$TRANSCODE_DAILY_LOG")
|
||||
WEEK_SSD=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||||
'$1 >= cutoff {sum+=$5} END {print sum+0}' \
|
||||
"$TRANSCODE_DAILY_LOG")
|
||||
|
||||
DIGEST_LINES+=("$ICON_RAM Transcode week: peak ${WEEK_PEAK}GB | flips: ${WEEK_FLIPS} | cleaned: ${WEEK_FILES} files")
|
||||
DIGEST_LINES+=("$ICON_RAM Session storage: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD")
|
||||
|
||||
# Warn if peak is getting close to threshold
|
||||
PEAK_INT=$(printf "%.0f" "$WEEK_PEAK")
|
||||
WARN_INT=$(printf "%.0f" "$RAMDISK_WARN_GB")
|
||||
if [[ "$PEAK_INT" -ge "$WARN_INT" ]]; then
|
||||
DIGEST_LINES+=("$ICON_WARN Transcode peak ${WEEK_PEAK}GB near threshold ${RAMDISK_WARN_GB}GB — consider increasing RAMDISK_SIZE")
|
||||
fi
|
||||
fi
|
||||
# ── Container Skip List ───────────────────────────────────────────────────────────────────────
|
||||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||||
SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ')
|
||||
DIGEST_LINES+=("$ICON_NOT_RUNNING Skip list: $SKIP_COUNT containers — $SKIP_LIST")
|
||||
ISSUES+=("Containers on skip list (manual intervention needed): $SKIP_LIST")
|
||||
SHOULD_SEND=true
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_RAM Transcodes: ramdisk not mounted")
|
||||
ISSUES+=("Ramdisk not mounted")
|
||||
DIGEST_LINES+=("$ICON_RUNNING Skip list: empty ✅")
|
||||
fi
|
||||
|
||||
# ── Container Watchdog Strikes ──────────────────────────────────────────────────────────────
|
||||
# ── Container Watchdog Strikes ────────────────────────────────────────────────────────────────
|
||||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | wc -l)
|
||||
ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -c "." || echo 0)
|
||||
if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then
|
||||
STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
|
||||
DIGEST_LINES+=("$ICON_WATCHDOG Container strikes: $ACTIVE_STRIKES active — $STRIKE_LIST")
|
||||
FINDINGS+=("Container watchdog: $ACTIVE_STRIKES active strikes")
|
||||
[[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_WATCHDOG Container watchdog: no active strikes")
|
||||
DIGEST_LINES+=("$ICON_WATCHDOG Container watchdog: no active strikes ✅")
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── System Watchdog Strikes ─────────────────────────────────────────────────────────────────
|
||||
# ── System Watchdog Strikes ───────────────────────────────────────────────────────────────────
|
||||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||||
SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | wc -l)
|
||||
SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -c "." || echo 0)
|
||||
if [[ "$SYS_STRIKES" -gt 0 ]]; then
|
||||
SYS_STRIKE_LIST=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
|
||||
DIGEST_LINES+=("$ICON_REBOOT_SMART System strikes: $SYS_STRIKES active — $SYS_STRIKE_LIST")
|
||||
FINDINGS+=("System watchdog: $SYS_STRIKES active strikes")
|
||||
[[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_REBOOT_SMART System watchdog: no active strikes")
|
||||
DIGEST_LINES+=("$ICON_REBOOT_SMART System watchdog: no active strikes ✅")
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Container Skip List ─────────────────────────────────────────────────────────────────────
|
||||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||||
SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ')
|
||||
DIGEST_LINES+=("$ICON_NOT_RUNNING Skip list: $SKIP_COUNT containers — $SKIP_LIST")
|
||||
ISSUES+=("Containers on skip list: $SKIP_LIST")
|
||||
SHOULD_SEND=true
|
||||
# ── Transcode Ramdisk ─────────────────────────────────────────────────────────────────────────
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
|
||||
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
||||
DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used → $SYMLINK_TARGET")
|
||||
|
||||
# Weekly transcode stats from TRANSCODE_DAILY_LOG
|
||||
if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]]; then
|
||||
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d')
|
||||
WEEK_PEAK=$(awk -F'|' -v c="$WEEK_CUTOFF" \
|
||||
'$1>=c{if($2>max)max=$2} END{printf "%.2f",max+0}' "$TRANSCODE_DAILY_LOG")
|
||||
WEEK_FLIPS=$(awk -F'|' -v c="$WEEK_CUTOFF" \
|
||||
'$1>=c{sum+=$3} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||||
WEEK_RAM=$(awk -F'|' -v c="$WEEK_CUTOFF" \
|
||||
'$1>=c{sum+=$4} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||||
WEEK_SSD=$(awk -F'|' -v c="$WEEK_CUTOFF" \
|
||||
'$1>=c{sum+=$5} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||||
WEEK_FILES=$(awk -F'|' -v c="$WEEK_CUTOFF" \
|
||||
'$1>=c{sum+=$6} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||||
|
||||
DIGEST_LINES+=("$ICON_RAM Transcode week: peak ${WEEK_PEAK}GB | flips: $WEEK_FLIPS | sessions: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD | cleaned: ${WEEK_FILES} files")
|
||||
|
||||
PEAK_INT=$(printf "%.0f" "$WEEK_PEAK")
|
||||
WARN_INT=$(printf "%.0f" "${RAMDISK_WARN_GB:-6.8}")
|
||||
if [[ "$PEAK_INT" -ge "$WARN_INT" ]]; then
|
||||
DIGEST_LINES+=("$ICON_WARN Peak ${WEEK_PEAK}GB near threshold ${RAMDISK_WARN_GB}GB — consider increasing HOST*_RAMDISK_SIZE")
|
||||
FINDINGS+=("Transcode ramdisk near threshold: ${WEEK_PEAK}GB / ${RAMDISK_WARN_GB}GB")
|
||||
fi
|
||||
fi
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_RUNNING Skip list: empty — all containers healthy")
|
||||
DIGEST_LINES+=("$ICON_RAM Transcodes: ramdisk not mounted")
|
||||
ISSUES+=("Ramdisk not mounted at $RAMDISK_PATH")
|
||||
SHOULD_SEND=true
|
||||
fi
|
||||
|
||||
# ── Bandwidth (yesterday's total) ───────────────────────────────────────────────────────────
|
||||
if [[ -f "$BANDWIDTH_LOG" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
|
||||
# ── Bandwidth ─────────────────────────────────────────────────────────────────────────────────
|
||||
# Updated for new log format: date|time|profile|duration|status|bytes|warn_flag
|
||||
if [[ -f "${BANDWIDTH_LOG:-}" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
|
||||
YESTERDAY=$(date -d "yesterday" '+%Y-%m-%d')
|
||||
YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$3} END{print sum+0}' "$BANDWIDTH_LOG")
|
||||
YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", $YESTERDAY_BYTES / 1073741824}")
|
||||
OVER_WARN=$(awk "BEGIN {print ($YESTERDAY_BYTES > $BANDWIDTH_WARN_GB * 1073741824) ? 1 : 0}")
|
||||
YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$6} END{print sum+0}' \
|
||||
"$BANDWIDTH_LOG")
|
||||
YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", ${YESTERDAY_BYTES:-0} / 1073741824}")
|
||||
YESTERDAY_LARGE=$(awk -F'|' -v d="$YESTERDAY" '$1==d && $7=="LARGE"' \
|
||||
"$BANDWIDTH_LOG" | wc -l)
|
||||
|
||||
if [[ "$OVER_WARN" == "1" ]]; then
|
||||
DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday's transfers: ${YESTERDAY_GB}GB ← exceeded ${BANDWIDTH_WARN_GB}GB threshold")
|
||||
if [[ "${YESTERDAY_LARGE:-0}" -gt 0 ]]; then
|
||||
DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday: ${YESTERDAY_GB}GB — $YESTERDAY_LARGE large transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB")
|
||||
FINDINGS+=("High bandwidth day: ${YESTERDAY_GB}GB transferred")
|
||||
[[ "$DIGEST_SMART_ON_BANDWIDTH" == true ]] && SHOULD_SEND=true
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday's transfers: ${YESTERDAY_GB}GB")
|
||||
DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday: ${YESTERDAY_GB}GB")
|
||||
fi
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_BANDWIDTH Bandwidth: no data yet")
|
||||
fi
|
||||
|
||||
# ── SSL Certificates ────────────────────────────────────────────────────────────────────────
|
||||
# ── SSL Certificates ──────────────────────────────────────────────────────────────────────────
|
||||
if [[ ${#CERT_MONITOR_DOMAINS[@]} -gt 0 ]] && command -v openssl >/dev/null 2>&1; then
|
||||
CERT_ISSUES=()
|
||||
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
||||
[[ -z "$domain" ]] && continue
|
||||
expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \
|
||||
expiry_str=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \
|
||||
-connect "${domain}:443" -servername "$domain" \
|
||||
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||||
|
||||
if [[ -n "$expiry_str" ]]; then
|
||||
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
|
||||
days_remaining=$(( (expiry_epoch - $(date +%s)) / 86400 ))
|
||||
|
||||
if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then
|
||||
if [[ "$days_remaining" -le "${CERT_CRIT_DAYS:-7}" ]]; then
|
||||
CERT_ISSUES+=("$domain: ${days_remaining}d CRITICAL")
|
||||
SHOULD_SEND=true
|
||||
elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then
|
||||
CERT_ISSUES+=("$domain: ${days_remaining}d WARNING")
|
||||
elif [[ "$days_remaining" -le "${CERT_WARN_DAYS:-30}" ]]; then
|
||||
CERT_ISSUES+=("$domain: ${days_remaining}d warning")
|
||||
[[ "$DIGEST_SMART_ON_CERT_WARN" == true ]] && SHOULD_SEND=true
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${#CERT_ISSUES[@]} -gt 0 ]]; then
|
||||
DIGEST_LINES+=("$ICON_CERT Certificates: ${CERT_ISSUES[*]}")
|
||||
FINDINGS+=("Cert issues: ${CERT_ISSUES[*]}")
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_CERT Certificates: all healthy")
|
||||
DIGEST_LINES+=("$ICON_CERT Certificates: all healthy ✅")
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Smart profile final decision ────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# ── Smart profile — exit silently if nothing to report ────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
if [[ "$DIGEST_PROFILE" == "smart" && "$SHOULD_SEND" == false ]]; then
|
||||
info "Profile: smart — no findings worth reporting — skipping notification"
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━"
|
||||
echo "$ICON_SUCCESS Everything looks healthy — no digest sent (smart profile)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
log "Profile: smart — no findings worth reporting — silent exit"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_DIGEST Build and Send Digest ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Build and Send Digest ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_DIGEST Health Digest — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
echo "━━━ $ICON_DIGEST Health Digest ━━━"
|
||||
|
||||
DIGEST_DATE=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
DIGEST_HOST=$(hostname)
|
||||
|
||||
# Build notification message
|
||||
NOTIFY_MSG="Health Digest — $DIGEST_HOST — $DIGEST_DATE"
|
||||
if [[ ${#ISSUES[@]} -gt 0 ]]; then
|
||||
NOTIFY_MSG+=" | Issues: ${ISSUES[*]}"
|
||||
fi
|
||||
if [[ ${#FINDINGS[@]} -gt 0 ]]; then
|
||||
NOTIFY_MSG+=" | Findings: ${FINDINGS[*]}"
|
||||
fi
|
||||
|
||||
# Print full digest to console
|
||||
for line in "${DIGEST_LINES[@]}"; do
|
||||
echo " $line"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━"
|
||||
echo "$ICON_DIGEST Profile: $DIGEST_PROFILE"
|
||||
echo "$ICON_TIME Generated: $DIGEST_DATE"
|
||||
echo "$ICON_ERROR Issues: ${#ISSUES[@]}"
|
||||
echo "$ICON_WARN Findings: ${#FINDINGS[@]}"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_DIGEST Profile: $DIGEST_PROFILE"
|
||||
echo "$ICON_ERROR Issues: ${#ISSUES[@]}"
|
||||
echo "$ICON_WARN Findings: ${#FINDINGS[@]}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Build notification message
|
||||
NOTIFY_MSG="Health Digest — $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
[[ ${#ISSUES[@]} -gt 0 ]] && NOTIFY_MSG+=" | Issues: ${ISSUES[*]}"
|
||||
[[ ${#FINDINGS[@]} -gt 0 ]] && NOTIFY_MSG+=" | Findings: ${FINDINGS[*]}"
|
||||
[[ ${#ISSUES[@]} -eq 0 && ${#FINDINGS[@]} -eq 0 ]] && NOTIFY_MSG+=" | All systems healthy"
|
||||
|
||||
NOTIFY_SEV="normal"
|
||||
[[ ${#ISSUES[@]} -gt 0 ]] && NOTIFY_SEV="warning"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — digest generated but not sent"
|
||||
elif [[ "$SHOULD_SEND" == true ]]; then
|
||||
notify "$NOTIFY_MSG" "Health Digest" "$([[ ${#ISSUES[@]} -gt 0 ]] && echo "warning" || echo "normal")"
|
||||
success "Digest sent"
|
||||
notify "$NOTIFY_MSG" "Health Digest" "$NOTIFY_SEV"
|
||||
log "Digest sent"
|
||||
fi
|
||||
Reference in New Issue
Block a user