#!/bin/bash # ----------------------------------------------------------------------------------------------- # --------------------------------- Health Digest ---------------------------------------------- # ----------------------------------------------------------------------------------------------- # Aggregates system health data from across the ecosystem into a single digest report. # Reads existing state files — no new writes to flash drive. # # Three profiles controlled by DIGEST_PROFILE in Master.conf: # always — sends every run regardless of findings (schedule daily for daily digest) # smart — sends only if something worth reporting was found (intelligent filtering) # weekly — sends once per week on DIGEST_DAY regardless of schedule frequency # # The cron schedule stays the same regardless of profile — just change DIGEST_PROFILE # in Master.conf to switch behavior. Run daily, profile controls when it actually notifies. # # Data sources (reads only — no writes): # /tmp/transcode_state.db — ramdisk symlink and usage # /tmp/container_watchdog_state.db — active container strikes # /tmp/system_watchdog_state.db — active system strikes # /boot/config/failover_state.db — current failover state # /boot/config/system_watchdog_failed.db — container skip list # /boot/config/bandwidth_history.db — recent transfer totals # SSL certs via openssl (live check) — days remaining per domain # # All configuration in Master.conf under Health Digest section. # Supports --dry-run to generate report without sending notification. # ----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" parse_args "$@" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_GEAR Setup ━━━" success "Health Digest — profile: $DIGEST_PROFILE" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — report generated but no notification sent" # ----------------------------------------------------------------------------------------------- # Profile check — should we send today? # ----------------------------------------------------------------------------------------------- SHOULD_SEND=false case "$DIGEST_PROFILE" in always) SHOULD_SEND=true log "Profile: always — will send" ;; weekly) TODAY_NAME=$(date '+%A') if [[ "$TODAY_NAME" == "$DIGEST_DAY" ]]; then SHOULD_SEND=true log "Profile: weekly — today is $DIGEST_DAY, will send" else info "Profile: weekly — today is $TODAY_NAME, digest day is $DIGEST_DAY — skipping" exit 0 fi ;; smart) log "Profile: smart — will evaluate findings before deciding" SHOULD_SEND=false # determined after gathering data ;; *) warn "Unknown DIGEST_PROFILE: $DIGEST_PROFILE — defaulting to weekly behavior" TODAY_NAME=$(date '+%A') [[ "$TODAY_NAME" == "$DIGEST_DAY" ]] && SHOULD_SEND=true ;; esac # ----------------------------------------------------------------------------------------------- # DATA GATHERING # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_DIGEST Gathering System Data ━━━" FINDINGS=() # things worth noting ISSUES=() # things that need attention DIGEST_LINES=() # full report lines # ── Failover State ────────────────────────────────────────────────────────────────────────── if [[ -f "$FAILOVER_STATE_FILE" ]]; then FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) FAILOVER_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) DIGEST_LINES+=("$ICON_FAILOVER Failover: $FAILOVER_STATE (last change: ${FAILOVER_CHANGE:-unknown})") if [[ "$FAILOVER_STATE" != "NORMAL" && -n "$FAILOVER_STATE" ]]; then ISSUES+=("Failover state: $FAILOVER_STATE") [[ "$DIGEST_SMART_ON_FAILOVER" == true ]] && SHOULD_SEND=true fi else DIGEST_LINES+=("$ICON_FAILOVER Failover: state file not found") fi # ── Transcode Ramdisk ─────────────────────────────────────────────────────────────────────── if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ') RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}") RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail | tail -1 | tr -d ' ') RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}") SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown") DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET") else DIGEST_LINES+=("$ICON_RAM Transcodes: ramdisk not mounted") ISSUES+=("Ramdisk not mounted") fi # ── Container Watchdog Strikes ────────────────────────────────────────────────────────────── if [[ -f "$WATCHDOG_STATE_FILE" ]]; then ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | wc -l) if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ') DIGEST_LINES+=("$ICON_WATCHDOG Container strikes: $ACTIVE_STRIKES active — $STRIKE_LIST") FINDINGS+=("Container watchdog: $ACTIVE_STRIKES active strikes") [[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true else DIGEST_LINES+=("$ICON_WATCHDOG Container watchdog: no active strikes") fi fi # ── System Watchdog Strikes ───────────────────────────────────────────────────────────────── if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | wc -l) if [[ "$SYS_STRIKES" -gt 0 ]]; then SYS_STRIKE_LIST=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ') DIGEST_LINES+=("$ICON_REBOOT_SMART System strikes: $SYS_STRIKES active — $SYS_STRIKE_LIST") FINDINGS+=("System watchdog: $SYS_STRIKES active strikes") [[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true else DIGEST_LINES+=("$ICON_REBOOT_SMART System watchdog: no active strikes") fi fi # ── Container Skip List ───────────────────────────────────────────────────────────────────── if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE") SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ') DIGEST_LINES+=("$ICON_NOT_RUNNING Skip list: $SKIP_COUNT containers — $SKIP_LIST") ISSUES+=("Containers on skip list: $SKIP_LIST") SHOULD_SEND=true else DIGEST_LINES+=("$ICON_RUNNING Skip list: empty — all containers healthy") fi # ── Bandwidth (yesterday's total) ─────────────────────────────────────────────────────────── if [[ -f "$BANDWIDTH_LOG" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then YESTERDAY=$(date -d "yesterday" '+%Y-%m-%d') YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$3} END{print sum+0}' "$BANDWIDTH_LOG") YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", $YESTERDAY_BYTES / 1073741824}") OVER_WARN=$(awk "BEGIN {print ($YESTERDAY_BYTES > $BANDWIDTH_WARN_GB * 1073741824) ? 1 : 0}") if [[ "$OVER_WARN" == "1" ]]; then DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday's transfers: ${YESTERDAY_GB}GB ← exceeded ${BANDWIDTH_WARN_GB}GB threshold") FINDINGS+=("High bandwidth day: ${YESTERDAY_GB}GB transferred") [[ "$DIGEST_SMART_ON_BANDWIDTH" == true ]] && SHOULD_SEND=true else DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday's transfers: ${YESTERDAY_GB}GB") fi else DIGEST_LINES+=("$ICON_BANDWIDTH Bandwidth: no data yet") fi # ── SSL Certificates ──────────────────────────────────────────────────────────────────────── if [[ ${#CERT_MONITOR_DOMAINS[@]} -gt 0 ]] && command -v openssl >/dev/null 2>&1; then CERT_ISSUES=() for domain in "${CERT_MONITOR_DOMAINS[@]}"; do [[ -z "$domain" ]] && continue expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \ -connect "${domain}:443" -servername "$domain" \ 2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) if [[ -n "$expiry_str" ]]; then expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null) days_remaining=$(( (expiry_epoch - $(date +%s)) / 86400 )) if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then CERT_ISSUES+=("$domain: ${days_remaining}d CRITICAL") SHOULD_SEND=true elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then CERT_ISSUES+=("$domain: ${days_remaining}d WARNING") [[ "$DIGEST_SMART_ON_CERT_WARN" == true ]] && SHOULD_SEND=true fi fi done if [[ ${#CERT_ISSUES[@]} -gt 0 ]]; then DIGEST_LINES+=("$ICON_CERT Certificates: ${CERT_ISSUES[*]}") FINDINGS+=("Cert issues: ${CERT_ISSUES[*]}") else DIGEST_LINES+=("$ICON_CERT Certificates: all healthy") fi fi # ── Smart profile final decision ──────────────────────────────────────────────────────────── if [[ "$DIGEST_PROFILE" == "smart" && "$SHOULD_SEND" == false ]]; then info "Profile: smart — no findings worth reporting — skipping notification" echo "" echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━" echo "$ICON_SUCCESS Everything looks healthy — no digest sent (smart profile)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_DIGEST Build and Send Digest ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_DIGEST Health Digest ━━━" DIGEST_DATE=$(date '+%Y-%m-%d %H:%M:%S') DIGEST_HOST=$(hostname) # Build notification message NOTIFY_MSG="Health Digest — $DIGEST_HOST — $DIGEST_DATE" if [[ ${#ISSUES[@]} -gt 0 ]]; then NOTIFY_MSG+=" | Issues: ${ISSUES[*]}" fi if [[ ${#FINDINGS[@]} -gt 0 ]]; then NOTIFY_MSG+=" | Findings: ${FINDINGS[*]}" fi # Print full digest to console for line in "${DIGEST_LINES[@]}"; do echo " $line" done echo "" echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━" echo "$ICON_DIGEST Profile: $DIGEST_PROFILE" echo "$ICON_TIME Generated: $DIGEST_DATE" echo "$ICON_ERROR Issues: ${#ISSUES[@]}" echo "$ICON_WARN Findings: ${#FINDINGS[@]}" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — digest generated but not sent" elif [[ "$SHOULD_SEND" == true ]]; then notify "$NOTIFY_MSG" "Health Digest" "$([[ ${#ISSUES[@]} -gt 0 ]] && echo "warning" || echo "normal")" success "Digest sent" fi