#!/bin/bash # ============================================================================================== # ============================= Sunday Morning Coffee Report =================================== # ============================================================================================== # Weekly system overview — everything that happened this week in one clean read. # Designed to be read over coffee Sunday morning while the system is fully caught up # from the 2:30am maintenance window. # Schedule: 0 7 * * 0 (7am Sunday — after weekly_sync_maintenance.sh finishes at ~3am) # Maintenance window completes → 4 hours of fresh data → report ready ☕ # # ── SECTIONS ────────────────────────────────────────────────────────────────────────────────── # 🖥️ System — uptime, memory, boot drive, cache drive, reboots this week # 📀 Array — disk count, parity status, ZFS health, drive temps # 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split # 🎵 Media Activity — arr cleanup stats, arr recovery stats, queue depth # 🌐 Rsync — weekly transfer totals, per-share breakdown, failures # 🛡️ Watchdog — resource watchdog, system watchdog, docker watchdog, fallback state # 🔐 Security — SSL cert expiry per domain # 📊 Emby — weekly stream count, active now, top users # ⚙️ System Health — SMART summary, inotify, php-fpm, Docker, Gitea sync # ⚠️ Issues — anything requiring attention collected from above sections # # ── DATA SOURCES (reads only) ───────────────────────────────────────────────────────────────── # DATA_DIR stats files — arr cleanup, recovery, transcode, bandwidth history # /boot/config — fallback state, watchdog reboot log # /tmp — watchdog strike state files # /proc, /sys — system memory, uptime, inotify # /var/local/emhttp/ — unRAID array info # Emby API — session history, active streams # Arr APIs — current queue depth (SONARR_URL etc. from detect_hosts()) # tailscale — remote server reachability # openssl — live SSL cert check per domain # smartctl — drive SMART health # # ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── # detect_hosts() sets MY_ID and aliases all HOST*_ vars. # Report header and footer show MY_ID — clear which server's weekly report this is. # No manual HOST1/HOST2 comparisons — all via MY_ID/REMOTE_ID. # # ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # Root check — smartctl, docker, openssl need root # acquire_lock — prevents duplicate reports # detect_hosts() — correct vars per server # DOCKER_TIMEOUT — all docker calls protected # validate_unraid_cmd — notify validated before use # openssl check — security section skipped gracefully if not available # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # NOTIFY_UNRAID — send via unRAID notification system # DISCORD_WEBHOOK — send to Discord channel (MY_ID_ prefixed per host) # CERT_MONITOR_DOMAINS / CERT_WARN_DAYS / CERT_CRIT_DAYS # ZFS_REPORT_IGNORE_POOLS / SMART_IGNORE_DRIVES # All threshold vars read from master.conf at runtime # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # sunday_morning_coffee_report.sh — generate and send report # sunday_morning_coffee_report.sh --dry-run — generate without sending notification # sunday_morning_coffee_report.sh --status — show data file availability # sunday_morning_coffee_report.sh --log — verbose section output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" DOCKER_TIMEOUT=15 # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root — smartctl and docker require root" exit 1 fi validate_unraid_cmd \ "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ "" "" \ "unRAID notify script" || warn "unRAID notify script not found — unRAID notifications disabled" acquire_lock detect_hosts REPORT_DATE=$(date '+%A, %B %-d, %Y') WEEK_START=$(date -d "7 days ago" '+%Y-%m-%d') WEEK_EPOCH=$(date -d "$WEEK_START" +%s) TODAY=$(date '+%Y-%m-%d') NOW=$(date +%s) REPORT=() ISSUES=() FINDINGS=() # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY COFFEE REPORT STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "" echo "━━━ Data Files ━━━" for f in \ "$ARR_CLEANUP_STATS:arr cleanup stats" \ "$ARR_RECOVERY_STATS:arr recovery stats" \ "$TRANSCODE_DAILY_LOG:transcode daily log" \ "$BANDWIDTH_LOG:bandwidth log" \ "$TUNING_MONITOR_LOG:tuning monitor log" \ "$SYS_WATCHDOG_STATE_FILE:system watchdog state" \ "$SYS_WATCHDOG_REBOOT_LOG:watchdog reboot log" \ "$WATCHDOG_STATE_FILE:docker watchdog state" \ "$WATCHDOG_CONTAINER_RESTART_LOG:container restart log" \ "$FALLBACK_STATE_FILE:fallback state"; do path="${f%%:*}" label="${f##*:}" if [[ -f "$path" ]] && [[ -s "$path" ]]; then COUNT=$(wc -l < "$path" 2>/dev/null || echo "?") echo " $ICON_SUCCESS $label ($COUNT lines)" elif [[ -f "$path" ]]; then echo " $ICON_WARN $label (exists but empty)" else echo " $ICON_SKIP $label (not found)" fi done echo "" echo "━━━ Runtime Dependencies ━━━" command -v smartctl >/dev/null 2>&1 && echo " $ICON_SUCCESS smartctl" || \ echo " $ICON_SKIP smartctl (not installed)" command -v openssl >/dev/null 2>&1 && echo " $ICON_SUCCESS openssl" || \ echo " $ICON_SKIP openssl (security section disabled)" command -v docker >/dev/null 2>&1 && echo " $ICON_SUCCESS docker" || \ echo " $ICON_SKIP docker" command -v zpool >/dev/null 2>&1 && echo " $ICON_SUCCESS zpool" || \ echo " $ICON_SKIP zpool (ZFS section disabled)" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — report generated, notification not sent" # ============================================================================================== # ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── # ============================================================================================== section() { REPORT+=("") REPORT+=("$1") REPORT+=("$(printf '%.0s─' {1..50})") } line() { REPORT+=(" $1"); } issue() { ISSUES+=("$1"); REPORT+=(" ⚠️ $1"); } finding() { FINDINGS+=("$1"); REPORT+=(" ℹ️ $1"); } get_array() { eval "echo \"\${${1}[*]}\""; } format_bytes() { local bytes=$1 if (( bytes > 1073741824 )); then awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" elif (( bytes > 1048576 )); then awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" else echo "${bytes}B" fi } # Lock/uptime helpers — local copies needed in report context _get_lock_pid() { local f="$LOCK_DIR/${1}.lock" [[ -f "$f" ]] && { local c; c=$(cat "$f" 2>/dev/null); echo "${c%%:*}"; } } _get_lock_name() { local f="$LOCK_DIR/${1}.lock" [[ -f "$f" ]] && { local c; c=$(cat "$f" 2>/dev/null); echo "${c##*:}"; } } _is_running() { local pid name pid=$(_get_lock_pid "$1"); name=$(_get_lock_name "$1") [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$name" == "$1" ]] } _lock_age() { local f="$LOCK_DIR/${1}.lock" [[ -f "$f" ]] && echo $(( NOW - $(stat -c %Y "$f" 2>/dev/null || echo "$NOW") )) || echo 0 } _fmt_uptime() { local s=$1 d=$(($1/86400)) h=$((($1%86400)/3600)) m=$((($1%3600)/60)) (( d > 0 )) && echo "${d}d ${h}h ${m}m" || \ (( h > 0 )) && echo "${h}h ${m}m" || echo "${m}m" } # ============================================================================================== # ━━━ 🖥️ SYSTEM ━━━ # ============================================================================================== section "🖥️ SYSTEM — $MY_ID ($LOCAL_SERVER_NAME)" UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime) UPTIME_DAYS=$(( UPTIME_SECONDS / 86400 )) UPTIME_HOURS=$(( (UPTIME_SECONDS % 86400) / 3600 )) BOOT_TIME=$(date -d "@$(( NOW - UPTIME_SECONDS ))" '+%A %-d %b at %-I:%M%p') line "Uptime: ${UPTIME_DAYS}d ${UPTIME_HOURS}hr (up since $BOOT_TIME)" REBOOT_COUNT=0 if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then REBOOT_COUNT=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \ "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l) fi [[ "${REBOOT_COUNT:-0}" -gt 0 ]] && \ issue "Reboots this week: $REBOOT_COUNT (system watchdog triggered)" || \ line "Reboots this week: 0 ✅" MEM_TOTAL_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo) MEM_AVAIL_KB=$(awk '/MemAvailable/{print $2}' /proc/meminfo) MEM_USED_GB=$(awk "BEGIN {printf \"%.1f\", ($MEM_TOTAL_KB - $MEM_AVAIL_KB) / 1048576}") MEM_TOTAL_GB=$(awk "BEGIN {printf \"%.0f\", $MEM_TOTAL_KB / 1048576}") ARC_SIZE="n/a" if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then ARC_B=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) ARC_SIZE=$(awk "BEGIN {printf \"%.1f\", $ARC_B / 1073741824}") fi line "Memory: ${MEM_USED_GB}GB used / ${MEM_TOTAL_GB}GB total (ARC: ${ARC_SIZE}GB)" BOOT_PCT=$(df /boot --output=pcent 2>/dev/null | tail -1 | tr -d ' %') BOOT_USED=$(df /boot -h --output=used 2>/dev/null | tail -1 | tr -d ' ') BOOT_SIZE=$(df /boot -h --output=size 2>/dev/null | tail -1 | tr -d ' ') [[ "${BOOT_PCT:-0}" -ge 80 ]] && \ issue "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE}) — getting full" || \ line "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE})" CACHE_PCT=$(df /mnt/cache --output=pcent 2>/dev/null | tail -1 | tr -d ' %') CACHE_AVAIL=$(df /mnt/cache -h --output=avail 2>/dev/null | tail -1 | tr -d ' ') CACHE_SIZE=$(df /mnt/cache -h --output=size 2>/dev/null | tail -1 | tr -d ' ') [[ "${CACHE_PCT:-0}" -ge 85 ]] && \ issue "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})" || \ line "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})" # ============================================================================================== # ━━━ 📀 ARRAY ━━━ # ============================================================================================== section "📀 ARRAY" if [[ -f /var/local/emhttp/disks.ini ]]; then DISK_COUNT=$(grep -c '^\["disk[0-9]' /var/local/emhttp/disks.ini 2>/dev/null || echo "?") PARITY_COUNT=$(grep -c '^\["parity' /var/local/emhttp/disks.ini 2>/dev/null || echo "?") DISK_COUNT="${DISK_COUNT//[^0-9]/}"; DISK_COUNT="${DISK_COUNT:-?}" PARITY_COUNT="${PARITY_COUNT//[^0-9]/}"; PARITY_COUNT="${PARITY_COUNT:-?}" line "Array: ${DISK_COUNT} data disks + ${PARITY_COUNT} parity" else line "Array: disks.ini not found" fi PARITY_LOG="/boot/config/parity-checks.log" if [[ -f "$PARITY_LOG" ]]; then LAST_CHECK=$(tail -1 "$PARITY_LOG" 2>/dev/null) if [[ -n "$LAST_CHECK" ]]; then PARITY_DATE=$(echo "$LAST_CHECK" | cut -d'|' -f1 | xargs) PARITY_ERRORS=$(echo "$LAST_CHECK" | cut -d'|' -f4) PARITY_ACTION=$(echo "$LAST_CHECK" | cut -d'|' -f6) PARITY_DURATION=$(echo "$LAST_CHECK" | cut -d'|' -f2) PARITY_DUR_HR=$(( PARITY_DURATION / 3600 )) PARITY_DUR_MIN=$(( (PARITY_DURATION % 3600) / 60 )) if [[ "${PARITY_ERRORS:-0}" -gt 0 ]]; then issue "Parity: $PARITY_ERRORS errors on last check ($PARITY_DATE)" elif [[ "${PARITY_ERRORS:-0}" -lt 0 ]]; then CORRECTIONS=$(( PARITY_ERRORS * -1 )) line "Parity: OK — last: $PARITY_DATE ($PARITY_ACTION, ${PARITY_DUR_HR}h${PARITY_DUR_MIN}m, $CORRECTIONS correction(s)) ✅" else line "Parity: OK — last: $PARITY_DATE ($PARITY_ACTION, ${PARITY_DUR_HR}h${PARITY_DUR_MIN}m, 0 errors) ✅" fi RESYNC=$(grep "^mdResync=" /var/local/emhttp/var.ini 2>/dev/null | cut -d= -f2 | tr -d '"') [[ "$RESYNC" != "0" ]] && finding "Parity check currently in progress" fi else line "Parity: no check log found" fi if command -v zpool >/dev/null 2>&1; then while IFS=$'\t' read -r pool health; do [[ -z "$pool" ]] && continue SKIP=false for ignore in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do [[ "$pool" == "$ignore" ]] && SKIP=true && break done [[ "$SKIP" == true ]] && continue [[ "$health" == "ONLINE" ]] && \ line "ZFS $pool: ONLINE ✅" || \ issue "ZFS $pool: $health — check immediately" done < <(zpool list -H -o name,health 2>/dev/null) fi if command -v smartctl >/dev/null 2>&1; then declare -A DISK_TEMPS ALL_NORMAL=true get_unraid_temp_thresholds for disk in /dev/sd? /dev/nvme?; do [[ ! -e "$disk" ]] && continue DISK_NAME=$(basename "$disk") SKIP=false for ignore in "${SMART_IGNORE_DRIVES[@]}"; do [[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break done [[ "$SKIP" == true ]] && continue TEMP=$(smartctl -A "$disk" 2>/dev/null | awk ' /^190 / || /^194 / { print $10; exit } /Temperature_Celsius/ { print $10; exit } ') [[ -z "$TEMP" ]] && TEMP=$(smartctl -A "$disk" 2>/dev/null | \ awk '/^Temperature:/ { print $2; exit }') TEMP="${TEMP//[^0-9]/}" [[ -z "$TEMP" ]] && continue DISK_TEMPS["$DISK_NAME"]="$TEMP" if is_ssd "$disk"; then WARN_T="${UNRAID_SSD_HOT:-50}"; CRIT_T="${UNRAID_SSD_MAX:-60}" else WARN_T="${UNRAID_DISK_HOT:-45}"; CRIT_T="${UNRAID_DISK_MAX:-55}" fi if [[ "$TEMP" -ge "$CRIT_T" ]]; then issue "Drive $DISK_NAME: ${TEMP}°C — CRITICAL" ALL_NORMAL=false elif [[ "$TEMP" -ge "$WARN_T" ]]; then finding "Drive $DISK_NAME: ${TEMP}°C — warm" ALL_NORMAL=false fi done [[ "$ALL_NORMAL" == true ]] && line "Drive temps: all normal ✅" if [[ -f /var/local/emhttp/disks.ini ]] && [[ ${#DISK_TEMPS[@]} -gt 0 ]]; then DISK_SUMMARY="" CURRENT_DISK="" while IFS= read -r ini_line; do if echo "$ini_line" | grep -qE '^\["(disk[0-9]+|parity[0-9]?|cache)"\]'; then CURRENT_DISK=$(echo "$ini_line" | grep -o '"[^"]*"' | head -1 | tr -d '"') elif echo "$ini_line" | grep -q '^device='; then DEV=$(echo "$ini_line" | cut -d= -f2 | tr -d '"') TEMP="${DISK_TEMPS[$DEV]:-}" if [[ -n "$TEMP" ]]; then [[ -n "$DISK_SUMMARY" ]] && DISK_SUMMARY+=", " DISK_SUMMARY+="${CURRENT_DISK}(${DEV}):${TEMP}°C" fi fi done < /var/local/emhttp/disks.ini [[ -n "$DISK_SUMMARY" ]] && line "Temps: $DISK_SUMMARY" fi fi # ============================================================================================== # ━━━ 🎬 TRANSCODES ━━━ # ============================================================================================== section "🎬 TRANSCODES" if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ') RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail 2>/dev/null | tail -1 | tr -d ' ') RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}") RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}") SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null | xargs basename 2>/dev/null || echo "unknown") line "Ramdisk now: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET" else issue "Ramdisk not mounted at $RAMDISK_PATH" fi if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]] && [[ -s "$TRANSCODE_DAILY_LOG" ]]; then WEEK_PEAK=$(awk -F'|' -v c="$WEEK_START" '$1>=c {if($2>max)max=$2} END{printf "%.2f",max+0}' "$TRANSCODE_DAILY_LOG") WEEK_FLIPS=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$3} END{print sum+0}' "$TRANSCODE_DAILY_LOG") WEEK_RAM=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$4} END{print sum+0}' "$TRANSCODE_DAILY_LOG") WEEK_SSD=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$5} END{print sum+0}' "$TRANSCODE_DAILY_LOG") line "Week peak: ${WEEK_PEAK}GB | flips: ${WEEK_FLIPS}" line "Sessions: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD" PEAK_INT=$(printf "%.0f" "$WEEK_PEAK" 2>/dev/null || echo 0) WARN_INT=$(printf "%.0f" "${RAMDISK_WARN_GB:-6.8}" 2>/dev/null || echo 0) [[ "$PEAK_INT" -ge "$WARN_INT" ]] && \ finding "Transcode peak ${WEEK_PEAK}GB near threshold — consider increasing HOST*_RAMDISK_SIZE" [[ "${WEEK_FLIPS:-0}" -ge "${TRANSCODE_FLIP_WARN:-3}" ]] && \ finding "Transcode flips this week: $WEEK_FLIPS — monitor ramdisk headroom" fi # ============================================================================================== # ━━━ 🎵 MEDIA ACTIVITY ━━━ # ============================================================================================== section "🎵 MEDIA ACTIVITY" if [[ -f "${ARR_CLEANUP_STATS:-}" ]] && [[ -s "$ARR_CLEANUP_STATS" ]]; then for arr in lidarr sonarr radarr; do WEEK_ORPHANS=$(awk -F'|' -v c="$WEEK_START" -v a="$arr" \ '$1>=c && $2==a {sum+=$3} END{print sum+0}' "$ARR_CLEANUP_STATS") WEEK_BYTES=$(awk -F'|' -v c="$WEEK_START" -v a="$arr" \ '$1>=c && $2==a {sum+=$4} END{print sum+0}' "$ARR_CLEANUP_STATS") WEEK_TRACKED=$(awk -F'|' -v c="$WEEK_START" -v a="$arr" \ 'BEGIN{max=0} $1>=c && $2==a && $8+0>max {max=$8} END{print max+0}' "$ARR_CLEANUP_STATS") if [[ "${WEEK_ORPHANS:-0}" -gt 0 ]]; then FREED=$(format_bytes "${WEEK_BYTES:-0}") line "${arr^} cleanup: $WEEK_ORPHANS orphans removed ($FREED freed) | tracked: $WEEK_TRACKED files" else line "${arr^} cleanup: clean ✅ (tracked: $WEEK_TRACKED files)" fi done else line "Arr cleanup stats: no data yet (runs after first weekly cleanup)" fi if [[ -f "${ARR_RECOVERY_STATS:-}" ]] && [[ -s "$ARR_RECOVERY_STATS" ]]; then WEEK_ACTIONED=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$3} END{print sum+0}' "$ARR_RECOVERY_STATS") WEEK_RUNS=$(awk -F'|' -v c="$WEEK_START" '$1>=c {count++} END{print count+0}' "$ARR_RECOVERY_STATS") if [[ "${WEEK_ACTIONED:-0}" -gt 0 ]]; then line "Arr recovery: $WEEK_ACTIONED items auto-recovered across $WEEK_RUNS runs" else line "Arr recovery: no failed imports this week ✅" fi fi # Arr queue depth — uses detect_hosts() aliased SONARR_URL/RADARR_URL/LIDARR_URL for arr_entry in "Sonarr|${SONARR_URL:-}|${SONARR_API_KEY:-}|v3" \ "Radarr|${RADARR_URL:-}|${RADARR_API_KEY:-}|v3" \ "Lidarr|${LIDARR_URL:-}|${LIDARR_API_KEY:-}|v1"; do IFS='|' read -r name url key ver <<< "$arr_entry" [[ -z "$url" || -z "$key" ]] && continue QUEUE=$(curl -sf --max-time 5 -H "X-Api-Key: $key" \ "${url}/api/${ver}/queue?pageSize=1" 2>/dev/null | \ grep -o '"totalRecords":[0-9]*' | grep -o '[0-9]*' || echo "?") if [[ "$QUEUE" == "0" || -z "$QUEUE" ]]; then line "$name queue: empty ✅" elif [[ "$QUEUE" == "?" ]]; then finding "$name queue: API unavailable" else line "$name queue: $QUEUE items" fi done # ============================================================================================== # ━━━ 🌐 RSYNC ━━━ # ============================================================================================== section "🌐 RSYNC" # New bandwidth log format: date|time|profile|duration|status|bytes|warn_flag if [[ -f "${BANDWIDTH_LOG:-}" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then WEEK_TOTAL_BYTES=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$6} END{print sum+0}' "$BANDWIDTH_LOG") WEEK_SYNCS=$(awk -F'|' -v c="$WEEK_START" '$1>=c' "$BANDWIDTH_LOG" | wc -l) WEEK_FAILED=$(awk -F'|' -v c="$WEEK_START" '$1>=c && $5!="success"' "$BANDWIDTH_LOG" | wc -l) WEEK_LARGE=$(awk -F'|' -v c="$WEEK_START" '$1>=c && $7=="LARGE"' "$BANDWIDTH_LOG" | wc -l) WEEK_GB=$(awk "BEGIN {printf \"%.1f\", $WEEK_TOTAL_BYTES / 1073741824}") line "Total: ${WEEK_GB}GB across $WEEK_SYNCS syncs" [[ "${WEEK_FAILED:-0}" -gt 0 ]] && issue "Failed syncs this week: $WEEK_FAILED" || line "Sync failures: none ✅" [[ "${WEEK_LARGE:-0}" -gt 0 ]] && finding "Large transfers (>${BANDWIDTH_WARN_GB}GB): $WEEK_LARGE" # Top 3 profiles by transfer — profile is field $3 TOP_SHARES=$(awk -F'|' -v c="$WEEK_START" \ '$1>=c {bytes[$3]+=$6} END {for(s in bytes) print bytes[s], s}' \ "$BANDWIDTH_LOG" | sort -rn | head -3) if [[ -n "$TOP_SHARES" ]]; then while IFS=' ' read -r bytes profile; do [[ -z "$profile" ]] && continue line " → $profile: $(format_bytes "$bytes")" done <<< "$TOP_SHARES" fi else line "No bandwidth data yet" fi # ============================================================================================== # ━━━ 🛡️ WATCHDOG ━━━ # ============================================================================================== section "🛡️ WATCHDOG" # ── System Watchdog ─────────────────────────────────────────────────────────────────────────── line "⚙️ System Watchdog (cron via watchdog_orchestrator)" if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then _sw_last=$(stat -c %Y "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || echo 0) _sw_ago=$(( NOW - _sw_last )) if [[ "$_sw_ago" -lt 600 ]]; then line " ✅ Last run: $(_fmt_uptime "$_sw_ago") ago" else issue " Last run: $(_fmt_uptime "$_sw_ago") ago — watchdog_orchestrator may not be running" fi else issue " stability_watchdog has never run (state file missing)" fi if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then SYS_ACTIVE=$(grep -v ":0$\|^watchdog_cycle=" \ "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") if [[ -n "$SYS_ACTIVE" ]]; then while IFS=: read -r key count; do [[ -z "$key" ]] && continue issue " Strike: $key — $count/$SYS_WATCHDOG_STRIKE_LIMIT" done <<< "$SYS_ACTIVE" else line " ✅ Strikes: none" fi fi if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then WD_REBOOTS=$(awk -v c="$WEEK_EPOCH" '$1>=c' "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l) [[ "${WD_REBOOTS:-0}" -gt 0 ]] && \ issue " Watchdog reboots this week: $WD_REBOOTS" || \ line " ✅ Watchdog reboots this week: 0" fi if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE") SKIP_LIST=$(tr '\n' ' ' < "$SYS_WATCHDOG_FAILED_FILE") issue " Skip list ($SKIP_COUNT): $SKIP_LIST" else line " ✅ Skip list: empty" fi ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') MEM_AVAIL_GB=$(awk '/MemAvailable/{printf "%.1f",$2/1048576}' /proc/meminfo) MEM_TOTAL_GB_SYS=$(awk '/MemTotal/{printf "%.0f",$2/1048576}' /proc/meminfo) LOAD_NOW=$(awk '{print $1}' /proc/loadavg) ZOMBIE_NOW=$(ps aux 2>/dev/null | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0) ZOMBIE_NOW="${ZOMBIE_NOW//[^0-9]/}"; ZOMBIE_NOW="${ZOMBIE_NOW:-0}" ARC_NOW="n/a" [[ -f /proc/spl/kstat/zfs/arcstats ]] && \ ARC_NOW=$(awk '/^size /{printf "%.1f",$3/1073741824}' /proc/spl/kstat/zfs/arcstats) line " 📊 rootfs:${ROOTFS_PCT}% │ RAM:${MEM_AVAIL_GB}GB free/${MEM_TOTAL_GB_SYS}GB │ ARC:${ARC_NOW}GB │ load:${LOAD_NOW} │ zombies:${ZOMBIE_NOW}" REPORT+=("") # ── Docker Watchdog ─────────────────────────────────────────────────────────────────────────── line "🐳 Docker Watchdog (cron via watchdog_orchestrator)" if [[ -f "$WATCHDOG_STATE_FILE" ]]; then _dw_last=$(stat -c %Y "$WATCHDOG_STATE_FILE" 2>/dev/null || echo 0) _dw_ago=$(( NOW - _dw_last )) if [[ "$_dw_ago" -lt 600 ]]; then line " ✅ Last run: $(_fmt_uptime "$_dw_ago") ago" else issue " Last run: $(_fmt_uptime "$_dw_ago") ago — watchdog_orchestrator may not be running" fi else issue " docker_watchdog has never run (state file missing)" fi if [[ -f "$WATCHDOG_STATE_FILE" ]]; then DOCK_ACTIVE=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") if [[ -n "$DOCK_ACTIVE" ]]; then while IFS=: read -r key count; do [[ -z "$key" ]] && continue issue " Strike: $key — $count" done <<< "$DOCK_ACTIVE" else line " ✅ Container strikes: none" fi fi if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then WK_RESTARTS=$(awk -F'|' -v c="$WEEK_START" '$2>=c' \ "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l) if [[ "${WK_RESTARTS:-0}" -gt 0 ]]; then RESTARTED=$(awk -F'|' -v c="$WEEK_START" '$2>=c{print $1}' \ "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \ sort | uniq -c | sort -rn | head -5 | \ awk '{print $2"("$1")"}' | tr '\n' ' ') finding " Container restarts this week: $WK_RESTARTS — $RESTARTED" else line " ✅ Container restarts this week: none" fi fi # ── Resource Watchdog ───────────────────────────────────────────────────────────────────────── line "🎛️ Resource Watchdog" if [[ -f "$RW_STATE_FILE" ]]; then _rm_last=$(stat -c %Y "$RW_STATE_FILE" 2>/dev/null || echo 0) _rm_ago=$(( NOW - _rm_last )) _rm_level=$(grep "^current_level:" "$RW_STATE_FILE" 2>/dev/null | cut -d: -f2) _rm_level="${_rm_level:-0}" if [[ "$_rm_level" -gt 0 ]]; then issue " Pressure level ${_rm_level} active │ Last run: $(_fmt_uptime "$_rm_ago") ago" elif [[ "$_rm_ago" -lt 600 ]]; then line " ✅ Level 0 (normal) │ Last run: $(_fmt_uptime "$_rm_ago") ago" else issue " Last run: $(_fmt_uptime "$_rm_ago") ago — watchdog_orchestrator may not be running" fi else line " ℹ️ State file not found (resource_watchdog may not have run yet)" fi REPORT+=("") if command -v docker >/dev/null 2>&1; then RUNNING_NOW=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l) TOTAL_NOW=$( timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l) UNHEALTHY_NOW=$(timeout "$DOCKER_TIMEOUT" docker ps \ --filter health=unhealthy -q 2>/dev/null | wc -l) STOPPED_NAMES=() while IFS= read -r name; do [[ -z "$name" ]] && continue SKIP=false for ignore in "${WATCHDOG_SCAN_IGNORE[@]}"; do [[ "$name" == "$ignore" ]] && SKIP=true && break done [[ "$SKIP" == false ]] && STOPPED_NAMES+=("$name") done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \ --format "{{.Names}}" 2>/dev/null) line " 📦 $RUNNING_NOW/$TOTAL_NOW running │ unhealthy: $UNHEALTHY_NOW" [[ ${#STOPPED_NAMES[@]} -gt 0 ]] && \ issue " Stopped (unexpected): ${STOPPED_NAMES[*]}" || \ line " ✅ All containers running" fi REPORT+=("") # ── Failover ───────────────────────────────────────────────────────────────────────────────── line "🔀 Fallback" FO_PID=$(_get_lock_pid "fallback") if _is_running "fallback"; then FO_AGE=$(_lock_age "fallback") line " ✅ Running │ PID: $FO_PID │ Uptime: $(_fmt_uptime "$FO_AGE")" elif [[ "${FALLBACK_ENABLED:-true}" == false ]]; then line " ⏸️ Not running — FALLBACK_ENABLED=false" else issue "fallback NOT RUNNING" fi FO_STATE="UNKNOWN" FO_STATE_SECS=0 if [[ -f "$FALLBACK_STATE_FILE" ]]; then FO_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) FO_EPOCH=$(grep "^last_change_epoch=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) [[ -n "$FO_EPOCH" ]] && FO_STATE_SECS=$(( NOW - FO_EPOCH )) fi FO_DUR=$(_fmt_uptime "${FO_STATE_SECS:-0}") case "$FO_STATE" in NORMAL) line " ✅ State: NORMAL │ Duration: $FO_DUR" ;; FALLBACK) issue " State: FALLBACK — $REMOTE_SERVER_NAME down for $FO_DUR" FO_MINS=$(( FO_STATE_SECS / 60 )) # Use REMOTE_ID-based tier delay vars — no HOST1/HOST2 hardcoding T2_VAR="${REMOTE_ID}_TIER2_DELAY"; T3_VAR="${REMOTE_ID}_TIER3_DELAY"; T4_VAR="${REMOTE_ID}_TIER4_DELAY" T2="${!T2_VAR:-240}"; T3="${!T3_VAR:-720}"; T4="${!T4_VAR:-1440}" (( FO_MINS >= T2 )) && line " Tier 2: ✅ active" || line " Tier 2: ⏳ in $(( T2 - FO_MINS ))min" (( FO_MINS >= T3 )) && line " Tier 3: ✅ active" || line " Tier 3: ⏳ in $(( T3 - FO_MINS ))min" (( FO_MINS >= T4 )) && line " Tier 4: ✅ active" || line " Tier 4: ⏳ in $(( T4 - FO_MINS ))min" ;; NO_INTERNET) issue " State: NO_INTERNET — DDNS stopped │ Duration: $FO_DUR" ;; DARK) issue " State: DARK — remote down AND no internet │ Duration: $FO_DUR" ;; *) finding " State: ${FO_STATE:-unknown}" ;; esac if command -v tailscale >/dev/null 2>&1; then REMOTE_TS_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null) if [[ -n "$REMOTE_TS_IP" ]]; then ping -c 1 -W 2 "$REMOTE_TS_IP" >/dev/null 2>&1 && \ line " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_TS_IP — reachable ✅" || \ issue " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_TS_IP — not responding" else issue " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME) not visible on Tailscale" fi fi # ============================================================================================== # ━━━ 🔗 MESH ━━━ # ============================================================================================== section "🔗 MESH" _ALL_HOST_IDS=() for _h in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do [[ -n "${!_h:-}" ]] && _ALL_HOST_IDS+=("$_h") done # ── Members ── if [[ ${#_ALL_HOST_IDS[@]} -eq 0 ]]; then line "No hosts defined" else line "Members:" for _h in "${_ALL_HOST_IDS[@]}"; do _server="${!_h}" _owner_var="${_h}_OWNER"; _owner="${!_owner_var:-unknown}" _email_var="${_h}_OWNER_EMAIL"; _email="${!_email_var:-(not set)}" line " $_h $_server / $_owner / $_email" done fi # ── Protected Services ── _coverage_found=false for _covered in "${_ALL_HOST_IDS[@]}"; do _covered_owner_var="${_covered}_OWNER"; _covered_owner="${!_covered_owner_var:-$_covered}" _covered_email_var="${_covered}_OWNER_EMAIL"; _covered_email="${!_covered_email_var:-}" declare -A _tier_containers=() declare -A _tier_delays=() _covered_by="" _any_tiers=false for _covering in "${_ALL_HOST_IDS[@]}"; do [[ "$_covering" == "$_covered" ]] && continue for _tier in 1 2 3 4; do _containers=$(get_array "FALLBACK_${_covering}_COVERS_${_covered}_TIER${_tier}") [[ -z "$_containers" ]] && continue _any_tiers=true _tier_containers[$_tier]="$_containers" _delay_var="${_covered}_TIER${_tier}_DELAY" _tier_delays[$_tier]="${!_delay_var:-0}" done if [[ "$_any_tiers" == true ]]; then _cov_owner_var="${_covering}_OWNER"; _cov_owner="${!_cov_owner_var:-$_covering}" _covered_by="$_covering ($_cov_owner)" fi done [[ "$_any_tiers" == false ]] && { unset _tier_containers _tier_delays; declare -A _tier_containers=() _tier_delays=(); continue; } _coverage_found=true _hdr="$_covered_owner" [[ -n "$_covered_email" ]] && _hdr+=" — $_covered_email" line "Protected: $_hdr" for _tier in 1 2 3 4; do [[ -z "${_tier_containers[$_tier]:-}" ]] && continue _d="${_tier_delays[$_tier]:-0}" if (( _d == 0 )); then _dlabel="immediate" elif (( _d >= 1440 )); then _dlabel="$(( _d / 1440 ))d" elif (( _d >= 60 )); then _dlabel="$(( _d / 60 ))hr" else _dlabel="${_d}min" fi line " Tier $_tier (${_dlabel}): ${_tier_containers[$_tier]// /, }" done [[ -n "$_covered_by" ]] && line " Covered by: $_covered_by" unset _tier_containers _tier_delays declare -A _tier_containers=() _tier_delays=() done [[ "$_coverage_found" == false ]] && line "No fallback coverage configured" # ── Partnership ── if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then _po_host="${PARTNERSHIP_OWNER_HOST:-HOST1}" _po_server="${!_po_host:-unknown}" _po_name_var="${_po_host}_OWNER"; _po_name="${!_po_name_var:-unknown}" line "Partnership: enabled — owner $_po_host ($_po_server / $_po_name), sync every ${PARTNERSHIP_SYNC_INTERVAL:-15}min" else line "Partnership: disabled" fi # ============================================================================================== # ━━━ 🔐 SECURITY ━━━ # ============================================================================================== section "🔐 SECURITY" if ! command -v openssl >/dev/null 2>&1; then line "SSL certs: openssl not available — skipping" elif [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then line "SSL certs: no domains configured (CERT_MONITOR_DOMAINS empty)" else for domain in "${CERT_MONITOR_DOMAINS[@]}"; do [[ -z "$domain" ]] && continue EXPIRY=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \ -connect "${domain}:443" -servername "$domain" 2>/dev/null | \ openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) if [[ -z "$EXPIRY" ]]; then issue "$domain: could not check certificate" continue fi EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s 2>/dev/null || echo 0) DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW) / 86400 )) if [[ "$DAYS_LEFT" -le "${CERT_CRIT_DAYS:-7}" ]]; then issue "$domain: ${DAYS_LEFT} days remaining — CRITICAL, renew now" elif [[ "$DAYS_LEFT" -le "${CERT_WARN_DAYS:-30}" ]]; then finding "$domain: ${DAYS_LEFT} days remaining — renew soon" else line "$domain: ${DAYS_LEFT} days remaining ✅" fi done fi # ============================================================================================== # ━━━ 📊 EMBY ━━━ # ============================================================================================== section "📊 EMBY" # detect_hosts() already aliased EMBY_URL and EMBY_API_KEY if [[ -z "${EMBY_API_KEY:-}" ]]; then line "Emby: API key not configured" elif [[ "${EMBY_API_KEY:-}" == *"your-"* ]]; then line "Emby: placeholder API key — configure HOST*_EMBY_API_KEY" else EMBY_SYSTEM=$(curl -sf --max-time 5 \ -H "X-Emby-Token: $EMBY_API_KEY" \ "${EMBY_URL}/System/Info" 2>/dev/null) if [[ -z "$EMBY_SYSTEM" ]]; then line "Emby: API unavailable — check if Emby is running" else EMBY_VERSION=$(echo "$EMBY_SYSTEM" | grep -o '"Version":"[^"]*"' | cut -d'"' -f4) line "Emby: v${EMBY_VERSION:-unknown} — reachable ✅" SESSIONS=$(curl -sf --max-time 5 \ -H "X-Emby-Token: $EMBY_API_KEY" \ "${EMBY_URL}/Sessions" 2>/dev/null) ACTIVE_COUNT=$(echo "$SESSIONS" | grep -o '"NowPlayingItem"' | wc -l) ACTIVE_COUNT="${ACTIVE_COUNT//[^0-9]/}"; ACTIVE_COUNT="${ACTIVE_COUNT:-0}" line "Active streams now: $ACTIVE_COUNT" ACTIVITY_LOG=$(curl -sf --max-time 10 \ -H "X-Emby-Token: $EMBY_API_KEY" \ "${EMBY_URL}/user_usage_stats/user_activity?days=7" 2>/dev/null) if [[ -n "$ACTIVITY_LOG" ]] && echo "$ACTIVITY_LOG" | grep -q "user_name"; then TOTAL_PLAYS=$(echo "$ACTIVITY_LOG" | \ grep -o '"total_plays":[0-9]*' | \ awk -F: '{sum+=$2} END {print sum+0}') line "Streams this week: $TOTAL_PLAYS total plays" echo "$ACTIVITY_LOG" | \ grep -o '"user_name":"[^"]*","total_plays":[0-9]*' | \ awk -F'"' '{name=$4; plays=$NF; gsub(/.*:/,"",plays); print plays, name}' | \ sort -rn | head -3 | \ while read -r plays name; do line " → $name: $plays plays" done else line "Weekly stats: user_usage_stats plugin not available" fi fi fi # ============================================================================================== # ━━━ ⚙️ SYSTEM HEALTH ━━━ # ============================================================================================== section "⚙️ SYSTEM HEALTH" if [[ -f "${TUNING_MONITOR_LOG:-}" ]] && [[ -s "$TUNING_MONITOR_LOG" ]]; then INOTIFY_PEAK=$(awk -F'|' -v c="$WEEK_START" \ '$1>=c {if($3+0>max) max=$3+0} END{print max+0}' "$TUNING_MONITOR_LOG") INOTIFY_AVG=$(awk -F'|' -v c="$WEEK_START" \ '$1>=c {sum+=$3;cnt++} END{if(cnt>0)printf "%.0f",sum/cnt;else print 0}' "$TUNING_MONITOR_LOG") INOTIFY_WARNS=$(awk -F'|' -v c="$WEEK_START" \ '$1>=c && $6==1 {cnt++} END{print cnt+0}' "$TUNING_MONITOR_LOG") INOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l) INOW="${INOW//[^0-9]/}"; INOW="${INOW:-0}" ILIM=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024) IPCT=$(( INOW * 100 / ILIM )) [[ "${INOTIFY_WARNS:-0}" -gt 0 ]] && \ issue "inotify: ${INOW}/${ILIM} now (${IPCT}%) | week peak:$INOTIFY_PEAK avg:$INOTIFY_AVG | ⚠️ $INOTIFY_WARNS warning(s)" || \ line "inotify: ${INOW}/${ILIM} now (${IPCT}%) | week peak:$INOTIFY_PEAK avg:$INOTIFY_AVG ✅" PHPFPM_PEAK=$(awk -F'|' -v c="$WEEK_START" \ '$1>=c {if($7+0>max) max=$7+0} END{print max+0}' "$TUNING_MONITOR_LOG") PHPFPM_AVG=$(awk -F'|' -v c="$WEEK_START" \ '$1>=c {sum+=$7;cnt++} END{if(cnt>0)printf "%.0f",sum/cnt;else print 0}' "$TUNING_MONITOR_LOG") PHPFPM_WARNS=$(awk -F'|' -v c="$WEEK_START" \ '$1>=c && $10==1 {cnt++} END{print cnt+0}' "$TUNING_MONITOR_LOG") PNOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0) PNOW="${PNOW//[^0-9]/}"; PNOW="${PNOW:-0}" PMAX="${PHP_MAX_CHILDREN:-250}" [[ "${PHPFPM_WARNS:-0}" -gt 0 ]] && \ issue "php-fpm: ${PNOW}/${PMAX} workers now | week peak:$PHPFPM_PEAK avg:$PHPFPM_AVG | ⚠️ $PHPFPM_WARNS warning(s)" || \ line "php-fpm: ${PNOW}/${PMAX} workers now | week peak:$PHPFPM_PEAK avg:$PHPFPM_AVG ✅" else INOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l) INOW="${INOW//[^0-9]/}"; INOW="${INOW:-0}" ILIM=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024) PNOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0) PNOW="${PNOW//[^0-9]/}"; PNOW="${PNOW:-0}" line "inotify: ${INOW}/${ILIM} — no weekly data yet" line "php-fpm: ${PNOW}/${PHP_MAX_CHILDREN:-250} workers — no weekly data yet" fi if command -v smartctl >/dev/null 2>&1; then SMART_ISSUES=0 for disk in /dev/sd? /dev/nvme?; do [[ ! -e "$disk" ]] && continue DISK_NAME=$(basename "$disk") SKIP=false for ignore in "${SMART_IGNORE_DRIVES[@]}"; do [[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break done [[ "$SKIP" == true ]] && continue HEALTH=$(smartctl -H "$disk" 2>/dev/null | grep "SMART overall-health" | awk '{print $NF}') if [[ "$HEALTH" != "PASSED" ]] && [[ -n "$HEALTH" ]]; then issue "SMART $DISK_NAME: $HEALTH — check immediately" (( SMART_ISSUES++ )) fi done [[ "$SMART_ISSUES" -eq 0 ]] && line "SMART: all drives PASSED ✅" fi if command -v git >/dev/null 2>&1 && [[ -d "${TARGET_DIR:-}/.git" ]]; then CURRENT_COMMIT=$(git -C "$TARGET_DIR" rev-parse --short HEAD 2>/dev/null || echo "unknown") LAST_PULL=$(git -C "$TARGET_DIR" log -1 --format="%ar" 2>/dev/null || echo "unknown") line "Gitea: commit $CURRENT_COMMIT (pulled $LAST_PULL)" fi # ============================================================================================== # ━━━ ⚠️ ISSUES REQUIRING ATTENTION ━━━ # ============================================================================================== if [[ ${#ISSUES[@]} -gt 0 ]]; then REPORT+=("") REPORT+=("⚠️ ISSUES REQUIRING ATTENTION") REPORT+=("$(printf '%.0s─' {1..50})") for issue_line in "${ISSUES[@]}"; do REPORT+=(" ❌ $issue_line") done fi if [[ ${#FINDINGS[@]} -gt 0 ]]; then REPORT+=("") REPORT+=("ℹ️ NOTABLE") REPORT+=("$(printf '%.0s─' {1..50})") for finding_line in "${FINDINGS[@]}"; do REPORT+=(" → $finding_line") done fi # ============================================================================================== # ━━━ Footer ━━━ # ============================================================================================== REPORT+=("") if [[ ${#ISSUES[@]} -eq 0 ]]; then REPORT+=("✅ All systems healthy — enjoy your Sunday ☕") else REPORT+=("⚠️ ${#ISSUES[@]} issue(s) need attention") fi REPORT+=("$MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S')") REPORT+=("$(printf '%.0s━' {1..50})") # ============================================================================================== # ━━━ Output and Send ━━━ # ============================================================================================== HEADER="☕ SUNDAY MORNING COFFEE REPORT — $REPORT_DATE" DIVIDER="$(printf '%.0s━' {1..50})" BODY=$(printf '%s\n' "$HEADER" "${REPORT[@]}") echo "" echo "$DIVIDER" echo "$HEADER" echo "$DIVIDER" for report_line in "${REPORT[@]}"; do echo "$report_line" done echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — notification not sent" else if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then NOTIFY_SCRIPT="/usr/local/emhttp/plugins/dynamix/scripts/notify" if [[ -x "$NOTIFY_SCRIPT" ]]; then "$NOTIFY_SCRIPT" -s "☕ Weekly Report — $MY_ID" -d "$BODY" -i "normal" 2>/dev/null echo "unRAID notification sent" fi fi if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then # Escape body for JSON ESCAPED_BODY=$(echo "$BODY" | python3 -c \ 'import sys,json; print(json.dumps(sys.stdin.read()))' 2>/dev/null || \ echo "\"$BODY\"") PAYLOAD="{\"content\": ${ESCAPED_BODY}}" curl -sf -H "Content-Type: application/json" \ -d "$PAYLOAD" "$DISCORD_WEBHOOK" >/dev/null 2>&1 && \ echo "Discord notification sent" || \ warn "Discord notification failed" fi echo "Report generated — $MY_ID — ${#ISSUES[@]} issue(s) ${#FINDINGS[@]} finding(s)" fi