927 lines
37 KiB
Bash
927 lines
37 KiB
Bash
#!/bin/bash
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ----------------------------- Sunday Morning Coffee Report -----------------------------------
|
||
# -----------------------------------------------------------------------------------------------
|
||
# Weekly system overview — everything that happened this week in one clean read.
|
||
# Designed to be read over coffee Sunday morning while the system is fully caught up
|
||
# from the 2:30am maintenance window.
|
||
# Schedule: 0 7 * * 0 (7am Sunday — after weekly_sync_maintenance.sh finishes at ~3am)
|
||
# Maintenance window completes → 4 hours of fresh data → report ready ☕
|
||
#
|
||
# Sections:
|
||
# 🖥️ System — uptime, memory, boot drive, cache drive, reboots
|
||
# 📀 Array — disk count, parity status, ZFS health, drive temps
|
||
# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split
|
||
# 🎵 Media Activity — arr cleanup stats, arr recovery stats, library health
|
||
# 🌐 Rsync — weekly transfer totals, per-share breakdown
|
||
# 🛡️ Watchdog — system watchdog, docker watchdog, failover (all continuous loops)
|
||
# strikes, skip list, restarts, system snapshot, container overview
|
||
# 🔐 Security — SSL cert expiry per domain
|
||
# 📊 Emby — weekly stream count, top users, top content
|
||
# ⚙️ Health — SMART summary, docker container count, Gitea sync status
|
||
# ⚠️ Issues — anything requiring attention collected above
|
||
#
|
||
# Data sources (reads only — no writes except the notification):
|
||
# DATA_DIR stats files — arr cleanup, recovery, transcode, bandwidth history
|
||
# /boot/config — failover state, watchdog reboot log
|
||
# /tmp — watchdog strike state files
|
||
# /proc, /sys — system memory, uptime
|
||
# /var/local/emhttp/ — unRAID array info
|
||
# Emby API — session history
|
||
# Arr APIs — current queue depth
|
||
# tailscale — network status
|
||
# openssl — live SSL cert check
|
||
# smartctl — drive health
|
||
#
|
||
# All configuration in Master.conf.
|
||
# Supports --dry-run to preview report without sending notification.
|
||
# -----------------------------------------------------------------------------------------------
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../Master.conf"
|
||
source "$SCRIPT_DIR/../common.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ Setup ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
acquire_lock
|
||
|
||
detect_hosts
|
||
|
||
REPORT_DATE=$(date '+%A, %B %-d, %Y')
|
||
WEEK_START=$(date -d "7 days ago" '+%Y-%m-%d')
|
||
TODAY=$(date '+%Y-%m-%d')
|
||
NOW=$(date +%s)
|
||
|
||
REPORT=() # all report lines
|
||
ISSUES=() # items needing attention
|
||
FINDINGS=() # notable but not critical
|
||
|
||
section() {
|
||
REPORT+=("")
|
||
REPORT+=("$1")
|
||
REPORT+=("$(printf '%.0s─' {1..50})")
|
||
}
|
||
|
||
line() {
|
||
REPORT+=(" $1")
|
||
}
|
||
|
||
issue() {
|
||
ISSUES+=("$1")
|
||
REPORT+=(" ⚠️ $1")
|
||
}
|
||
|
||
finding() {
|
||
FINDINGS+=("$1")
|
||
REPORT+=(" ℹ️ $1")
|
||
}
|
||
|
||
format_bytes() {
|
||
local bytes=$1
|
||
if (( bytes > 1073741824 )); then
|
||
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
||
elif (( bytes > 1048576 )); then
|
||
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
||
else
|
||
echo "${bytes}B"
|
||
fi
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 🖥️ System ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "🖥️ SYSTEM"
|
||
|
||
# Uptime
|
||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||
UPTIME_DAYS=$(( UPTIME_SECONDS / 86400 ))
|
||
UPTIME_HOURS=$(( (UPTIME_SECONDS % 86400) / 3600 ))
|
||
BOOT_TIME=$(date -d "@$(( NOW - UPTIME_SECONDS ))" '+%A %-d %b at %-I:%M%p')
|
||
line "Uptime: ${UPTIME_DAYS}d ${UPTIME_HOURS}hr (up since $BOOT_TIME)"
|
||
|
||
# Reboot history this week
|
||
REBOOT_COUNT=0
|
||
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
|
||
WEEK_EPOCH=$(date -d "$WEEK_START" +%s)
|
||
REBOOT_COUNT=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \
|
||
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
|
||
fi
|
||
if [[ "$REBOOT_COUNT" -gt 0 ]]; then
|
||
issue "Reboots this week: $REBOOT_COUNT (system watchdog triggered)"
|
||
else
|
||
line "Reboots this week: 0 ✅"
|
||
fi
|
||
|
||
# Memory
|
||
MEM_TOTAL_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
|
||
MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||
MEM_USED_KB=$(( MEM_TOTAL_KB - MEM_AVAIL_KB ))
|
||
MEM_TOTAL_GB=$(awk "BEGIN {printf \"%.0f\", $MEM_TOTAL_KB / 1048576}")
|
||
MEM_USED_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_USED_KB / 1048576}")
|
||
MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}")
|
||
|
||
ARC_SIZE=0
|
||
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||
ARC_BYTES=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||
ARC_SIZE=$(awk "BEGIN {printf \"%.1f\", $ARC_BYTES / 1073741824}")
|
||
fi
|
||
line "Memory: ${MEM_USED_GB}GB used / ${MEM_TOTAL_GB}GB total (ARC: ${ARC_SIZE}GB)"
|
||
|
||
# Boot drive
|
||
BOOT_PCT=$(df /boot --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||
BOOT_USED=$(df /boot -h --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||
BOOT_SIZE=$(df /boot -h --output=size 2>/dev/null | tail -1 | tr -d ' ')
|
||
if [[ "${BOOT_PCT:-0}" -ge 80 ]]; then
|
||
issue "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE}) — getting full"
|
||
else
|
||
line "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE})"
|
||
fi
|
||
|
||
# Cache drive
|
||
CACHE_PCT=$(df /mnt/cache --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||
CACHE_AVAIL=$(df /mnt/cache -h --output=avail 2>/dev/null | tail -1 | tr -d ' ')
|
||
CACHE_SIZE=$(df /mnt/cache -h --output=size 2>/dev/null | tail -1 | tr -d ' ')
|
||
if [[ "${CACHE_PCT:-0}" -ge 85 ]]; then
|
||
issue "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})"
|
||
else
|
||
line "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})"
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 📀 Array ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "📀 ARRAY"
|
||
|
||
# unRAID array info — parse disks.ini correctly
|
||
# Format uses ["diskN"] ["parity"] ["parity2"] etc.
|
||
if [[ -f /var/local/emhttp/disks.ini ]]; then
|
||
DISK_COUNT=$(grep -c '^\["disk[0-9]' /var/local/emhttp/disks.ini 2>/dev/null || echo "?")
|
||
PARITY_COUNT=$(grep -c '^\["parity' /var/local/emhttp/disks.ini 2>/dev/null || echo "?")
|
||
DISK_COUNT="${DISK_COUNT//[^0-9]/}"; DISK_COUNT="${DISK_COUNT:-?}"
|
||
PARITY_COUNT="${PARITY_COUNT//[^0-9]/}"; PARITY_COUNT="${PARITY_COUNT:-?}"
|
||
line "Array: ${DISK_COUNT} data disks + ${PARITY_COUNT} parity"
|
||
else
|
||
line "Array: disks.ini not found"
|
||
fi
|
||
|
||
# Parity status from parity-checks.log
|
||
PARITY_LOG="/boot/config/parity-checks.log"
|
||
if [[ -f "$PARITY_LOG" ]]; then
|
||
LAST_CHECK=$(tail -1 "$PARITY_LOG" 2>/dev/null)
|
||
if [[ -n "$LAST_CHECK" ]]; then
|
||
PARITY_DATE=$(echo "$LAST_CHECK" | cut -d'|' -f1 | xargs)
|
||
PARITY_ERRORS=$(echo "$LAST_CHECK" | cut -d'|' -f4)
|
||
PARITY_ACTION=$(echo "$LAST_CHECK" | cut -d'|' -f6)
|
||
PARITY_DURATION=$(echo "$LAST_CHECK" | cut -d'|' -f2)
|
||
PARITY_DUR_HR=$(( PARITY_DURATION / 3600 ))
|
||
PARITY_DUR_MIN=$(( (PARITY_DURATION % 3600) / 60 ))
|
||
|
||
# Negative = sync corrections (not data errors), positive = real errors
|
||
if [[ "$PARITY_ERRORS" -gt 0 ]]; then
|
||
issue "Parity: $PARITY_ERRORS errors on last check ($PARITY_DATE)"
|
||
elif [[ "$PARITY_ERRORS" -lt 0 ]]; then
|
||
CORRECTIONS=$(( PARITY_ERRORS * -1 ))
|
||
line "Parity: OK — last: $PARITY_DATE ($PARITY_ACTION, ${PARITY_DUR_HR}h${PARITY_DUR_MIN}m, $CORRECTIONS correction(s)) ✅"
|
||
else
|
||
line "Parity: OK — last: $PARITY_DATE ($PARITY_ACTION, ${PARITY_DUR_HR}h${PARITY_DUR_MIN}m, 0 errors) ✅"
|
||
fi
|
||
|
||
# Check if currently running
|
||
RESYNC=$(grep "^mdResync=" /var/local/emhttp/var.ini 2>/dev/null | cut -d= -f2 | tr -d '"')
|
||
if [[ "$RESYNC" != "0" ]]; then
|
||
finding "Parity check currently in progress"
|
||
fi
|
||
fi
|
||
else
|
||
line "Parity: no check log found"
|
||
fi
|
||
|
||
# ZFS pool health — filter ignored pools
|
||
if command -v zpool >/dev/null 2>&1; then
|
||
while IFS=$'\t' read -r pool health; do
|
||
[[ -z "$pool" ]] && continue
|
||
SKIP=false
|
||
for ignore in "${ZFS_REPORT_IGNORE_POOLS[@]:-}"; do
|
||
[[ "$pool" == "$ignore" ]] && SKIP=true && break
|
||
done
|
||
[[ "$SKIP" == true ]] && continue
|
||
if [[ "$health" == "ONLINE" ]]; then
|
||
line "ZFS $pool: ONLINE ✅"
|
||
else
|
||
issue "ZFS $pool: $health — check immediately"
|
||
fi
|
||
done < <(zpool list -H -o name,health 2>/dev/null)
|
||
fi
|
||
|
||
# Drive temperatures from SMART — build lookup table first then display per disk
|
||
if command -v smartctl >/dev/null 2>&1; then
|
||
declare -A DISK_TEMPS
|
||
TEMP_WARN=false
|
||
ALL_NORMAL=true
|
||
|
||
for disk in /dev/sd? /dev/nvme?; do
|
||
[[ ! -e "$disk" ]] && continue
|
||
DISK_NAME=$(basename "$disk")
|
||
SKIP=false
|
||
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
|
||
[[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break
|
||
done
|
||
[[ "$SKIP" == true ]] && continue
|
||
|
||
TEMP=$(smartctl -A "$disk" 2>/dev/null | awk '
|
||
/^190 / || /^194 / { print $10; exit }
|
||
/Temperature_Celsius/ { print $10; exit }
|
||
/Airflow_Temperature/ { print $10; exit }
|
||
')
|
||
if [[ -z "$TEMP" ]]; then
|
||
TEMP=$(smartctl -A "$disk" 2>/dev/null | \
|
||
awk '/^Temperature:/ { print $2; exit }')
|
||
fi
|
||
TEMP="${TEMP//[^0-9]/}"
|
||
[[ -z "$TEMP" ]] && continue
|
||
|
||
DISK_TEMPS["$DISK_NAME"]="$TEMP"
|
||
|
||
if [[ "$TEMP" -ge "${SMART_TEMP_CRIT:-55}" ]]; then
|
||
issue "Drive $DISK_NAME: ${TEMP}°C — CRITICAL"
|
||
TEMP_WARN=true
|
||
ALL_NORMAL=false
|
||
elif [[ "$TEMP" -ge "${SMART_TEMP_WARN:-45}" ]]; then
|
||
finding "Drive $DISK_NAME: ${TEMP}°C — warm"
|
||
TEMP_WARN=true
|
||
ALL_NORMAL=false
|
||
fi
|
||
done
|
||
|
||
if [[ "$ALL_NORMAL" == true ]]; then
|
||
line "Drive temps: all normal ✅"
|
||
fi
|
||
|
||
# Show per-disk summary with device name from disks.ini + temp
|
||
if [[ -f /var/local/emhttp/disks.ini ]] && [[ ${#DISK_TEMPS[@]} -gt 0 ]]; then
|
||
# Parse disk → device mapping from disks.ini
|
||
DISK_SUMMARY=""
|
||
CURRENT_DISK=""
|
||
while IFS= read -r ini_line; do
|
||
if echo "$ini_line" | grep -qE '^\["(disk[0-9]+|parity[0-9]?|cache)"\]'; then
|
||
CURRENT_DISK=$(echo "$ini_line" | grep -o '"[^"]*"' | head -1 | tr -d '"')
|
||
elif echo "$ini_line" | grep -q '^device='; then
|
||
DEV=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
|
||
TEMP="${DISK_TEMPS[$DEV]:-}"
|
||
if [[ -n "$TEMP" ]]; then
|
||
[[ -n "$DISK_SUMMARY" ]] && DISK_SUMMARY+=", "
|
||
DISK_SUMMARY+="${CURRENT_DISK}(${DEV}):${TEMP}°C"
|
||
fi
|
||
fi
|
||
done < /var/local/emhttp/disks.ini
|
||
[[ -n "$DISK_SUMMARY" ]] && line "Temps: $DISK_SUMMARY"
|
||
fi
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 🎬 Transcodes ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "🎬 TRANSCODES"
|
||
|
||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
|
||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
|
||
RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail | tail -1 | tr -d ' ')
|
||
RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}")
|
||
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null | xargs basename 2>/dev/null || echo "unknown")
|
||
line "Ramdisk now: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET"
|
||
else
|
||
issue "Ramdisk not mounted"
|
||
fi
|
||
|
||
if [[ -f "$TRANSCODE_DAILY_LOG" ]]; then
|
||
WEEK_PEAK=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {if ($2 > max) max=$2} END {printf "%.2f", max+0}' \
|
||
"$TRANSCODE_DAILY_LOG")
|
||
WEEK_FLIPS=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {sum+=$3} END {print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||
WEEK_RAM=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {sum+=$4} END {print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||
WEEK_SSD=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {sum+=$5} END {print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||
WEEK_FILES=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {sum+=$6} END {print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||
|
||
line "Week peak: ${WEEK_PEAK}GB | flips: ${WEEK_FLIPS} | files cleaned: ${WEEK_FILES}"
|
||
line "Sessions: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD"
|
||
|
||
PEAK_INT=$(printf "%.0f" "$WEEK_PEAK" 2>/dev/null || echo 0)
|
||
WARN_INT=$(printf "%.0f" "$RAMDISK_WARN_GB" 2>/dev/null || echo 0)
|
||
if [[ "$PEAK_INT" -ge "$WARN_INT" ]]; then
|
||
finding "Transcode peak ${WEEK_PEAK}GB reached warn threshold — consider increasing RAMDISK_SIZE"
|
||
fi
|
||
if [[ "${WEEK_FLIPS:-0}" -ge "${TRANSCODE_FLIP_WARN:-3}" ]]; then
|
||
finding "Transcode flips this week: $WEEK_FLIPS — monitor ramdisk headroom"
|
||
fi
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 🎵 Media Activity ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "🎵 MEDIA ACTIVITY"
|
||
|
||
# Arr cleanup stats
|
||
if [[ -f "$ARR_CLEANUP_STATS" ]]; then
|
||
for arr in lidarr sonarr radarr; do
|
||
WEEK_ORPHANS=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \
|
||
'$1 >= cutoff && $2 == a {sum+=$3} END {print sum+0}' "$ARR_CLEANUP_STATS")
|
||
WEEK_BYTES=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \
|
||
'$1 >= cutoff && $2 == a {sum+=$4} END {print sum+0}' "$ARR_CLEANUP_STATS")
|
||
WEEK_TRACKED=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \
|
||
'BEGIN{max=0} $1 >= cutoff && $2 == a && $8 > max {max=$8} END {print max+0}' \
|
||
"$ARR_CLEANUP_STATS")
|
||
if [[ "${WEEK_ORPHANS:-0}" -gt 0 ]]; then
|
||
FREED=$(format_bytes "${WEEK_BYTES:-0}")
|
||
line "${arr^} cleanup: $WEEK_ORPHANS orphans removed ($FREED freed) | tracked: $WEEK_TRACKED files"
|
||
else
|
||
line "${arr^} cleanup: clean ✅ (tracked: $WEEK_TRACKED files)"
|
||
fi
|
||
done
|
||
else
|
||
line "Arr cleanup stats: no data yet (runs after first cleanup)"
|
||
fi
|
||
|
||
# Arr recovery stats
|
||
if [[ -f "$ARR_RECOVERY_STATS" ]]; then
|
||
WEEK_ACTIONED=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {sum+=$3} END {print sum+0}' "$ARR_RECOVERY_STATS")
|
||
WEEK_RUNS=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {count++} END {print count+0}' "$ARR_RECOVERY_STATS")
|
||
if [[ "${WEEK_ACTIONED:-0}" -gt 0 ]]; then
|
||
line "Arr recovery: $WEEK_ACTIONED items auto-recovered across $WEEK_RUNS runs"
|
||
else
|
||
line "Arr recovery: no failed imports this week ✅"
|
||
fi
|
||
fi
|
||
|
||
# Arr queue depth — current snapshot
|
||
for arr_name in "Sonarr|${HOST1_SONARR_URL}|${HOST1_SONARR_API_KEY}|v3" \
|
||
"Radarr|${HOST1_RADARR_URL}|${HOST1_RADARR_API_KEY}|v3" \
|
||
"Lidarr|${HOST1_LIDARR_URL}|${HOST1_LIDARR_API_KEY}|v1"; do
|
||
IFS='|' read -r name url key ver <<< "$arr_name"
|
||
if [[ "$url" == *"your-"* ]] || [[ -z "$key" ]]; then continue; fi
|
||
QUEUE=$(curl -sf --max-time 5 -H "X-Api-Key: $key" \
|
||
"${url}/api/${ver}/queue?pageSize=1" 2>/dev/null | \
|
||
grep -o '"totalRecords":[0-9]*' | grep -o '[0-9]*' || echo "?")
|
||
if [[ "$QUEUE" == "0" ]] || [[ -z "$QUEUE" ]]; then
|
||
line "$name queue: empty ✅"
|
||
elif [[ "$QUEUE" == "?" ]]; then
|
||
finding "$name queue: API unavailable"
|
||
else
|
||
line "$name queue: $QUEUE items"
|
||
fi
|
||
done
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 🌐 Rsync ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "🌐 RSYNC"
|
||
|
||
if [[ -f "$BANDWIDTH_LOG" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
|
||
WEEK_TOTAL_BYTES=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {sum+=$3} END {print sum+0}' "$BANDWIDTH_LOG")
|
||
WEEK_SYNCS=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff' "$BANDWIDTH_LOG" | wc -l)
|
||
WEEK_FAILED=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff && $5 != "success"' "$BANDWIDTH_LOG" | wc -l)
|
||
WEEK_GB=$(awk "BEGIN {printf \"%.1f\", $WEEK_TOTAL_BYTES / 1073741824}")
|
||
|
||
line "Total transferred: ${WEEK_GB}GB across $WEEK_SYNCS syncs"
|
||
if [[ "${WEEK_FAILED:-0}" -gt 0 ]]; then
|
||
issue "Failed syncs this week: $WEEK_FAILED"
|
||
else
|
||
line "Sync failures: none ✅"
|
||
fi
|
||
|
||
# Top 3 shares by transfer this week
|
||
TOP_SHARES=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {bytes[$4]+=$3} END {for(s in bytes) print bytes[s], s}' \
|
||
"$BANDWIDTH_LOG" | sort -rn | head -3)
|
||
if [[ -n "$TOP_SHARES" ]]; then
|
||
while IFS=' ' read -r bytes share; do
|
||
[[ -z "$share" ]] && continue
|
||
SIZE=$(format_bytes "$bytes")
|
||
line " → $share: $SIZE"
|
||
done <<< "$TOP_SHARES"
|
||
fi
|
||
else
|
||
line "No bandwidth data yet"
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 🛡️ Watchdog ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "🛡️ WATCHDOG"
|
||
|
||
get_lock_pid_cr() {
|
||
local script_name="$1"
|
||
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||
if [[ -f "$lockfile" ]]; then
|
||
local content
|
||
content=$(cat "$lockfile" 2>/dev/null)
|
||
echo "${content%%:*}"
|
||
fi
|
||
}
|
||
|
||
get_lock_name_cr() {
|
||
local script_name="$1"
|
||
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||
if [[ -f "$lockfile" ]]; then
|
||
local content
|
||
content=$(cat "$lockfile" 2>/dev/null)
|
||
echo "${content##*:}"
|
||
fi
|
||
}
|
||
|
||
is_loop_running() {
|
||
local script_name="$1"
|
||
local pid name
|
||
pid=$(get_lock_pid_cr "$script_name")
|
||
name=$(get_lock_name_cr "$script_name")
|
||
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$name" == "$script_name" ]]
|
||
}
|
||
|
||
get_lock_age_cr() {
|
||
local script_name="$1"
|
||
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||
if [[ -f "$lockfile" ]]; then
|
||
local mtime
|
||
mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0)
|
||
echo $(( $(date +%s) - mtime ))
|
||
else
|
||
echo 0
|
||
fi
|
||
}
|
||
|
||
format_uptime_cr() {
|
||
local seconds=$1
|
||
local days=$(( seconds / 86400 ))
|
||
local hours=$(( (seconds % 86400) / 3600 ))
|
||
local mins=$(( (seconds % 3600) / 60 ))
|
||
if (( days > 0 )); then echo "${days}d ${hours}h ${mins}m"
|
||
elif (( hours > 0 )); then echo "${hours}h ${mins}m"
|
||
else echo "${mins}m"; fi
|
||
}
|
||
|
||
# ── System Watchdog ──
|
||
line "⚙️ System Watchdog"
|
||
|
||
SYS_PID_CR=$(get_lock_pid_cr "system_watchdog")
|
||
if is_loop_running "system_watchdog"; then
|
||
SYS_AGE_CR=$(get_lock_age_cr "system_watchdog")
|
||
SYS_UP_CR=$(format_uptime_cr "$SYS_AGE_CR")
|
||
SYS_CYCLE_CR=$(( SYS_AGE_CR / SYSTEM_WATCHDOG_INTERVAL ))
|
||
line " ✅ Running │ PID: $SYS_PID_CR │ Uptime: $SYS_UP_CR │ ~Cycle: $SYS_CYCLE_CR"
|
||
else
|
||
issue "system_watchdog NOT RUNNING"
|
||
fi
|
||
|
||
# System strikes
|
||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||
SYS_ACTIVE=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||
if [[ -n "$SYS_ACTIVE" ]]; then
|
||
while IFS=: read -r key count; do
|
||
[[ -z "$key" ]] && continue
|
||
issue " ⚠️ Strike: $key — $count/$SYS_WATCHDOG_STRIKE_LIMIT"
|
||
done <<< "$SYS_ACTIVE"
|
||
else
|
||
line " ✅ Strikes: none"
|
||
fi
|
||
fi
|
||
|
||
# Reboots this week
|
||
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
|
||
WEEK_EPOCH=$(date -d "$WEEK_START" +%s)
|
||
REBOOT_COUNT=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \
|
||
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
|
||
if [[ "${REBOOT_COUNT:-0}" -gt 0 ]]; then
|
||
issue " 🔄 Reboots this week: $REBOOT_COUNT (watchdog triggered)"
|
||
else
|
||
line " ✅ Reboots this week: 0"
|
||
fi
|
||
fi
|
||
|
||
# Skip list
|
||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||
SKIP_LIST=$(tr '\n' ' ' < "$SYS_WATCHDOG_FAILED_FILE")
|
||
issue " ⛔ Skip list ($SKIP_COUNT): $SKIP_LIST"
|
||
else
|
||
line " ✅ Skip list: empty"
|
||
fi
|
||
|
||
# System snapshot
|
||
ROOTFS_PCT_CR=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||
MEM_AVAIL_CR=$(awk '/MemAvailable/ {printf "%.1f", $2/1048576}' /proc/meminfo)
|
||
MEM_TOTAL_CR=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
|
||
LOAD_CR=$(awk '{print $1}' /proc/loadavg)
|
||
ZOMBIE_CR=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null)
|
||
ZOMBIE_CR="${ZOMBIE_CR//[^0-9]/}"
|
||
ZOMBIE_CR="${ZOMBIE_CR:-0}"
|
||
ARC_GB_CR="n/a"
|
||
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||
ARC_B=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||
ARC_GB_CR=$(awk "BEGIN {printf \"%.1f\", $ARC_B / 1073741824}")
|
||
fi
|
||
line " 📊 rootfs: ${ROOTFS_PCT_CR}% │ RAM: ${MEM_AVAIL_CR}GB free/${MEM_TOTAL_CR}GB │ ARC: ${ARC_GB_CR}GB │ load: ${LOAD_CR} │ zombies: ${ZOMBIE_CR}"
|
||
|
||
REPORT+=("")
|
||
|
||
# ── Docker Watchdog ──
|
||
line "🐳 Docker Watchdog"
|
||
|
||
DOCKER_PID_CR=$(get_lock_pid_cr "docker_watchdog")
|
||
if is_loop_running "docker_watchdog"; then
|
||
DOCKER_AGE_CR=$(get_lock_age_cr "docker_watchdog")
|
||
DOCKER_UP_CR=$(format_uptime_cr "$DOCKER_AGE_CR")
|
||
DOCKER_CYCLE_CR=$(( DOCKER_AGE_CR / DOCKER_WATCHDOG_INTERVAL ))
|
||
line " ✅ Running │ PID: $DOCKER_PID_CR │ Uptime: $DOCKER_UP_CR │ ~Cycle: $DOCKER_CYCLE_CR"
|
||
else
|
||
issue "docker_watchdog NOT RUNNING"
|
||
fi
|
||
|
||
# Container strikes
|
||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||
DOCK_ACTIVE=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||
if [[ -n "$DOCK_ACTIVE" ]]; then
|
||
while IFS=: read -r key count; do
|
||
[[ -z "$key" ]] && continue
|
||
issue " ⚠️ Strike: $key — $count"
|
||
done <<< "$DOCK_ACTIVE"
|
||
else
|
||
line " ✅ Container strikes: none"
|
||
fi
|
||
fi
|
||
|
||
# Container restarts this week
|
||
if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
|
||
WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l)
|
||
if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then
|
||
RESTARTED=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
|
||
sort | uniq -c | sort -rn | head -5 | \
|
||
awk '{print $2"("$1")"}' | tr '\n' ' ')
|
||
finding " 🔄 Restarts this week: $WEEK_RESTARTS — $RESTARTED"
|
||
else
|
||
line " ✅ Container restarts this week: none"
|
||
fi
|
||
fi
|
||
|
||
# Docker overview — filter WATCHDOG_SCAN_IGNORE from stopped list
|
||
if command -v docker >/dev/null 2>&1; then
|
||
RUNNING_CR=$(docker ps -q 2>/dev/null | wc -l)
|
||
TOTAL_CR=$(docker ps -aq 2>/dev/null | wc -l)
|
||
UNHEALTHY_CR=$(docker ps --filter health=unhealthy -q 2>/dev/null | wc -l)
|
||
|
||
STOPPED_FILTERED_CR=()
|
||
while IFS= read -r name; do
|
||
[[ -z "$name" ]] && continue
|
||
SKIP=false
|
||
for ignore in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do
|
||
[[ "$name" == "$ignore" ]] && SKIP=true && break
|
||
done
|
||
[[ "$SKIP" == false ]] && STOPPED_FILTERED_CR+=("$name")
|
||
done < <(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null)
|
||
|
||
STOPPED_COUNT_CR="${#STOPPED_FILTERED_CR[@]}"
|
||
line " 📦 $RUNNING_CR/$TOTAL_CR running │ unhealthy: $UNHEALTHY_CR"
|
||
if [[ "$STOPPED_COUNT_CR" -gt 0 ]]; then
|
||
issue " ⚠️ Stopped (unexpected): ${STOPPED_FILTERED_CR[*]}"
|
||
else
|
||
line " ✅ All containers running"
|
||
fi
|
||
fi
|
||
|
||
REPORT+=("")
|
||
|
||
# ── Failover ──
|
||
line "🔀 Failover"
|
||
|
||
FAILOVER_PID_CR=$(get_lock_pid_cr "failover")
|
||
if is_loop_running "failover"; then
|
||
FAILOVER_AGE_CR=$(get_lock_age_cr "failover")
|
||
FAILOVER_UP_CR=$(format_uptime_cr "$FAILOVER_AGE_CR")
|
||
line " ✅ Running │ PID: $FAILOVER_PID_CR │ Uptime: $FAILOVER_UP_CR"
|
||
else
|
||
if [[ "${FAILOVER_ENABLED:-true}" == false ]]; then
|
||
line " ⏸️ Not running — disabled in Master.conf (FAILOVER_ENABLED=false)"
|
||
else
|
||
issue "failover NOT RUNNING"
|
||
fi
|
||
fi
|
||
|
||
FAILOVER_STATE_CR="UNKNOWN"
|
||
FAILOVER_STATE_SECS_CR=0
|
||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||
FAILOVER_STATE_CR=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||
FAILOVER_LAST_EPOCH_CR=$(grep "^last_change_epoch=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||
[[ -n "$FAILOVER_LAST_EPOCH_CR" ]] && \
|
||
FAILOVER_STATE_SECS_CR=$(( $(date +%s) - FAILOVER_LAST_EPOCH_CR ))
|
||
fi
|
||
|
||
STATE_DUR_CR=$(format_uptime_cr "${FAILOVER_STATE_SECS_CR:-0}")
|
||
|
||
case "$FAILOVER_STATE_CR" in
|
||
NORMAL)
|
||
line " ✅ State: NORMAL │ Duration: $STATE_DUR_CR"
|
||
;;
|
||
FAILOVER)
|
||
issue " ⚠️ State: FAILOVER — remote down for $STATE_DUR_CR"
|
||
FAILOVER_MINS_CR=$(( FAILOVER_STATE_SECS_CR / 60 ))
|
||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||
T2=$HOST2_TIER2_DELAY; T3=$HOST2_TIER3_DELAY; T4=$HOST2_TIER4_DELAY
|
||
else
|
||
T2=$HOST1_TIER2_DELAY; T3=$HOST1_TIER3_DELAY; T4=$HOST1_TIER4_DELAY
|
||
fi
|
||
(( FAILOVER_MINS_CR >= T2 )) && line " 🔄 Tier 2: ✅ active" || \
|
||
line " 🔄 Tier 2: ⏳ in $(( T2 - FAILOVER_MINS_CR ))min"
|
||
(( FAILOVER_MINS_CR >= T3 )) && line " 🔄 Tier 3: ✅ active" || \
|
||
line " 🔄 Tier 3: ⏳ in $(( T3 - FAILOVER_MINS_CR ))min"
|
||
(( FAILOVER_MINS_CR >= T4 )) && line " 🔄 Tier 4: ✅ active" || \
|
||
line " 🔄 Tier 4: ⏳ in $(( T4 - FAILOVER_MINS_CR ))min"
|
||
;;
|
||
NO_INTERNET)
|
||
issue " ❌ State: NO_INTERNET — DDNS stopped │ Duration: $STATE_DUR_CR"
|
||
;;
|
||
DARK)
|
||
issue " ❌ State: DARK — remote down AND no internet │ Duration: $STATE_DUR_CR"
|
||
;;
|
||
*)
|
||
finding " ❓ State: ${FAILOVER_STATE_CR:-unknown}"
|
||
;;
|
||
esac
|
||
|
||
# Tailscale
|
||
if command -v tailscale >/dev/null 2>&1; then
|
||
REMOTE_IP_CR=$(tailscale ip -4 "$HOST2" 2>/dev/null)
|
||
if [[ -n "$REMOTE_IP_CR" ]]; then
|
||
if ping -c 1 -W 2 "$REMOTE_IP_CR" >/dev/null 2>&1; then
|
||
line " 🌐 $HOST2: $REMOTE_IP_CR — reachable ✅"
|
||
else
|
||
issue " 🌐 $HOST2: $REMOTE_IP_CR — not responding"
|
||
fi
|
||
else
|
||
issue " 🌐 $HOST2 not visible on Tailscale"
|
||
fi
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 🔐 Security ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "🔐 SECURITY"
|
||
|
||
for domain in "${CERT_MONITOR_DOMAINS[@]:-}"; do
|
||
[[ -z "$domain" ]] && continue
|
||
EXPIRY=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \
|
||
-connect "${domain}:443" -servername "$domain" 2>/dev/null | \
|
||
openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||
if [[ -z "$EXPIRY" ]]; then
|
||
issue "$domain: could not check certificate"
|
||
continue
|
||
fi
|
||
EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s 2>/dev/null || echo 0)
|
||
DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW) / 86400 ))
|
||
if [[ "$DAYS_LEFT" -le "${CERT_CRIT_DAYS:-7}" ]]; then
|
||
issue "$domain: ${DAYS_LEFT} days remaining — CRITICAL, renew now"
|
||
elif [[ "$DAYS_LEFT" -le "${CERT_WARN_DAYS:-30}" ]]; then
|
||
finding "$domain: ${DAYS_LEFT} days remaining — renew soon"
|
||
else
|
||
line "$domain: ${DAYS_LEFT} days remaining ✅"
|
||
fi
|
||
done
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 📊 Emby ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "📊 EMBY"
|
||
|
||
# Select correct Emby URL and key for this host
|
||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||
EMBY_URL="$HOST1_EMBY_URL"
|
||
EMBY_KEY="$HOST1_EMBY_API_KEY"
|
||
else
|
||
EMBY_URL="$HOST2_EMBY_URL"
|
||
EMBY_KEY="$HOST2_EMBY_API_KEY"
|
||
fi
|
||
|
||
if [[ "$EMBY_KEY" != *"your-"* ]] && [[ -n "$EMBY_KEY" ]]; then
|
||
|
||
# Check if Emby is reachable
|
||
EMBY_SYSTEM=$(curl -sf --max-time 5 \
|
||
-H "X-Emby-Token: $EMBY_KEY" \
|
||
"${EMBY_URL}/System/Info?api_key=$EMBY_KEY" 2>/dev/null)
|
||
|
||
if [[ -z "$EMBY_SYSTEM" ]]; then
|
||
line "Emby: API unavailable — check if Emby is running"
|
||
else
|
||
EMBY_VERSION=$(echo "$EMBY_SYSTEM" | grep -o '"Version":"[^"]*"' | cut -d'"' -f4)
|
||
line "Emby: v${EMBY_VERSION:-unknown} — reachable ✅"
|
||
|
||
# Active sessions right now
|
||
SESSIONS=$(curl -sf --max-time 5 \
|
||
-H "X-Emby-Token: $EMBY_KEY" \
|
||
"${EMBY_URL}/Sessions?api_key=$EMBY_KEY" 2>/dev/null)
|
||
ACTIVE_COUNT=$(echo "$SESSIONS" | grep -o '"NowPlayingItem"' | wc -l)
|
||
ACTIVE_COUNT="${ACTIVE_COUNT//[^0-9]/}"; ACTIVE_COUNT="${ACTIVE_COUNT:-0}"
|
||
line "Active streams now: $ACTIVE_COUNT"
|
||
|
||
# Weekly stats via user_usage_stats plugin if available
|
||
ACTIVITY_LOG=$(curl -sf --max-time 10 \
|
||
-H "X-Emby-Token: $EMBY_KEY" \
|
||
"${EMBY_URL}/user_usage_stats/user_activity?days=7&api_key=$EMBY_KEY" \
|
||
2>/dev/null)
|
||
|
||
if [[ -n "$ACTIVITY_LOG" ]] && echo "$ACTIVITY_LOG" | grep -q "user_name"; then
|
||
# Parse total plays — sum all total_plays fields
|
||
TOTAL_PLAYS=$(echo "$ACTIVITY_LOG" | \
|
||
grep -o '"total_plays":[0-9]*' | \
|
||
awk -F: '{sum+=$2} END {print sum+0}')
|
||
line "Streams this week: $TOTAL_PLAYS total plays"
|
||
|
||
# Top 3 users by play count
|
||
echo "$ACTIVITY_LOG" | \
|
||
grep -o '"user_name":"[^"]*","total_plays":[0-9]*' | \
|
||
awk -F'"' '{name=$4; plays=$NF; gsub(/.*:/,"",plays); print plays, name}' | \
|
||
sort -rn | head -3 | \
|
||
while read -r plays name; do
|
||
line " → $name: $plays plays"
|
||
done
|
||
else
|
||
line "Weekly stats: user_usage_stats plugin not available"
|
||
fi
|
||
fi
|
||
else
|
||
line "Emby: API key not configured"
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ ⚙️ System Health ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "⚙️ SYSTEM HEALTH"
|
||
|
||
# inotify + php-fpm weekly stats
|
||
if [[ -f "${TUNING_MONITOR_LOG:-}" ]] && [[ -s "$TUNING_MONITOR_LOG" ]]; then
|
||
# inotify weekly stats
|
||
INOTIFY_PEAK=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {if ($3+0 > max) max=$3+0} END {print max+0}' "$TUNING_MONITOR_LOG")
|
||
INOTIFY_AVG=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {sum+=$3; count++} END {if(count>0) printf "%.0f", sum/count; else print 0}' \
|
||
"$TUNING_MONITOR_LOG")
|
||
INOTIFY_LOW=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'BEGIN{min=99999} $1 >= cutoff {if($3+0 < min) min=$3+0} END {print min+0}' \
|
||
"$TUNING_MONITOR_LOG")
|
||
INOTIFY_LIMIT_NOW=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024)
|
||
INOTIFY_NOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
|
||
INOTIFY_NOW="${INOTIFY_NOW//[^0-9]/}"; INOTIFY_NOW="${INOTIFY_NOW:-0}"
|
||
INOTIFY_NOW_PCT=$(( INOTIFY_NOW * 100 / INOTIFY_LIMIT_NOW ))
|
||
INOTIFY_WARN_COUNT=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff && $6 == 1 {count++} END {print count+0}' "$TUNING_MONITOR_LOG")
|
||
|
||
if [[ "${INOTIFY_WARN_COUNT:-0}" -gt 0 ]]; then
|
||
issue "inotify: ${INOTIFY_NOW}/${INOTIFY_LIMIT_NOW} now (${INOTIFY_NOW_PCT}%) | week peak: $INOTIFY_PEAK avg: $INOTIFY_AVG low: $INOTIFY_LOW | ⚠️ warnings: $INOTIFY_WARN_COUNT"
|
||
else
|
||
line "inotify: ${INOTIFY_NOW}/${INOTIFY_LIMIT_NOW} now (${INOTIFY_NOW_PCT}%) | week peak: $INOTIFY_PEAK avg: $INOTIFY_AVG low: $INOTIFY_LOW ✅"
|
||
fi
|
||
|
||
# php-fpm weekly stats
|
||
PHPFPM_PEAK=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {if ($7+0 > max) max=$7+0} END {print max+0}' "$TUNING_MONITOR_LOG")
|
||
PHPFPM_AVG=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {sum+=$7; count++} END {if(count>0) printf "%.0f", sum/count; else print 0}' \
|
||
"$TUNING_MONITOR_LOG")
|
||
PHPFPM_MAX_NOW="${PHP_MAX_CHILDREN:-250}"
|
||
PHPFPM_NOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0)
|
||
PHPFPM_NOW="${PHPFPM_NOW//[^0-9]/}"; PHPFPM_NOW="${PHPFPM_NOW:-0}"
|
||
PHPFPM_WARN_COUNT=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff && $10 == 1 {count++} END {print count+0}' "$TUNING_MONITOR_LOG")
|
||
|
||
if [[ "${PHPFPM_WARN_COUNT:-0}" -gt 0 ]]; then
|
||
issue "php-fpm: ${PHPFPM_NOW}/${PHPFPM_MAX_NOW} workers now | week peak: $PHPFPM_PEAK avg: $PHPFPM_AVG | ⚠️ warnings: $PHPFPM_WARN_COUNT"
|
||
else
|
||
line "php-fpm: ${PHPFPM_NOW}/${PHPFPM_MAX_NOW} workers now | week peak: $PHPFPM_PEAK avg: $PHPFPM_AVG ✅"
|
||
fi
|
||
else
|
||
# No log yet — just show live values
|
||
INOTIFY_LIMIT_NOW=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024)
|
||
INOTIFY_NOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
|
||
INOTIFY_NOW="${INOTIFY_NOW//[^0-9]/}"; INOTIFY_NOW="${INOTIFY_NOW:-0}"
|
||
INOTIFY_NOW_PCT=$(( INOTIFY_NOW * 100 / INOTIFY_LIMIT_NOW ))
|
||
PHPFPM_NOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0)
|
||
PHPFPM_NOW="${PHPFPM_NOW//[^0-9]/}"; PHPFPM_NOW="${PHPFPM_NOW:-0}"
|
||
line "inotify: ${INOTIFY_NOW}/${INOTIFY_LIMIT_NOW} (${INOTIFY_NOW_PCT}%) — no weekly data yet"
|
||
line "php-fpm: ${PHPFPM_NOW}/${PHP_MAX_CHILDREN:-250} workers — no weekly data yet"
|
||
fi
|
||
|
||
# SMART summary
|
||
if command -v smartctl >/dev/null 2>&1; then
|
||
SMART_ISSUES=0
|
||
for disk in /dev/sd? /dev/nvme?; do
|
||
[[ ! -e "$disk" ]] && continue
|
||
DISK_NAME=$(basename "$disk")
|
||
SKIP=false
|
||
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
|
||
[[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break
|
||
done
|
||
[[ "$SKIP" == true ]] && continue
|
||
HEALTH=$(smartctl -H "$disk" 2>/dev/null | grep "SMART overall-health" | \
|
||
awk '{print $NF}')
|
||
if [[ "$HEALTH" != "PASSED" ]] && [[ -n "$HEALTH" ]]; then
|
||
issue "SMART $DISK_NAME: $HEALTH — check immediately"
|
||
((SMART_ISSUES++))
|
||
fi
|
||
done
|
||
if [[ "$SMART_ISSUES" -eq 0 ]]; then
|
||
line "SMART: all drives PASSED ✅"
|
||
fi
|
||
fi
|
||
|
||
# Gitea sync status
|
||
if command -v git >/dev/null 2>&1 && [[ -d "$TARGET_DIR/.git" ]]; then
|
||
CURRENT_COMMIT=$(git -C "$TARGET_DIR" rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||
LAST_PULL=$(git -C "$TARGET_DIR" log -1 --format="%ar" 2>/dev/null || echo "unknown")
|
||
line "Gitea: commit $CURRENT_COMMIT (pulled $LAST_PULL)"
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ ⚠️ Issues Requiring Attention ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
if [[ ${#ISSUES[@]} -gt 0 ]]; then
|
||
REPORT+=("")
|
||
REPORT+=("⚠️ ISSUES REQUIRING ATTENTION")
|
||
REPORT+=("$(printf '%.0s─' {1..50})")
|
||
for issue_line in "${ISSUES[@]}"; do
|
||
REPORT+=(" ❌ $issue_line")
|
||
done
|
||
fi
|
||
|
||
if [[ ${#FINDINGS[@]} -gt 0 ]]; then
|
||
REPORT+=("")
|
||
REPORT+=("ℹ️ NOTABLE")
|
||
REPORT+=("$(printf '%.0s─' {1..50})")
|
||
for finding_line in "${FINDINGS[@]}"; do
|
||
REPORT+=(" → $finding_line")
|
||
done
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ Footer ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
REPORT+=("")
|
||
if [[ ${#ISSUES[@]} -eq 0 ]]; then
|
||
STATUS="✅ All systems healthy — enjoy your Sunday"
|
||
else
|
||
STATUS="⚠️ ${#ISSUES[@]} issue(s) need attention"
|
||
fi
|
||
REPORT+=("$STATUS")
|
||
REPORT+=("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ Output and Send ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
HEADER="☕ SUNDAY MORNING COFFEE REPORT — $REPORT_DATE"
|
||
DIVIDER="$(printf '%.0s━' {1..50})"
|
||
BODY=$(printf '%s\n' "$HEADER" "${REPORT[@]}")
|
||
|
||
echo ""
|
||
echo "$DIVIDER"
|
||
echo "$HEADER"
|
||
echo "$DIVIDER"
|
||
|
||
for report_line in "${REPORT[@]}"; do
|
||
echo "$report_line"
|
||
done
|
||
|
||
echo ""
|
||
|
||
# Send notification
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — notification not sent"
|
||
else
|
||
# Notify with full body — suppress body from terminal log to avoid duplication
|
||
if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then
|
||
NOTIFY_SCRIPT="/usr/local/emhttp/plugins/dynamix/scripts/notify"
|
||
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
||
"$NOTIFY_SCRIPT" -s "☕ Weekly Report" -d "$BODY" -i "normal" 2>/dev/null
|
||
log "$ICON_NOTIFY unRAID notification sent"
|
||
fi
|
||
fi
|
||
if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then
|
||
PAYLOAD=$(printf '{"content": "%s — **%s**\\n%s"}' \
|
||
"$ICON_NOTIFY" "☕ Weekly Report" "$BODY")
|
||
curl -s -H "Content-Type: application/json" \
|
||
-d "$PAYLOAD" "$DISCORD_WEBHOOK" >/dev/null 2>&1 && \
|
||
log "$ICON_NOTIFY Discord notification sent" || \
|
||
warn "Discord notification failed"
|
||
fi
|
||
success "Report sent"
|
||
fi |