633 lines
25 KiB
Bash
633 lines
25 KiB
Bash
#!/bin/bash
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ----------------------------- Sunday Morning Coffee Report -----------------------------------
|
||
# -----------------------------------------------------------------------------------------------
|
||
# Weekly system overview — everything that happened this week in one clean read.
|
||
# Designed to be read over coffee Sunday morning while the system is fully caught up
|
||
# from the 2:30am maintenance window.
|
||
# Schedule: 0 7 * * 0 (7am Sunday — after weekly_sync_maintenance.sh finishes at ~3am)
|
||
# Maintenance window completes → 4 hours of fresh data → report ready ☕
|
||
#
|
||
# Sections:
|
||
# 🖥️ System — uptime, memory, boot drive, cache drive, reboots
|
||
# 📀 Array — disk count, parity status, ZFS health, drive temps
|
||
# 🔀 Failover — state, last change, Tailscale connectivity
|
||
# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split
|
||
# 🎵 Media Activity — arr cleanup stats, arr recovery stats, library health
|
||
# 🌐 Rsync — weekly transfer totals, per-share breakdown
|
||
# 🛡️ Watchdog — container strikes, restarts, system strikes, skip list
|
||
# 🔐 Security — SSL cert expiry per domain
|
||
# 📊 Emby — weekly stream count, top users, top content
|
||
# ⚙️ Health — SMART summary, docker container count, Gitea sync status
|
||
# ⚠️ Issues — anything requiring attention collected above
|
||
#
|
||
# Data sources (reads only — no writes except the notification):
|
||
# DATA_DIR stats files — arr cleanup, recovery, transcode, bandwidth history
|
||
# /boot/config — failover state, watchdog reboot log
|
||
# /tmp — watchdog strike state files
|
||
# /proc, /sys — system memory, uptime
|
||
# /var/local/emhttp/ — unRAID array info
|
||
# Emby API — session history
|
||
# Arr APIs — current queue depth
|
||
# tailscale — network status
|
||
# openssl — live SSL cert check
|
||
# smartctl — drive health
|
||
#
|
||
# All configuration in Master.conf.
|
||
# Supports --dry-run to preview report without sending notification.
|
||
# -----------------------------------------------------------------------------------------------
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../Master.conf"
|
||
source "$SCRIPT_DIR/../common.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ Setup ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
acquire_lock
|
||
|
||
detect_hosts
|
||
|
||
REPORT_DATE=$(date '+%A, %B %-d, %Y')
|
||
WEEK_START=$(date -d "7 days ago" '+%Y-%m-%d')
|
||
TODAY=$(date '+%Y-%m-%d')
|
||
NOW=$(date +%s)
|
||
|
||
REPORT=() # all report lines
|
||
ISSUES=() # items needing attention
|
||
FINDINGS=() # notable but not critical
|
||
|
||
section() {
|
||
REPORT+=("")
|
||
REPORT+=("$1")
|
||
REPORT+=("$(printf '%.0s─' {1..50})")
|
||
}
|
||
|
||
line() {
|
||
REPORT+=(" $1")
|
||
}
|
||
|
||
issue() {
|
||
ISSUES+=("$1")
|
||
REPORT+=(" ⚠️ $1")
|
||
}
|
||
|
||
finding() {
|
||
FINDINGS+=("$1")
|
||
REPORT+=(" ℹ️ $1")
|
||
}
|
||
|
||
format_bytes() {
|
||
local bytes=$1
|
||
if (( bytes > 1073741824 )); then
|
||
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
||
elif (( bytes > 1048576 )); then
|
||
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
||
else
|
||
echo "${bytes}B"
|
||
fi
|
||
}
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 🖥️ System ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "🖥️ SYSTEM"
|
||
|
||
# Uptime
|
||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||
UPTIME_DAYS=$(( UPTIME_SECONDS / 86400 ))
|
||
UPTIME_HOURS=$(( (UPTIME_SECONDS % 86400) / 3600 ))
|
||
BOOT_TIME=$(date -d "@$(( NOW - UPTIME_SECONDS ))" '+%A %-d %b at %-I:%M%p')
|
||
line "Uptime: ${UPTIME_DAYS}d ${UPTIME_HOURS}hr (up since $BOOT_TIME)"
|
||
|
||
# Reboot history this week
|
||
REBOOT_COUNT=0
|
||
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
|
||
WEEK_EPOCH=$(date -d "$WEEK_START" +%s)
|
||
REBOOT_COUNT=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \
|
||
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
|
||
fi
|
||
if [[ "$REBOOT_COUNT" -gt 0 ]]; then
|
||
issue "Reboots this week: $REBOOT_COUNT (system watchdog triggered)"
|
||
else
|
||
line "Reboots this week: 0 ✅"
|
||
fi
|
||
|
||
# Memory
|
||
MEM_TOTAL_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
|
||
MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||
MEM_USED_KB=$(( MEM_TOTAL_KB - MEM_AVAIL_KB ))
|
||
MEM_TOTAL_GB=$(awk "BEGIN {printf \"%.0f\", $MEM_TOTAL_KB / 1048576}")
|
||
MEM_USED_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_USED_KB / 1048576}")
|
||
MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}")
|
||
|
||
ARC_SIZE=0
|
||
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||
ARC_BYTES=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||
ARC_SIZE=$(awk "BEGIN {printf \"%.1f\", $ARC_BYTES / 1073741824}")
|
||
fi
|
||
line "Memory: ${MEM_USED_GB}GB used / ${MEM_TOTAL_GB}GB total (ARC: ${ARC_SIZE}GB)"
|
||
|
||
# Boot drive
|
||
BOOT_PCT=$(df /boot --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||
BOOT_USED=$(df /boot -h --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||
BOOT_SIZE=$(df /boot -h --output=size 2>/dev/null | tail -1 | tr -d ' ')
|
||
if [[ "${BOOT_PCT:-0}" -ge 80 ]]; then
|
||
issue "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE}) — getting full"
|
||
else
|
||
line "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE})"
|
||
fi
|
||
|
||
# Cache drive
|
||
CACHE_PCT=$(df /mnt/cache --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||
CACHE_AVAIL=$(df /mnt/cache -h --output=avail 2>/dev/null | tail -1 | tr -d ' ')
|
||
CACHE_SIZE=$(df /mnt/cache -h --output=size 2>/dev/null | tail -1 | tr -d ' ')
|
||
if [[ "${CACHE_PCT:-0}" -ge 85 ]]; then
|
||
issue "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})"
|
||
else
|
||
line "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})"
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 📀 Array ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "📀 ARRAY"
|
||
|
||
# unRAID array info
|
||
if [[ -f /var/local/emhttp/disks.ini ]]; then
|
||
DISK_COUNT=$(grep -c "^\[disk" /var/local/emhttp/disks.ini 2>/dev/null || echo "?")
|
||
line "Array disks: $DISK_COUNT"
|
||
fi
|
||
|
||
# Parity
|
||
if [[ -f /var/local/emhttp/parity-date.txt ]]; then
|
||
PARITY_INFO=$(cat /var/local/emhttp/parity-date.txt 2>/dev/null)
|
||
if echo "$PARITY_INFO" | grep -q "progress"; then
|
||
finding "Parity check in progress"
|
||
else
|
||
PARITY_DATE=$(echo "$PARITY_INFO" | grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -1)
|
||
PARITY_ERRORS=$(echo "$PARITY_INFO" | grep -oE 'errors=[0-9]+' | cut -d= -f2 || echo 0)
|
||
if [[ "${PARITY_ERRORS:-0}" -gt 0 ]]; then
|
||
issue "Parity: $PARITY_ERRORS errors on last check ($PARITY_DATE)"
|
||
else
|
||
line "Parity: OK (last check: ${PARITY_DATE:-unknown}, 0 errors)"
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
# ZFS pool health
|
||
if command -v zpool >/dev/null 2>&1; then
|
||
POOLS=$(zpool list -H -o name,health 2>/dev/null)
|
||
while IFS=$'\t' read -r pool health; do
|
||
[[ -z "$pool" ]] && continue
|
||
# Skip single-disk unRAID array pools
|
||
if [[ "$health" == "ONLINE" ]]; then
|
||
line "ZFS $pool: ONLINE ✅"
|
||
else
|
||
issue "ZFS $pool: $health — check immediately"
|
||
fi
|
||
done <<< "$POOLS"
|
||
fi
|
||
|
||
# Drive temperatures from SMART
|
||
if command -v smartctl >/dev/null 2>&1; then
|
||
TEMP_WARN=false
|
||
TEMP_SUMMARY=""
|
||
for disk in /dev/sd? /dev/nvme?; do
|
||
[[ ! -e "$disk" ]] && continue
|
||
DISK_NAME=$(basename "$disk")
|
||
# Skip ignored drives
|
||
SKIP=false
|
||
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
|
||
[[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break
|
||
done
|
||
[[ "$SKIP" == true ]] && continue
|
||
TEMP=$(smartctl -A "$disk" 2>/dev/null | \
|
||
awk '/Temperature_Celsius|Temperature:/ {print $NF; exit}' | tr -d '°C')
|
||
[[ -z "$TEMP" ]] && continue
|
||
TEMP_INT=$(printf "%.0f" "$TEMP" 2>/dev/null || echo 0)
|
||
if [[ "$TEMP_INT" -ge "${SMART_TEMP_CRIT:-55}" ]]; then
|
||
issue "Drive $DISK_NAME: ${TEMP_INT}°C — CRITICAL"
|
||
TEMP_WARN=true
|
||
elif [[ "$TEMP_INT" -ge "${SMART_TEMP_WARN:-45}" ]]; then
|
||
finding "Drive $DISK_NAME: ${TEMP_INT}°C — warm"
|
||
TEMP_WARN=true
|
||
fi
|
||
[[ -n "$TEMP_SUMMARY" ]] && TEMP_SUMMARY+=", "
|
||
TEMP_SUMMARY+="${DISK_NAME}:${TEMP_INT}°C"
|
||
done
|
||
if [[ "$TEMP_WARN" == false ]]; then
|
||
line "Drive temps: all normal"
|
||
fi
|
||
[[ -n "$TEMP_SUMMARY" ]] && line "Temps: $TEMP_SUMMARY"
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 🔀 Failover ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "🔀 FAILOVER"
|
||
|
||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||
FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||
FAILOVER_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||
if [[ "$FAILOVER_STATE" == "NORMAL" ]]; then
|
||
line "State: NORMAL ✅"
|
||
else
|
||
issue "Failover state: $FAILOVER_STATE — not normal"
|
||
fi
|
||
line "Last change: ${FAILOVER_CHANGE:-unknown}"
|
||
else
|
||
issue "Failover state file not found"
|
||
fi
|
||
|
||
# Tailscale connectivity
|
||
if command -v tailscale >/dev/null 2>&1; then
|
||
TS_STATUS=$(tailscale status 2>/dev/null)
|
||
REMOTE_VISIBLE=$(echo "$TS_STATUS" | grep -c "$HOST2" 2>/dev/null || echo 0)
|
||
if [[ "$REMOTE_VISIBLE" -gt 0 ]]; then
|
||
REMOTE_IP=$(tailscale ip -4 "$HOST2" 2>/dev/null || echo "unknown")
|
||
line "Tailscale: $HOST2 visible at $REMOTE_IP ✅"
|
||
else
|
||
issue "Tailscale: $HOST2 not visible — check connectivity"
|
||
fi
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 🎬 Transcodes ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "🎬 TRANSCODES"
|
||
|
||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
|
||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
|
||
RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail | tail -1 | tr -d ' ')
|
||
RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}")
|
||
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null | xargs basename 2>/dev/null || echo "unknown")
|
||
line "Ramdisk now: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET"
|
||
else
|
||
issue "Ramdisk not mounted"
|
||
fi
|
||
|
||
if [[ -f "$TRANSCODE_DAILY_LOG" ]]; then
|
||
WEEK_PEAK=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {if ($2 > max) max=$2} END {printf "%.2f", max+0}' \
|
||
"$TRANSCODE_DAILY_LOG")
|
||
WEEK_FLIPS=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {sum+=$3} END {print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||
WEEK_RAM=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {sum+=$4} END {print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||
WEEK_SSD=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {sum+=$5} END {print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||
WEEK_FILES=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {sum+=$6} END {print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||
|
||
line "Week peak: ${WEEK_PEAK}GB | flips: ${WEEK_FLIPS} | files cleaned: ${WEEK_FILES}"
|
||
line "Sessions: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD"
|
||
|
||
PEAK_INT=$(printf "%.0f" "$WEEK_PEAK" 2>/dev/null || echo 0)
|
||
WARN_INT=$(printf "%.0f" "$RAMDISK_WARN_GB" 2>/dev/null || echo 0)
|
||
if [[ "$PEAK_INT" -ge "$WARN_INT" ]]; then
|
||
finding "Transcode peak ${WEEK_PEAK}GB reached warn threshold — consider increasing RAMDISK_SIZE"
|
||
fi
|
||
if [[ "${WEEK_FLIPS:-0}" -ge "${TRANSCODE_FLIP_WARN:-3}" ]]; then
|
||
finding "Transcode flips this week: $WEEK_FLIPS — monitor ramdisk headroom"
|
||
fi
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 🎵 Media Activity ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "🎵 MEDIA ACTIVITY"
|
||
|
||
# Arr cleanup stats
|
||
if [[ -f "$ARR_CLEANUP_STATS" ]]; then
|
||
for arr in lidarr sonarr radarr; do
|
||
WEEK_ORPHANS=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \
|
||
'$1 >= cutoff && $2 == a {sum+=$3} END {print sum+0}' "$ARR_CLEANUP_STATS")
|
||
WEEK_BYTES=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \
|
||
'$1 >= cutoff && $2 == a {sum+=$4} END {print sum+0}' "$ARR_CLEANUP_STATS")
|
||
WEEK_TRACKED=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \
|
||
'BEGIN{max=0} $1 >= cutoff && $2 == a && $8 > max {max=$8} END {print max+0}' \
|
||
"$ARR_CLEANUP_STATS")
|
||
if [[ "${WEEK_ORPHANS:-0}" -gt 0 ]]; then
|
||
FREED=$(format_bytes "${WEEK_BYTES:-0}")
|
||
line "${arr^} cleanup: $WEEK_ORPHANS orphans removed ($FREED freed) | tracked: $WEEK_TRACKED files"
|
||
else
|
||
line "${arr^} cleanup: clean ✅ (tracked: $WEEK_TRACKED files)"
|
||
fi
|
||
done
|
||
else
|
||
line "Arr cleanup stats: no data yet (runs after first cleanup)"
|
||
fi
|
||
|
||
# Arr recovery stats
|
||
if [[ -f "$ARR_RECOVERY_STATS" ]]; then
|
||
WEEK_ACTIONED=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {sum+=$3} END {print sum+0}' "$ARR_RECOVERY_STATS")
|
||
WEEK_RUNS=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {count++} END {print count+0}' "$ARR_RECOVERY_STATS")
|
||
if [[ "${WEEK_ACTIONED:-0}" -gt 0 ]]; then
|
||
line "Arr recovery: $WEEK_ACTIONED items auto-recovered across $WEEK_RUNS runs"
|
||
else
|
||
line "Arr recovery: no failed imports this week ✅"
|
||
fi
|
||
fi
|
||
|
||
# Arr queue depth — current snapshot
|
||
for arr_name in "Sonarr|${HOST1_SONARR_URL}|${HOST1_SONARR_API_KEY}|v3" \
|
||
"Radarr|${HOST1_RADARR_URL}|${HOST1_RADARR_API_KEY}|v3" \
|
||
"Lidarr|${HOST1_LIDARR_URL}|${HOST1_LIDARR_API_KEY}|v1"; do
|
||
IFS='|' read -r name url key ver <<< "$arr_name"
|
||
if [[ "$url" == *"your-"* ]] || [[ -z "$key" ]]; then continue; fi
|
||
QUEUE=$(curl -sf --max-time 5 -H "X-Api-Key: $key" \
|
||
"${url}/api/${ver}/queue?pageSize=1" 2>/dev/null | \
|
||
python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('totalRecords',0))" \
|
||
2>/dev/null || echo "?")
|
||
if [[ "$QUEUE" == "0" ]] || [[ -z "$QUEUE" ]]; then
|
||
line "$name queue: empty ✅"
|
||
else
|
||
line "$name queue: $QUEUE items"
|
||
fi
|
||
done
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 🌐 Rsync ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "🌐 RSYNC"
|
||
|
||
if [[ -f "$BANDWIDTH_LOG" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
|
||
WEEK_TOTAL_BYTES=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {sum+=$3} END {print sum+0}' "$BANDWIDTH_LOG")
|
||
WEEK_SYNCS=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff' "$BANDWIDTH_LOG" | wc -l)
|
||
WEEK_FAILED=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff && $5 != "success"' "$BANDWIDTH_LOG" | wc -l)
|
||
WEEK_GB=$(awk "BEGIN {printf \"%.1f\", $WEEK_TOTAL_BYTES / 1073741824}")
|
||
|
||
line "Total transferred: ${WEEK_GB}GB across $WEEK_SYNCS syncs"
|
||
if [[ "${WEEK_FAILED:-0}" -gt 0 ]]; then
|
||
issue "Failed syncs this week: $WEEK_FAILED"
|
||
else
|
||
line "Sync failures: none ✅"
|
||
fi
|
||
|
||
# Top 3 shares by transfer this week
|
||
TOP_SHARES=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$1 >= cutoff {bytes[$4]+=$3} END {for(s in bytes) print bytes[s], s}' \
|
||
"$BANDWIDTH_LOG" | sort -rn | head -3)
|
||
if [[ -n "$TOP_SHARES" ]]; then
|
||
while IFS=' ' read -r bytes share; do
|
||
[[ -z "$share" ]] && continue
|
||
SIZE=$(format_bytes "$bytes")
|
||
line " → $share: $SIZE"
|
||
done <<< "$TOP_SHARES"
|
||
fi
|
||
else
|
||
line "No bandwidth data yet"
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 🛡️ Watchdog ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "🛡️ WATCHDOG"
|
||
|
||
# Container watchdog strikes
|
||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||
ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$" | wc -l)
|
||
if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then
|
||
STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
|
||
issue "Container watchdog: $ACTIVE_STRIKES active strikes — $STRIKE_LIST"
|
||
else
|
||
line "Container watchdog: no active strikes ✅"
|
||
fi
|
||
fi
|
||
|
||
# System watchdog strikes
|
||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||
SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$" | wc -l)
|
||
if [[ "$SYS_STRIKES" -gt 0 ]]; then
|
||
SYS_LIST=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
|
||
issue "System watchdog: $SYS_STRIKES active strikes — $SYS_LIST"
|
||
else
|
||
line "System watchdog: no active strikes ✅"
|
||
fi
|
||
fi
|
||
|
||
# Container skip list
|
||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||
SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ')
|
||
issue "Skip list: $SKIP_COUNT containers — $SKIP_LIST — manual intervention needed"
|
||
else
|
||
line "Skip list: empty ✅"
|
||
fi
|
||
|
||
# Container restarts this week
|
||
if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
|
||
WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l)
|
||
if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then
|
||
RESTARTED=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||
'$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
|
||
sort | uniq -c | sort -rn | head -5 | \
|
||
awk '{print $2 "(" $1 ")"}' | tr '\n' ' ')
|
||
finding "Watchdog restarted $WEEK_RESTARTS containers this week: $RESTARTED"
|
||
else
|
||
line "Watchdog restarts: none this week ✅"
|
||
fi
|
||
fi
|
||
|
||
# Docker container count
|
||
if command -v docker >/dev/null 2>&1; then
|
||
RUNNING=$(docker ps -q 2>/dev/null | wc -l)
|
||
TOTAL=$(docker ps -aq 2>/dev/null | wc -l)
|
||
STOPPED=$(( TOTAL - RUNNING ))
|
||
if [[ "$STOPPED" -gt 0 ]]; then
|
||
STOPPED_NAMES=$(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null | \
|
||
head -5 | tr '\n' ' ')
|
||
finding "Docker: $RUNNING/$TOTAL running — $STOPPED stopped: $STOPPED_NAMES"
|
||
else
|
||
line "Docker: $RUNNING/$TOTAL containers running ✅"
|
||
fi
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 🔐 Security ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "🔐 SECURITY"
|
||
|
||
for domain in "${CERT_MONITOR_DOMAINS[@]:-}"; do
|
||
[[ -z "$domain" ]] && continue
|
||
EXPIRY=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \
|
||
-connect "${domain}:443" -servername "$domain" 2>/dev/null | \
|
||
openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||
if [[ -z "$EXPIRY" ]]; then
|
||
issue "$domain: could not check certificate"
|
||
continue
|
||
fi
|
||
EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s 2>/dev/null || echo 0)
|
||
DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW) / 86400 ))
|
||
if [[ "$DAYS_LEFT" -le "${CERT_CRIT_DAYS:-7}" ]]; then
|
||
issue "$domain: ${DAYS_LEFT} days remaining — CRITICAL, renew now"
|
||
elif [[ "$DAYS_LEFT" -le "${CERT_WARN_DAYS:-30}" ]]; then
|
||
finding "$domain: ${DAYS_LEFT} days remaining — renew soon"
|
||
else
|
||
line "$domain: ${DAYS_LEFT} days remaining ✅"
|
||
fi
|
||
done
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ 📊 Emby ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "📊 EMBY"
|
||
|
||
# Select correct Emby URL and key for this host
|
||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||
EMBY_URL="$HOST1_EMBY_URL"
|
||
EMBY_KEY="$HOST1_EMBY_API_KEY"
|
||
else
|
||
EMBY_URL="$HOST2_EMBY_URL"
|
||
EMBY_KEY="$HOST2_EMBY_API_KEY"
|
||
fi
|
||
|
||
if [[ "$EMBY_KEY" != *"your-"* ]] && [[ -n "$EMBY_KEY" ]]; then
|
||
WEEK_MS=$(( 7 * 24 * 3600 * 10000000 ))
|
||
ACTIVITY=$(curl -sf --max-time 10 \
|
||
-H "X-Emby-Token: $EMBY_KEY" \
|
||
"${EMBY_URL}/Sessions?ControllableByUserId=&api_key=$EMBY_KEY" \
|
||
2>/dev/null)
|
||
|
||
# Use activity log for weekly stats
|
||
ACTIVITY_LOG=$(curl -sf --max-time 10 \
|
||
-H "X-Emby-Token: $EMBY_KEY" \
|
||
"${EMBY_URL}/user_usage_stats/user_activity?days=7&api_key=$EMBY_KEY" \
|
||
2>/dev/null)
|
||
|
||
if [[ -n "$ACTIVITY_LOG" ]]; then
|
||
TOTAL_PLAYS=$(echo "$ACTIVITY_LOG" | \
|
||
python3 -c "import sys,json; d=json.load(sys.stdin); \
|
||
print(sum(u.get('total_plays',0) for u in d))" 2>/dev/null || echo "?")
|
||
line "Streams this week: $TOTAL_PLAYS total plays"
|
||
|
||
# Top 3 users
|
||
TOP_USERS=$(echo "$ACTIVITY_LOG" | \
|
||
python3 -c "
|
||
import sys,json
|
||
d=json.load(sys.stdin)
|
||
users=sorted(d,key=lambda x:x.get('total_plays',0),reverse=True)[:3]
|
||
for u in users:
|
||
print(f\" → {u.get('user_name','?')}: {u.get('total_plays',0)} plays\")
|
||
" 2>/dev/null)
|
||
[[ -n "$TOP_USERS" ]] && echo "$TOP_USERS" | while IFS= read -r l; do line "$l"; done
|
||
else
|
||
# Fallback — just show active sessions
|
||
ACTIVE=$(curl -sf --max-time 5 \
|
||
-H "X-Emby-Token: $EMBY_KEY" \
|
||
"${EMBY_URL}/Sessions?api_key=$EMBY_KEY" 2>/dev/null | \
|
||
python3 -c "import sys,json; \
|
||
d=json.load(sys.stdin); \
|
||
active=[s for s in d if s.get('NowPlayingItem')]; \
|
||
print(f'{len(active)} active streams now')" 2>/dev/null || echo "API unavailable")
|
||
line "Emby: $ACTIVE"
|
||
fi
|
||
else
|
||
line "Emby: API key not configured"
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ ⚙️ System Health ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
section "⚙️ SYSTEM HEALTH"
|
||
|
||
# SMART summary
|
||
if command -v smartctl >/dev/null 2>&1; then
|
||
SMART_ISSUES=0
|
||
for disk in /dev/sd? /dev/nvme?; do
|
||
[[ ! -e "$disk" ]] && continue
|
||
DISK_NAME=$(basename "$disk")
|
||
SKIP=false
|
||
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
|
||
[[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break
|
||
done
|
||
[[ "$SKIP" == true ]] && continue
|
||
HEALTH=$(smartctl -H "$disk" 2>/dev/null | grep "SMART overall-health" | \
|
||
awk '{print $NF}')
|
||
if [[ "$HEALTH" != "PASSED" ]] && [[ -n "$HEALTH" ]]; then
|
||
issue "SMART $DISK_NAME: $HEALTH — check immediately"
|
||
((SMART_ISSUES++))
|
||
fi
|
||
done
|
||
if [[ "$SMART_ISSUES" -eq 0 ]]; then
|
||
line "SMART: all drives PASSED ✅"
|
||
fi
|
||
fi
|
||
|
||
# Gitea sync status
|
||
if command -v git >/dev/null 2>&1 && [[ -d "$TARGET_DIR/.git" ]]; then
|
||
CURRENT_COMMIT=$(git -C "$TARGET_DIR" rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||
LAST_PULL=$(git -C "$TARGET_DIR" log -1 --format="%ar" 2>/dev/null || echo "unknown")
|
||
line "Gitea: commit $CURRENT_COMMIT (pulled $LAST_PULL)"
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ ⚠️ Issues Requiring Attention ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
if [[ ${#ISSUES[@]} -gt 0 ]]; then
|
||
REPORT+=("")
|
||
REPORT+=("⚠️ ISSUES REQUIRING ATTENTION")
|
||
REPORT+=("$(printf '%.0s─' {1..50})")
|
||
for issue_line in "${ISSUES[@]}"; do
|
||
REPORT+=(" ❌ $issue_line")
|
||
done
|
||
fi
|
||
|
||
if [[ ${#FINDINGS[@]} -gt 0 ]]; then
|
||
REPORT+=("")
|
||
REPORT+=("ℹ️ NOTABLE")
|
||
REPORT+=("$(printf '%.0s─' {1..50})")
|
||
for finding_line in "${FINDINGS[@]}"; do
|
||
REPORT+=(" → $finding_line")
|
||
done
|
||
fi
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ Footer ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
REPORT+=("")
|
||
if [[ ${#ISSUES[@]} -eq 0 ]]; then
|
||
STATUS="✅ All systems healthy — enjoy your Sunday"
|
||
else
|
||
STATUS="⚠️ ${#ISSUES[@]} issue(s) need attention"
|
||
fi
|
||
REPORT+=("$STATUS")
|
||
REPORT+=("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
||
|
||
# -----------------------------------------------------------------------------------------------
|
||
# ━━━ Output and Send ━━━
|
||
# -----------------------------------------------------------------------------------------------
|
||
HEADER="☕ SUNDAY MORNING COFFEE REPORT — $REPORT_DATE"
|
||
DIVIDER="$(printf '%.0s━' {1..50})"
|
||
|
||
echo ""
|
||
echo "$DIVIDER"
|
||
echo "$HEADER"
|
||
echo "$DIVIDER"
|
||
|
||
for report_line in "${REPORT[@]}"; do
|
||
echo "$report_line"
|
||
done
|
||
|
||
echo ""
|
||
|
||
# Send notification
|
||
BODY=$(printf '%s\n' "$HEADER" "${REPORT[@]}")
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — notification not sent"
|
||
else
|
||
notify "$BODY" "☕ Weekly Report" "normal"
|
||
success "Report sent"
|
||
fi |