Gmer4Lfe
95151c2278
Move all watchdog scripts to a dedicated Watchdogs/ folder: Docker_Essentials/docker_watchdog.sh → Watchdogs/ unRAID_Essentials/system_watchdog.sh → Watchdogs/ unRAID_Essentials/resource_watchdog.sh → Watchdogs/ Orchestrators/watchdog_orchestrator.sh → Watchdogs/ Tools/watchdog_skip_list_manager.sh → Watchdogs/ Rename host config files: master_host1.conf → host1.conf master_host2.conf → host2.conf Update all references across the ecosystem: master.conf: WATCHDOG_ORCHESTRATOR_SCRIPTS paths → Watchdogs/ load_config.sh: host*.conf glob + all comments git_pull_execute.sh: sparse checkout glob + all comments Partnership/ssh_setup.sh: HOST_CONF path construction user_script_plug-in.sh: all script paths + per-host conf path common.sh, README.md, README-User_Script_Plug-in.md: comment refs All Partnership, Fallback, Monitors, Transcodes, Tools scripts: comment refs
964 lines
45 KiB
Bash
964 lines
45 KiB
Bash
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ============================= Sunday Morning Coffee Report ===================================
|
||
# ==============================================================================================
|
||
# Weekly system overview — everything that happened this week in one clean read.
|
||
# Designed to be read over coffee Sunday morning while the system is fully caught up
|
||
# from the 2:30am maintenance window.
|
||
# Schedule: 0 7 * * 0 (7am Sunday — after weekly_sync_maintenance.sh finishes at ~3am)
|
||
# Maintenance window completes → 4 hours of fresh data → report ready ☕
|
||
#
|
||
# ── SECTIONS ──────────────────────────────────────────────────────────────────────────────────
|
||
# 🖥️ System — uptime, memory, boot drive, cache drive, reboots this week
|
||
# 📀 Array — disk count, parity status, ZFS health, drive temps
|
||
# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split
|
||
# 🎵 Media Activity — arr cleanup stats, arr recovery stats, queue depth
|
||
# 🌐 Rsync — weekly transfer totals, per-share breakdown, failures
|
||
# 🛡️ Watchdog — resource watchdog, system watchdog, docker watchdog, fallback state
|
||
# 🔐 Security — SSL cert expiry per domain
|
||
# 📊 Emby — weekly stream count, active now, top users
|
||
# ⚙️ System Health — SMART summary, inotify, php-fpm, Docker, Gitea sync
|
||
# ⚠️ Issues — anything requiring attention collected from above sections
|
||
#
|
||
# ── DATA SOURCES (reads only) ─────────────────────────────────────────────────────────────────
|
||
# DATA_DIR stats files — arr cleanup, recovery, transcode, bandwidth history
|
||
# /boot/config — fallback state, watchdog reboot log
|
||
# /tmp — watchdog strike state files
|
||
# /proc, /sys — system memory, uptime, inotify
|
||
# /var/local/emhttp/ — unRAID array info
|
||
# Emby API — session history, active streams
|
||
# Arr APIs — current queue depth (SONARR_URL etc. from detect_hosts())
|
||
# tailscale — remote server reachability
|
||
# openssl — live SSL cert check per domain
|
||
# smartctl — drive SMART health
|
||
#
|
||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||
# detect_hosts() sets MY_ID and aliases all HOST*_ vars.
|
||
# Report header and footer show MY_ID — clear which server's weekly report this is.
|
||
# No manual HOST1/HOST2 comparisons — all via MY_ID/REMOTE_ID.
|
||
#
|
||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||
# Root check — smartctl, docker, openssl need root
|
||
# acquire_lock — prevents duplicate reports
|
||
# detect_hosts() — correct vars per server
|
||
# DOCKER_TIMEOUT — all docker calls protected
|
||
# validate_unraid_cmd — notify validated before use
|
||
# openssl check — security section skipped gracefully if not available
|
||
#
|
||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||
# NOTIFY_UNRAID — send via unRAID notification system
|
||
# DISCORD_WEBHOOK — send to Discord channel (MY_ID_ prefixed per host)
|
||
# CERT_MONITOR_DOMAINS / CERT_WARN_DAYS / CERT_CRIT_DAYS
|
||
# ZFS_REPORT_IGNORE_POOLS / SMART_IGNORE_DRIVES
|
||
# All threshold vars read from master.conf at runtime
|
||
#
|
||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||
# sunday_morning_coffee_report.sh — generate and send report
|
||
# sunday_morning_coffee_report.sh --dry-run — generate without sending notification
|
||
# sunday_morning_coffee_report.sh --status — show data file availability
|
||
# sunday_morning_coffee_report.sh --log — verbose section output
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
DOCKER_TIMEOUT=15
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root — smartctl and docker require root"
|
||
exit 1
|
||
fi
|
||
|
||
validate_unraid_cmd \
|
||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||
"" "" \
|
||
"unRAID notify script" || warn "unRAID notify script not found — unRAID notifications disabled"
|
||
|
||
acquire_lock
|
||
|
||
detect_hosts
|
||
|
||
REPORT_DATE=$(date '+%A, %B %-d, %Y')
|
||
WEEK_START=$(date -d "7 days ago" '+%Y-%m-%d')
|
||
WEEK_EPOCH=$(date -d "$WEEK_START" +%s)
|
||
TODAY=$(date '+%Y-%m-%d')
|
||
NOW=$(date +%s)
|
||
|
||
REPORT=()
|
||
ISSUES=()
|
||
FINDINGS=()
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY COFFEE REPORT STATUS ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo ""
|
||
echo "━━━ Data Files ━━━"
|
||
for f in \
|
||
"$ARR_CLEANUP_STATS:arr cleanup stats" \
|
||
"$ARR_RECOVERY_STATS:arr recovery stats" \
|
||
"$TRANSCODE_DAILY_LOG:transcode daily log" \
|
||
"$BANDWIDTH_LOG:bandwidth log" \
|
||
"$TUNING_MONITOR_LOG:tuning monitor log" \
|
||
"$SYS_WATCHDOG_STATE_FILE:system watchdog state" \
|
||
"$SYS_WATCHDOG_REBOOT_LOG:watchdog reboot log" \
|
||
"$WATCHDOG_STATE_FILE:docker watchdog state" \
|
||
"$WATCHDOG_CONTAINER_RESTART_LOG:container restart log" \
|
||
"$FALLBACK_STATE_FILE:fallback state"; do
|
||
path="${f%%:*}"
|
||
label="${f##*:}"
|
||
if [[ -f "$path" ]] && [[ -s "$path" ]]; then
|
||
COUNT=$(wc -l < "$path" 2>/dev/null || echo "?")
|
||
echo " $ICON_SUCCESS $label ($COUNT lines)"
|
||
elif [[ -f "$path" ]]; then
|
||
echo " $ICON_WARN $label (exists but empty)"
|
||
else
|
||
echo " $ICON_SKIP $label (not found)"
|
||
fi
|
||
done
|
||
echo ""
|
||
echo "━━━ Runtime Dependencies ━━━"
|
||
command -v smartctl >/dev/null 2>&1 && echo " $ICON_SUCCESS smartctl" || \
|
||
echo " $ICON_SKIP smartctl (not installed)"
|
||
command -v openssl >/dev/null 2>&1 && echo " $ICON_SUCCESS openssl" || \
|
||
echo " $ICON_SKIP openssl (security section disabled)"
|
||
command -v docker >/dev/null 2>&1 && echo " $ICON_SUCCESS docker" || \
|
||
echo " $ICON_SKIP docker"
|
||
command -v zpool >/dev/null 2>&1 && echo " $ICON_SUCCESS zpool" || \
|
||
echo " $ICON_SKIP zpool (ZFS section disabled)"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — report generated, notification not sent"
|
||
|
||
# ==============================================================================================
|
||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
section() {
|
||
REPORT+=("")
|
||
REPORT+=("$1")
|
||
REPORT+=("$(printf '%.0s─' {1..50})")
|
||
}
|
||
|
||
line() { REPORT+=(" $1"); }
|
||
issue() { ISSUES+=("$1"); REPORT+=(" ⚠️ $1"); }
|
||
finding() { FINDINGS+=("$1"); REPORT+=(" ℹ️ $1"); }
|
||
|
||
get_array() { eval "echo \"\${${1}[*]}\""; }
|
||
|
||
format_bytes() {
|
||
local bytes=$1
|
||
if (( bytes > 1073741824 )); then
|
||
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
||
elif (( bytes > 1048576 )); then
|
||
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
||
else
|
||
echo "${bytes}B"
|
||
fi
|
||
}
|
||
|
||
# Lock/uptime helpers — local copies needed in report context
|
||
_get_lock_pid() {
|
||
local f="$LOCK_DIR/${1}.lock"
|
||
[[ -f "$f" ]] && { local c; c=$(cat "$f" 2>/dev/null); echo "${c%%:*}"; }
|
||
}
|
||
_get_lock_name() {
|
||
local f="$LOCK_DIR/${1}.lock"
|
||
[[ -f "$f" ]] && { local c; c=$(cat "$f" 2>/dev/null); echo "${c##*:}"; }
|
||
}
|
||
_is_running() {
|
||
local pid name
|
||
pid=$(_get_lock_pid "$1"); name=$(_get_lock_name "$1")
|
||
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$name" == "$1" ]]
|
||
}
|
||
_lock_age() {
|
||
local f="$LOCK_DIR/${1}.lock"
|
||
[[ -f "$f" ]] && echo $(( NOW - $(stat -c %Y "$f" 2>/dev/null || echo "$NOW") )) || echo 0
|
||
}
|
||
_fmt_uptime() {
|
||
local s=$1 d=$(($1/86400)) h=$((($1%86400)/3600)) m=$((($1%3600)/60))
|
||
(( d > 0 )) && echo "${d}d ${h}h ${m}m" || \
|
||
(( h > 0 )) && echo "${h}h ${m}m" || echo "${m}m"
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ 🖥️ SYSTEM ━━━
|
||
# ==============================================================================================
|
||
section "🖥️ SYSTEM — $MY_ID ($LOCAL_SERVER_NAME)"
|
||
|
||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||
UPTIME_DAYS=$(( UPTIME_SECONDS / 86400 ))
|
||
UPTIME_HOURS=$(( (UPTIME_SECONDS % 86400) / 3600 ))
|
||
BOOT_TIME=$(date -d "@$(( NOW - UPTIME_SECONDS ))" '+%A %-d %b at %-I:%M%p')
|
||
line "Uptime: ${UPTIME_DAYS}d ${UPTIME_HOURS}hr (up since $BOOT_TIME)"
|
||
|
||
REBOOT_COUNT=0
|
||
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
|
||
REBOOT_COUNT=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \
|
||
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
|
||
fi
|
||
[[ "${REBOOT_COUNT:-0}" -gt 0 ]] && \
|
||
issue "Reboots this week: $REBOOT_COUNT (system watchdog triggered)" || \
|
||
line "Reboots this week: 0 ✅"
|
||
|
||
MEM_TOTAL_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
|
||
MEM_AVAIL_KB=$(awk '/MemAvailable/{print $2}' /proc/meminfo)
|
||
MEM_USED_GB=$(awk "BEGIN {printf \"%.1f\", ($MEM_TOTAL_KB - $MEM_AVAIL_KB) / 1048576}")
|
||
MEM_TOTAL_GB=$(awk "BEGIN {printf \"%.0f\", $MEM_TOTAL_KB / 1048576}")
|
||
ARC_SIZE="n/a"
|
||
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||
ARC_B=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||
ARC_SIZE=$(awk "BEGIN {printf \"%.1f\", $ARC_B / 1073741824}")
|
||
fi
|
||
line "Memory: ${MEM_USED_GB}GB used / ${MEM_TOTAL_GB}GB total (ARC: ${ARC_SIZE}GB)"
|
||
|
||
BOOT_PCT=$(df /boot --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||
BOOT_USED=$(df /boot -h --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||
BOOT_SIZE=$(df /boot -h --output=size 2>/dev/null | tail -1 | tr -d ' ')
|
||
[[ "${BOOT_PCT:-0}" -ge 80 ]] && \
|
||
issue "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE}) — getting full" || \
|
||
line "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE})"
|
||
|
||
CACHE_PCT=$(df /mnt/cache --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||
CACHE_AVAIL=$(df /mnt/cache -h --output=avail 2>/dev/null | tail -1 | tr -d ' ')
|
||
CACHE_SIZE=$(df /mnt/cache -h --output=size 2>/dev/null | tail -1 | tr -d ' ')
|
||
[[ "${CACHE_PCT:-0}" -ge 85 ]] && \
|
||
issue "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})" || \
|
||
line "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ 📀 ARRAY ━━━
|
||
# ==============================================================================================
|
||
section "📀 ARRAY"
|
||
|
||
if [[ -f /var/local/emhttp/disks.ini ]]; then
|
||
DISK_COUNT=$(grep -c '^\["disk[0-9]' /var/local/emhttp/disks.ini 2>/dev/null || echo "?")
|
||
PARITY_COUNT=$(grep -c '^\["parity' /var/local/emhttp/disks.ini 2>/dev/null || echo "?")
|
||
DISK_COUNT="${DISK_COUNT//[^0-9]/}"; DISK_COUNT="${DISK_COUNT:-?}"
|
||
PARITY_COUNT="${PARITY_COUNT//[^0-9]/}"; PARITY_COUNT="${PARITY_COUNT:-?}"
|
||
line "Array: ${DISK_COUNT} data disks + ${PARITY_COUNT} parity"
|
||
else
|
||
line "Array: disks.ini not found"
|
||
fi
|
||
|
||
PARITY_LOG="/boot/config/parity-checks.log"
|
||
if [[ -f "$PARITY_LOG" ]]; then
|
||
LAST_CHECK=$(tail -1 "$PARITY_LOG" 2>/dev/null)
|
||
if [[ -n "$LAST_CHECK" ]]; then
|
||
PARITY_DATE=$(echo "$LAST_CHECK" | cut -d'|' -f1 | xargs)
|
||
PARITY_ERRORS=$(echo "$LAST_CHECK" | cut -d'|' -f4)
|
||
PARITY_ACTION=$(echo "$LAST_CHECK" | cut -d'|' -f6)
|
||
PARITY_DURATION=$(echo "$LAST_CHECK" | cut -d'|' -f2)
|
||
PARITY_DUR_HR=$(( PARITY_DURATION / 3600 ))
|
||
PARITY_DUR_MIN=$(( (PARITY_DURATION % 3600) / 60 ))
|
||
if [[ "${PARITY_ERRORS:-0}" -gt 0 ]]; then
|
||
issue "Parity: $PARITY_ERRORS errors on last check ($PARITY_DATE)"
|
||
elif [[ "${PARITY_ERRORS:-0}" -lt 0 ]]; then
|
||
CORRECTIONS=$(( PARITY_ERRORS * -1 ))
|
||
line "Parity: OK — last: $PARITY_DATE ($PARITY_ACTION, ${PARITY_DUR_HR}h${PARITY_DUR_MIN}m, $CORRECTIONS correction(s)) ✅"
|
||
else
|
||
line "Parity: OK — last: $PARITY_DATE ($PARITY_ACTION, ${PARITY_DUR_HR}h${PARITY_DUR_MIN}m, 0 errors) ✅"
|
||
fi
|
||
RESYNC=$(grep "^mdResync=" /var/local/emhttp/var.ini 2>/dev/null | cut -d= -f2 | tr -d '"')
|
||
[[ "$RESYNC" != "0" ]] && finding "Parity check currently in progress"
|
||
fi
|
||
else
|
||
line "Parity: no check log found"
|
||
fi
|
||
|
||
if command -v zpool >/dev/null 2>&1; then
|
||
while IFS=$'\t' read -r pool health; do
|
||
[[ -z "$pool" ]] && continue
|
||
SKIP=false
|
||
for ignore in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do
|
||
[[ "$pool" == "$ignore" ]] && SKIP=true && break
|
||
done
|
||
[[ "$SKIP" == true ]] && continue
|
||
[[ "$health" == "ONLINE" ]] && \
|
||
line "ZFS $pool: ONLINE ✅" || \
|
||
issue "ZFS $pool: $health — check immediately"
|
||
done < <(zpool list -H -o name,health 2>/dev/null)
|
||
fi
|
||
|
||
if command -v smartctl >/dev/null 2>&1; then
|
||
declare -A DISK_TEMPS
|
||
ALL_NORMAL=true
|
||
get_unraid_temp_thresholds
|
||
|
||
for disk in /dev/sd? /dev/nvme?; do
|
||
[[ ! -e "$disk" ]] && continue
|
||
DISK_NAME=$(basename "$disk")
|
||
SKIP=false
|
||
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
|
||
[[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break
|
||
done
|
||
[[ "$SKIP" == true ]] && continue
|
||
|
||
TEMP=$(smartctl -A "$disk" 2>/dev/null | awk '
|
||
/^190 / || /^194 / { print $10; exit }
|
||
/Temperature_Celsius/ { print $10; exit }
|
||
')
|
||
[[ -z "$TEMP" ]] && TEMP=$(smartctl -A "$disk" 2>/dev/null | \
|
||
awk '/^Temperature:/ { print $2; exit }')
|
||
TEMP="${TEMP//[^0-9]/}"
|
||
[[ -z "$TEMP" ]] && continue
|
||
|
||
DISK_TEMPS["$DISK_NAME"]="$TEMP"
|
||
|
||
if is_ssd "$disk"; then
|
||
WARN_T="${UNRAID_SSD_HOT:-50}"; CRIT_T="${UNRAID_SSD_MAX:-60}"
|
||
else
|
||
WARN_T="${UNRAID_DISK_HOT:-45}"; CRIT_T="${UNRAID_DISK_MAX:-55}"
|
||
fi
|
||
|
||
if [[ "$TEMP" -ge "$CRIT_T" ]]; then
|
||
issue "Drive $DISK_NAME: ${TEMP}°C — CRITICAL"
|
||
ALL_NORMAL=false
|
||
elif [[ "$TEMP" -ge "$WARN_T" ]]; then
|
||
finding "Drive $DISK_NAME: ${TEMP}°C — warm"
|
||
ALL_NORMAL=false
|
||
fi
|
||
done
|
||
|
||
[[ "$ALL_NORMAL" == true ]] && line "Drive temps: all normal ✅"
|
||
|
||
if [[ -f /var/local/emhttp/disks.ini ]] && [[ ${#DISK_TEMPS[@]} -gt 0 ]]; then
|
||
DISK_SUMMARY=""
|
||
CURRENT_DISK=""
|
||
while IFS= read -r ini_line; do
|
||
if echo "$ini_line" | grep -qE '^\["(disk[0-9]+|parity[0-9]?|cache)"\]'; then
|
||
CURRENT_DISK=$(echo "$ini_line" | grep -o '"[^"]*"' | head -1 | tr -d '"')
|
||
elif echo "$ini_line" | grep -q '^device='; then
|
||
DEV=$(echo "$ini_line" | cut -d= -f2 | tr -d '"')
|
||
TEMP="${DISK_TEMPS[$DEV]:-}"
|
||
if [[ -n "$TEMP" ]]; then
|
||
[[ -n "$DISK_SUMMARY" ]] && DISK_SUMMARY+=", "
|
||
DISK_SUMMARY+="${CURRENT_DISK}(${DEV}):${TEMP}°C"
|
||
fi
|
||
fi
|
||
done < /var/local/emhttp/disks.ini
|
||
[[ -n "$DISK_SUMMARY" ]] && line "Temps: $DISK_SUMMARY"
|
||
fi
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ 🎬 TRANSCODES ━━━
|
||
# ==============================================================================================
|
||
section "🎬 TRANSCODES"
|
||
|
||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||
RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail 2>/dev/null | tail -1 | tr -d ' ')
|
||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
|
||
RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}")
|
||
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null | xargs basename 2>/dev/null || echo "unknown")
|
||
line "Ramdisk now: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET"
|
||
else
|
||
issue "Ramdisk not mounted at $RAMDISK_PATH"
|
||
fi
|
||
|
||
if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]] && [[ -s "$TRANSCODE_DAILY_LOG" ]]; then
|
||
WEEK_PEAK=$(awk -F'|' -v c="$WEEK_START" '$1>=c {if($2>max)max=$2} END{printf "%.2f",max+0}' "$TRANSCODE_DAILY_LOG")
|
||
WEEK_FLIPS=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$3} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||
WEEK_RAM=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$4} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||
WEEK_SSD=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$5} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||
line "Week peak: ${WEEK_PEAK}GB | flips: ${WEEK_FLIPS}"
|
||
line "Sessions: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD"
|
||
PEAK_INT=$(printf "%.0f" "$WEEK_PEAK" 2>/dev/null || echo 0)
|
||
WARN_INT=$(printf "%.0f" "${RAMDISK_WARN_GB:-6.8}" 2>/dev/null || echo 0)
|
||
[[ "$PEAK_INT" -ge "$WARN_INT" ]] && \
|
||
finding "Transcode peak ${WEEK_PEAK}GB near threshold — consider increasing HOST*_RAMDISK_SIZE"
|
||
[[ "${WEEK_FLIPS:-0}" -ge "${TRANSCODE_FLIP_WARN:-3}" ]] && \
|
||
finding "Transcode flips this week: $WEEK_FLIPS — monitor ramdisk headroom"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ 🎵 MEDIA ACTIVITY ━━━
|
||
# ==============================================================================================
|
||
section "🎵 MEDIA ACTIVITY"
|
||
|
||
if [[ -f "${ARR_CLEANUP_STATS:-}" ]] && [[ -s "$ARR_CLEANUP_STATS" ]]; then
|
||
for arr in lidarr sonarr radarr; do
|
||
WEEK_ORPHANS=$(awk -F'|' -v c="$WEEK_START" -v a="$arr" \
|
||
'$1>=c && $2==a {sum+=$3} END{print sum+0}' "$ARR_CLEANUP_STATS")
|
||
WEEK_BYTES=$(awk -F'|' -v c="$WEEK_START" -v a="$arr" \
|
||
'$1>=c && $2==a {sum+=$4} END{print sum+0}' "$ARR_CLEANUP_STATS")
|
||
WEEK_TRACKED=$(awk -F'|' -v c="$WEEK_START" -v a="$arr" \
|
||
'BEGIN{max=0} $1>=c && $2==a && $8+0>max {max=$8} END{print max+0}' "$ARR_CLEANUP_STATS")
|
||
if [[ "${WEEK_ORPHANS:-0}" -gt 0 ]]; then
|
||
FREED=$(format_bytes "${WEEK_BYTES:-0}")
|
||
line "${arr^} cleanup: $WEEK_ORPHANS orphans removed ($FREED freed) | tracked: $WEEK_TRACKED files"
|
||
else
|
||
line "${arr^} cleanup: clean ✅ (tracked: $WEEK_TRACKED files)"
|
||
fi
|
||
done
|
||
else
|
||
line "Arr cleanup stats: no data yet (runs after first weekly cleanup)"
|
||
fi
|
||
|
||
if [[ -f "${ARR_RECOVERY_STATS:-}" ]] && [[ -s "$ARR_RECOVERY_STATS" ]]; then
|
||
WEEK_ACTIONED=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$3} END{print sum+0}' "$ARR_RECOVERY_STATS")
|
||
WEEK_RUNS=$(awk -F'|' -v c="$WEEK_START" '$1>=c {count++} END{print count+0}' "$ARR_RECOVERY_STATS")
|
||
if [[ "${WEEK_ACTIONED:-0}" -gt 0 ]]; then
|
||
line "Arr recovery: $WEEK_ACTIONED items auto-recovered across $WEEK_RUNS runs"
|
||
else
|
||
line "Arr recovery: no failed imports this week ✅"
|
||
fi
|
||
fi
|
||
|
||
# Arr queue depth — uses detect_hosts() aliased SONARR_URL/RADARR_URL/LIDARR_URL
|
||
for arr_entry in "Sonarr|${SONARR_URL:-}|${SONARR_API_KEY:-}|v3" \
|
||
"Radarr|${RADARR_URL:-}|${RADARR_API_KEY:-}|v3" \
|
||
"Lidarr|${LIDARR_URL:-}|${LIDARR_API_KEY:-}|v1"; do
|
||
IFS='|' read -r name url key ver <<< "$arr_entry"
|
||
[[ -z "$url" || -z "$key" ]] && continue
|
||
QUEUE=$(curl -sf --max-time 5 -H "X-Api-Key: $key" \
|
||
"${url}/api/${ver}/queue?pageSize=1" 2>/dev/null | \
|
||
grep -o '"totalRecords":[0-9]*' | grep -o '[0-9]*' || echo "?")
|
||
if [[ "$QUEUE" == "0" || -z "$QUEUE" ]]; then
|
||
line "$name queue: empty ✅"
|
||
elif [[ "$QUEUE" == "?" ]]; then
|
||
finding "$name queue: API unavailable"
|
||
else
|
||
line "$name queue: $QUEUE items"
|
||
fi
|
||
done
|
||
|
||
# ==============================================================================================
|
||
# ━━━ 🌐 RSYNC ━━━
|
||
# ==============================================================================================
|
||
section "🌐 RSYNC"
|
||
|
||
# New bandwidth log format: date|time|profile|duration|status|bytes|warn_flag
|
||
if [[ -f "${BANDWIDTH_LOG:-}" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
|
||
WEEK_TOTAL_BYTES=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$6} END{print sum+0}' "$BANDWIDTH_LOG")
|
||
WEEK_SYNCS=$(awk -F'|' -v c="$WEEK_START" '$1>=c' "$BANDWIDTH_LOG" | wc -l)
|
||
WEEK_FAILED=$(awk -F'|' -v c="$WEEK_START" '$1>=c && $5!="success"' "$BANDWIDTH_LOG" | wc -l)
|
||
WEEK_LARGE=$(awk -F'|' -v c="$WEEK_START" '$1>=c && $7=="LARGE"' "$BANDWIDTH_LOG" | wc -l)
|
||
WEEK_GB=$(awk "BEGIN {printf \"%.1f\", $WEEK_TOTAL_BYTES / 1073741824}")
|
||
line "Total: ${WEEK_GB}GB across $WEEK_SYNCS syncs"
|
||
[[ "${WEEK_FAILED:-0}" -gt 0 ]] && issue "Failed syncs this week: $WEEK_FAILED" || line "Sync failures: none ✅"
|
||
[[ "${WEEK_LARGE:-0}" -gt 0 ]] && finding "Large transfers (>${BANDWIDTH_WARN_GB}GB): $WEEK_LARGE"
|
||
|
||
# Top 3 profiles by transfer — profile is field $3
|
||
TOP_SHARES=$(awk -F'|' -v c="$WEEK_START" \
|
||
'$1>=c {bytes[$3]+=$6} END {for(s in bytes) print bytes[s], s}' \
|
||
"$BANDWIDTH_LOG" | sort -rn | head -3)
|
||
if [[ -n "$TOP_SHARES" ]]; then
|
||
while IFS=' ' read -r bytes profile; do
|
||
[[ -z "$profile" ]] && continue
|
||
line " → $profile: $(format_bytes "$bytes")"
|
||
done <<< "$TOP_SHARES"
|
||
fi
|
||
else
|
||
line "No bandwidth data yet"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ 🛡️ WATCHDOG ━━━
|
||
# ==============================================================================================
|
||
section "🛡️ WATCHDOG"
|
||
|
||
# ── System Watchdog ───────────────────────────────────────────────────────────────────────────
|
||
line "⚙️ System Watchdog (cron via watchdog_orchestrator)"
|
||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||
_sw_last=$(stat -c %Y "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || echo 0)
|
||
_sw_ago=$(( NOW - _sw_last ))
|
||
if [[ "$_sw_ago" -lt 600 ]]; then
|
||
line " ✅ Last run: $(_fmt_uptime "$_sw_ago") ago"
|
||
else
|
||
issue " Last run: $(_fmt_uptime "$_sw_ago") ago — watchdog_orchestrator may not be running"
|
||
fi
|
||
else
|
||
issue " system_watchdog has never run (state file missing)"
|
||
fi
|
||
|
||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||
SYS_ACTIVE=$(grep -v ":0$\|^watchdog_cycle=" \
|
||
"$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||
if [[ -n "$SYS_ACTIVE" ]]; then
|
||
while IFS=: read -r key count; do
|
||
[[ -z "$key" ]] && continue
|
||
issue " Strike: $key — $count/$SYS_WATCHDOG_STRIKE_LIMIT"
|
||
done <<< "$SYS_ACTIVE"
|
||
else
|
||
line " ✅ Strikes: none"
|
||
fi
|
||
fi
|
||
|
||
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
|
||
WD_REBOOTS=$(awk -v c="$WEEK_EPOCH" '$1>=c' "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
|
||
[[ "${WD_REBOOTS:-0}" -gt 0 ]] && \
|
||
issue " Watchdog reboots this week: $WD_REBOOTS" || \
|
||
line " ✅ Watchdog reboots this week: 0"
|
||
fi
|
||
|
||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||
SKIP_LIST=$(tr '\n' ' ' < "$SYS_WATCHDOG_FAILED_FILE")
|
||
issue " Skip list ($SKIP_COUNT): $SKIP_LIST"
|
||
else
|
||
line " ✅ Skip list: empty"
|
||
fi
|
||
|
||
ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||
MEM_AVAIL_GB=$(awk '/MemAvailable/{printf "%.1f",$2/1048576}' /proc/meminfo)
|
||
MEM_TOTAL_GB_SYS=$(awk '/MemTotal/{printf "%.0f",$2/1048576}' /proc/meminfo)
|
||
LOAD_NOW=$(awk '{print $1}' /proc/loadavg)
|
||
ZOMBIE_NOW=$(ps aux 2>/dev/null | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0)
|
||
ZOMBIE_NOW="${ZOMBIE_NOW//[^0-9]/}"; ZOMBIE_NOW="${ZOMBIE_NOW:-0}"
|
||
ARC_NOW="n/a"
|
||
[[ -f /proc/spl/kstat/zfs/arcstats ]] && \
|
||
ARC_NOW=$(awk '/^size /{printf "%.1f",$3/1073741824}' /proc/spl/kstat/zfs/arcstats)
|
||
line " 📊 rootfs:${ROOTFS_PCT}% │ RAM:${MEM_AVAIL_GB}GB free/${MEM_TOTAL_GB_SYS}GB │ ARC:${ARC_NOW}GB │ load:${LOAD_NOW} │ zombies:${ZOMBIE_NOW}"
|
||
|
||
REPORT+=("")
|
||
|
||
# ── Docker Watchdog ───────────────────────────────────────────────────────────────────────────
|
||
line "🐳 Docker Watchdog (cron via watchdog_orchestrator)"
|
||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||
_dw_last=$(stat -c %Y "$WATCHDOG_STATE_FILE" 2>/dev/null || echo 0)
|
||
_dw_ago=$(( NOW - _dw_last ))
|
||
if [[ "$_dw_ago" -lt 600 ]]; then
|
||
line " ✅ Last run: $(_fmt_uptime "$_dw_ago") ago"
|
||
else
|
||
issue " Last run: $(_fmt_uptime "$_dw_ago") ago — watchdog_orchestrator may not be running"
|
||
fi
|
||
else
|
||
issue " docker_watchdog has never run (state file missing)"
|
||
fi
|
||
|
||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||
DOCK_ACTIVE=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||
if [[ -n "$DOCK_ACTIVE" ]]; then
|
||
while IFS=: read -r key count; do
|
||
[[ -z "$key" ]] && continue
|
||
issue " Strike: $key — $count"
|
||
done <<< "$DOCK_ACTIVE"
|
||
else
|
||
line " ✅ Container strikes: none"
|
||
fi
|
||
fi
|
||
|
||
if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
|
||
WK_RESTARTS=$(awk -F'|' -v c="$WEEK_START" '$2>=c' \
|
||
"$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l)
|
||
if [[ "${WK_RESTARTS:-0}" -gt 0 ]]; then
|
||
RESTARTED=$(awk -F'|' -v c="$WEEK_START" '$2>=c{print $1}' \
|
||
"$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
|
||
sort | uniq -c | sort -rn | head -5 | \
|
||
awk '{print $2"("$1")"}' | tr '\n' ' ')
|
||
finding " Container restarts this week: $WK_RESTARTS — $RESTARTED"
|
||
else
|
||
line " ✅ Container restarts this week: none"
|
||
fi
|
||
fi
|
||
|
||
# ── Resource Watchdog ─────────────────────────────────────────────────────────────────────────
|
||
line "🎛️ Resource Watchdog"
|
||
if [[ -f "$RW_STATE_FILE" ]]; then
|
||
_rm_last=$(stat -c %Y "$RW_STATE_FILE" 2>/dev/null || echo 0)
|
||
_rm_ago=$(( NOW - _rm_last ))
|
||
_rm_level=$(grep "^current_level:" "$RW_STATE_FILE" 2>/dev/null | cut -d: -f2)
|
||
_rm_level="${_rm_level:-0}"
|
||
if [[ "$_rm_level" -gt 0 ]]; then
|
||
issue " Pressure level ${_rm_level} active │ Last run: $(_fmt_uptime "$_rm_ago") ago"
|
||
elif [[ "$_rm_ago" -lt 600 ]]; then
|
||
line " ✅ Level 0 (normal) │ Last run: $(_fmt_uptime "$_rm_ago") ago"
|
||
else
|
||
issue " Last run: $(_fmt_uptime "$_rm_ago") ago — watchdog_orchestrator may not be running"
|
||
fi
|
||
else
|
||
line " ℹ️ State file not found (resource_watchdog may not have run yet)"
|
||
fi
|
||
|
||
REPORT+=("")
|
||
|
||
if command -v docker >/dev/null 2>&1; then
|
||
RUNNING_NOW=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l)
|
||
TOTAL_NOW=$( timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l)
|
||
UNHEALTHY_NOW=$(timeout "$DOCKER_TIMEOUT" docker ps \
|
||
--filter health=unhealthy -q 2>/dev/null | wc -l)
|
||
STOPPED_NAMES=()
|
||
while IFS= read -r name; do
|
||
[[ -z "$name" ]] && continue
|
||
SKIP=false
|
||
for ignore in "${WATCHDOG_SCAN_IGNORE[@]}"; do
|
||
[[ "$name" == "$ignore" ]] && SKIP=true && break
|
||
done
|
||
[[ "$SKIP" == false ]] && STOPPED_NAMES+=("$name")
|
||
done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \
|
||
--format "{{.Names}}" 2>/dev/null)
|
||
line " 📦 $RUNNING_NOW/$TOTAL_NOW running │ unhealthy: $UNHEALTHY_NOW"
|
||
[[ ${#STOPPED_NAMES[@]} -gt 0 ]] && \
|
||
issue " Stopped (unexpected): ${STOPPED_NAMES[*]}" || \
|
||
line " ✅ All containers running"
|
||
fi
|
||
|
||
REPORT+=("")
|
||
|
||
# ── Failover ─────────────────────────────────────────────────────────────────────────────────
|
||
line "🔀 Fallback"
|
||
FO_PID=$(_get_lock_pid "fallback")
|
||
if _is_running "fallback"; then
|
||
FO_AGE=$(_lock_age "fallback")
|
||
line " ✅ Running │ PID: $FO_PID │ Uptime: $(_fmt_uptime "$FO_AGE")"
|
||
elif [[ "${FALLBACK_ENABLED:-true}" == false ]]; then
|
||
line " ⏸️ Not running — FALLBACK_ENABLED=false"
|
||
else
|
||
issue "fallback NOT RUNNING"
|
||
fi
|
||
|
||
FO_STATE="UNKNOWN"
|
||
FO_STATE_SECS=0
|
||
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
||
FO_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||
FO_EPOCH=$(grep "^last_change_epoch=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||
[[ -n "$FO_EPOCH" ]] && FO_STATE_SECS=$(( NOW - FO_EPOCH ))
|
||
fi
|
||
|
||
FO_DUR=$(_fmt_uptime "${FO_STATE_SECS:-0}")
|
||
|
||
case "$FO_STATE" in
|
||
NORMAL)
|
||
line " ✅ State: NORMAL │ Duration: $FO_DUR" ;;
|
||
FALLBACK)
|
||
issue " State: FALLBACK — $REMOTE_SERVER_NAME down for $FO_DUR"
|
||
FO_MINS=$(( FO_STATE_SECS / 60 ))
|
||
# Use REMOTE_ID-based tier delay vars — no HOST1/HOST2 hardcoding
|
||
T2_VAR="${REMOTE_ID}_TIER2_DELAY"; T3_VAR="${REMOTE_ID}_TIER3_DELAY"; T4_VAR="${REMOTE_ID}_TIER4_DELAY"
|
||
T2="${!T2_VAR:-240}"; T3="${!T3_VAR:-720}"; T4="${!T4_VAR:-1440}"
|
||
(( FO_MINS >= T2 )) && line " Tier 2: ✅ active" || line " Tier 2: ⏳ in $(( T2 - FO_MINS ))min"
|
||
(( FO_MINS >= T3 )) && line " Tier 3: ✅ active" || line " Tier 3: ⏳ in $(( T3 - FO_MINS ))min"
|
||
(( FO_MINS >= T4 )) && line " Tier 4: ✅ active" || line " Tier 4: ⏳ in $(( T4 - FO_MINS ))min"
|
||
;;
|
||
NO_INTERNET) issue " State: NO_INTERNET — DDNS stopped │ Duration: $FO_DUR" ;;
|
||
DARK) issue " State: DARK — remote down AND no internet │ Duration: $FO_DUR" ;;
|
||
*) finding " State: ${FO_STATE:-unknown}" ;;
|
||
esac
|
||
|
||
if command -v tailscale >/dev/null 2>&1; then
|
||
REMOTE_TS_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null)
|
||
if [[ -n "$REMOTE_TS_IP" ]]; then
|
||
ping -c 1 -W 2 "$REMOTE_TS_IP" >/dev/null 2>&1 && \
|
||
line " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_TS_IP — reachable ✅" || \
|
||
issue " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_TS_IP — not responding"
|
||
else
|
||
issue " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME) not visible on Tailscale"
|
||
fi
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ 🔗 MESH ━━━
|
||
# ==============================================================================================
|
||
section "🔗 MESH"
|
||
|
||
_ALL_HOST_IDS=()
|
||
for _h in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||
[[ -n "${!_h:-}" ]] && _ALL_HOST_IDS+=("$_h")
|
||
done
|
||
|
||
# ── Members ──
|
||
if [[ ${#_ALL_HOST_IDS[@]} -eq 0 ]]; then
|
||
line "No hosts defined"
|
||
else
|
||
line "Members:"
|
||
for _h in "${_ALL_HOST_IDS[@]}"; do
|
||
_server="${!_h}"
|
||
_owner_var="${_h}_OWNER"; _owner="${!_owner_var:-unknown}"
|
||
_email_var="${_h}_OWNER_EMAIL"; _email="${!_email_var:-(not set)}"
|
||
line " $_h $_server / $_owner / $_email"
|
||
done
|
||
fi
|
||
|
||
# ── Protected Services ──
|
||
_coverage_found=false
|
||
for _covered in "${_ALL_HOST_IDS[@]}"; do
|
||
_covered_owner_var="${_covered}_OWNER"; _covered_owner="${!_covered_owner_var:-$_covered}"
|
||
_covered_email_var="${_covered}_OWNER_EMAIL"; _covered_email="${!_covered_email_var:-}"
|
||
|
||
declare -A _tier_containers=()
|
||
declare -A _tier_delays=()
|
||
_covered_by=""
|
||
_any_tiers=false
|
||
|
||
for _covering in "${_ALL_HOST_IDS[@]}"; do
|
||
[[ "$_covering" == "$_covered" ]] && continue
|
||
for _tier in 1 2 3 4; do
|
||
_containers=$(get_array "FALLBACK_${_covering}_COVERS_${_covered}_TIER${_tier}")
|
||
[[ -z "$_containers" ]] && continue
|
||
_any_tiers=true
|
||
_tier_containers[$_tier]="$_containers"
|
||
_delay_var="${_covered}_TIER${_tier}_DELAY"
|
||
_tier_delays[$_tier]="${!_delay_var:-0}"
|
||
done
|
||
if [[ "$_any_tiers" == true ]]; then
|
||
_cov_owner_var="${_covering}_OWNER"; _cov_owner="${!_cov_owner_var:-$_covering}"
|
||
_covered_by="$_covering ($_cov_owner)"
|
||
fi
|
||
done
|
||
|
||
[[ "$_any_tiers" == false ]] && { unset _tier_containers _tier_delays; declare -A _tier_containers=() _tier_delays=(); continue; }
|
||
|
||
_coverage_found=true
|
||
_hdr="$_covered_owner"
|
||
[[ -n "$_covered_email" ]] && _hdr+=" — $_covered_email"
|
||
line "Protected: $_hdr"
|
||
for _tier in 1 2 3 4; do
|
||
[[ -z "${_tier_containers[$_tier]:-}" ]] && continue
|
||
_d="${_tier_delays[$_tier]:-0}"
|
||
if (( _d == 0 )); then _dlabel="immediate"
|
||
elif (( _d >= 1440 )); then _dlabel="$(( _d / 1440 ))d"
|
||
elif (( _d >= 60 )); then _dlabel="$(( _d / 60 ))hr"
|
||
else _dlabel="${_d}min"
|
||
fi
|
||
line " Tier $_tier (${_dlabel}): ${_tier_containers[$_tier]// /, }"
|
||
done
|
||
[[ -n "$_covered_by" ]] && line " Covered by: $_covered_by"
|
||
|
||
unset _tier_containers _tier_delays
|
||
declare -A _tier_containers=() _tier_delays=()
|
||
done
|
||
|
||
[[ "$_coverage_found" == false ]] && line "No fallback coverage configured"
|
||
|
||
# ── Partnership ──
|
||
if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
|
||
_po_host="${PARTNERSHIP_OWNER_HOST:-HOST1}"
|
||
_po_server="${!_po_host:-unknown}"
|
||
_po_name_var="${_po_host}_OWNER"; _po_name="${!_po_name_var:-unknown}"
|
||
line "Partnership: enabled — owner $_po_host ($_po_server / $_po_name), sync every ${PARTNERSHIP_SYNC_INTERVAL:-15}min"
|
||
else
|
||
line "Partnership: disabled"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ 🔐 SECURITY ━━━
|
||
# ==============================================================================================
|
||
section "🔐 SECURITY"
|
||
|
||
if ! command -v openssl >/dev/null 2>&1; then
|
||
line "SSL certs: openssl not available — skipping"
|
||
elif [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then
|
||
line "SSL certs: no domains configured (CERT_MONITOR_DOMAINS empty)"
|
||
else
|
||
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
||
[[ -z "$domain" ]] && continue
|
||
EXPIRY=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \
|
||
-connect "${domain}:443" -servername "$domain" 2>/dev/null | \
|
||
openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||
if [[ -z "$EXPIRY" ]]; then
|
||
issue "$domain: could not check certificate"
|
||
continue
|
||
fi
|
||
EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s 2>/dev/null || echo 0)
|
||
DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW) / 86400 ))
|
||
if [[ "$DAYS_LEFT" -le "${CERT_CRIT_DAYS:-7}" ]]; then
|
||
issue "$domain: ${DAYS_LEFT} days remaining — CRITICAL, renew now"
|
||
elif [[ "$DAYS_LEFT" -le "${CERT_WARN_DAYS:-30}" ]]; then
|
||
finding "$domain: ${DAYS_LEFT} days remaining — renew soon"
|
||
else
|
||
line "$domain: ${DAYS_LEFT} days remaining ✅"
|
||
fi
|
||
done
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ 📊 EMBY ━━━
|
||
# ==============================================================================================
|
||
section "📊 EMBY"
|
||
|
||
# detect_hosts() already aliased EMBY_URL and EMBY_API_KEY
|
||
if [[ -z "${EMBY_API_KEY:-}" ]]; then
|
||
line "Emby: API key not configured"
|
||
elif [[ "${EMBY_API_KEY:-}" == *"your-"* ]]; then
|
||
line "Emby: placeholder API key — configure HOST*_EMBY_API_KEY"
|
||
else
|
||
EMBY_SYSTEM=$(curl -sf --max-time 5 \
|
||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||
"${EMBY_URL}/System/Info" 2>/dev/null)
|
||
|
||
if [[ -z "$EMBY_SYSTEM" ]]; then
|
||
line "Emby: API unavailable — check if Emby is running"
|
||
else
|
||
EMBY_VERSION=$(echo "$EMBY_SYSTEM" | grep -o '"Version":"[^"]*"' | cut -d'"' -f4)
|
||
line "Emby: v${EMBY_VERSION:-unknown} — reachable ✅"
|
||
|
||
SESSIONS=$(curl -sf --max-time 5 \
|
||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||
"${EMBY_URL}/Sessions" 2>/dev/null)
|
||
ACTIVE_COUNT=$(echo "$SESSIONS" | grep -o '"NowPlayingItem"' | wc -l)
|
||
ACTIVE_COUNT="${ACTIVE_COUNT//[^0-9]/}"; ACTIVE_COUNT="${ACTIVE_COUNT:-0}"
|
||
line "Active streams now: $ACTIVE_COUNT"
|
||
|
||
ACTIVITY_LOG=$(curl -sf --max-time 10 \
|
||
-H "X-Emby-Token: $EMBY_API_KEY" \
|
||
"${EMBY_URL}/user_usage_stats/user_activity?days=7" 2>/dev/null)
|
||
|
||
if [[ -n "$ACTIVITY_LOG" ]] && echo "$ACTIVITY_LOG" | grep -q "user_name"; then
|
||
TOTAL_PLAYS=$(echo "$ACTIVITY_LOG" | \
|
||
grep -o '"total_plays":[0-9]*' | \
|
||
awk -F: '{sum+=$2} END {print sum+0}')
|
||
line "Streams this week: $TOTAL_PLAYS total plays"
|
||
echo "$ACTIVITY_LOG" | \
|
||
grep -o '"user_name":"[^"]*","total_plays":[0-9]*' | \
|
||
awk -F'"' '{name=$4; plays=$NF; gsub(/.*:/,"",plays); print plays, name}' | \
|
||
sort -rn | head -3 | \
|
||
while read -r plays name; do
|
||
line " → $name: $plays plays"
|
||
done
|
||
else
|
||
line "Weekly stats: user_usage_stats plugin not available"
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ ⚙️ SYSTEM HEALTH ━━━
|
||
# ==============================================================================================
|
||
section "⚙️ SYSTEM HEALTH"
|
||
|
||
if [[ -f "${TUNING_MONITOR_LOG:-}" ]] && [[ -s "$TUNING_MONITOR_LOG" ]]; then
|
||
INOTIFY_PEAK=$(awk -F'|' -v c="$WEEK_START" \
|
||
'$1>=c {if($3+0>max) max=$3+0} END{print max+0}' "$TUNING_MONITOR_LOG")
|
||
INOTIFY_AVG=$(awk -F'|' -v c="$WEEK_START" \
|
||
'$1>=c {sum+=$3;cnt++} END{if(cnt>0)printf "%.0f",sum/cnt;else print 0}' "$TUNING_MONITOR_LOG")
|
||
INOTIFY_WARNS=$(awk -F'|' -v c="$WEEK_START" \
|
||
'$1>=c && $6==1 {cnt++} END{print cnt+0}' "$TUNING_MONITOR_LOG")
|
||
INOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
|
||
INOW="${INOW//[^0-9]/}"; INOW="${INOW:-0}"
|
||
ILIM=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024)
|
||
IPCT=$(( INOW * 100 / ILIM ))
|
||
[[ "${INOTIFY_WARNS:-0}" -gt 0 ]] && \
|
||
issue "inotify: ${INOW}/${ILIM} now (${IPCT}%) | week peak:$INOTIFY_PEAK avg:$INOTIFY_AVG | ⚠️ $INOTIFY_WARNS warning(s)" || \
|
||
line "inotify: ${INOW}/${ILIM} now (${IPCT}%) | week peak:$INOTIFY_PEAK avg:$INOTIFY_AVG ✅"
|
||
|
||
PHPFPM_PEAK=$(awk -F'|' -v c="$WEEK_START" \
|
||
'$1>=c {if($7+0>max) max=$7+0} END{print max+0}' "$TUNING_MONITOR_LOG")
|
||
PHPFPM_AVG=$(awk -F'|' -v c="$WEEK_START" \
|
||
'$1>=c {sum+=$7;cnt++} END{if(cnt>0)printf "%.0f",sum/cnt;else print 0}' "$TUNING_MONITOR_LOG")
|
||
PHPFPM_WARNS=$(awk -F'|' -v c="$WEEK_START" \
|
||
'$1>=c && $10==1 {cnt++} END{print cnt+0}' "$TUNING_MONITOR_LOG")
|
||
PNOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0)
|
||
PNOW="${PNOW//[^0-9]/}"; PNOW="${PNOW:-0}"
|
||
PMAX="${PHP_MAX_CHILDREN:-250}"
|
||
[[ "${PHPFPM_WARNS:-0}" -gt 0 ]] && \
|
||
issue "php-fpm: ${PNOW}/${PMAX} workers now | week peak:$PHPFPM_PEAK avg:$PHPFPM_AVG | ⚠️ $PHPFPM_WARNS warning(s)" || \
|
||
line "php-fpm: ${PNOW}/${PMAX} workers now | week peak:$PHPFPM_PEAK avg:$PHPFPM_AVG ✅"
|
||
else
|
||
INOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
|
||
INOW="${INOW//[^0-9]/}"; INOW="${INOW:-0}"
|
||
ILIM=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024)
|
||
PNOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0)
|
||
PNOW="${PNOW//[^0-9]/}"; PNOW="${PNOW:-0}"
|
||
line "inotify: ${INOW}/${ILIM} — no weekly data yet"
|
||
line "php-fpm: ${PNOW}/${PHP_MAX_CHILDREN:-250} workers — no weekly data yet"
|
||
fi
|
||
|
||
if command -v smartctl >/dev/null 2>&1; then
|
||
SMART_ISSUES=0
|
||
for disk in /dev/sd? /dev/nvme?; do
|
||
[[ ! -e "$disk" ]] && continue
|
||
DISK_NAME=$(basename "$disk")
|
||
SKIP=false
|
||
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
|
||
[[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break
|
||
done
|
||
[[ "$SKIP" == true ]] && continue
|
||
HEALTH=$(smartctl -H "$disk" 2>/dev/null | grep "SMART overall-health" | awk '{print $NF}')
|
||
if [[ "$HEALTH" != "PASSED" ]] && [[ -n "$HEALTH" ]]; then
|
||
issue "SMART $DISK_NAME: $HEALTH — check immediately"
|
||
(( SMART_ISSUES++ ))
|
||
fi
|
||
done
|
||
[[ "$SMART_ISSUES" -eq 0 ]] && line "SMART: all drives PASSED ✅"
|
||
fi
|
||
|
||
if command -v git >/dev/null 2>&1 && [[ -d "${TARGET_DIR:-}/.git" ]]; then
|
||
CURRENT_COMMIT=$(git -C "$TARGET_DIR" rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||
LAST_PULL=$(git -C "$TARGET_DIR" log -1 --format="%ar" 2>/dev/null || echo "unknown")
|
||
line "Gitea: commit $CURRENT_COMMIT (pulled $LAST_PULL)"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ ⚠️ ISSUES REQUIRING ATTENTION ━━━
|
||
# ==============================================================================================
|
||
if [[ ${#ISSUES[@]} -gt 0 ]]; then
|
||
REPORT+=("")
|
||
REPORT+=("⚠️ ISSUES REQUIRING ATTENTION")
|
||
REPORT+=("$(printf '%.0s─' {1..50})")
|
||
for issue_line in "${ISSUES[@]}"; do
|
||
REPORT+=(" ❌ $issue_line")
|
||
done
|
||
fi
|
||
|
||
if [[ ${#FINDINGS[@]} -gt 0 ]]; then
|
||
REPORT+=("")
|
||
REPORT+=("ℹ️ NOTABLE")
|
||
REPORT+=("$(printf '%.0s─' {1..50})")
|
||
for finding_line in "${FINDINGS[@]}"; do
|
||
REPORT+=(" → $finding_line")
|
||
done
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Footer ━━━
|
||
# ==============================================================================================
|
||
REPORT+=("")
|
||
if [[ ${#ISSUES[@]} -eq 0 ]]; then
|
||
REPORT+=("✅ All systems healthy — enjoy your Sunday ☕")
|
||
else
|
||
REPORT+=("⚠️ ${#ISSUES[@]} issue(s) need attention")
|
||
fi
|
||
REPORT+=("$MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S')")
|
||
REPORT+=("$(printf '%.0s━' {1..50})")
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Output and Send ━━━
|
||
# ==============================================================================================
|
||
HEADER="☕ SUNDAY MORNING COFFEE REPORT — $REPORT_DATE"
|
||
DIVIDER="$(printf '%.0s━' {1..50})"
|
||
BODY=$(printf '%s\n' "$HEADER" "${REPORT[@]}")
|
||
|
||
echo ""
|
||
echo "$DIVIDER"
|
||
echo "$HEADER"
|
||
echo "$DIVIDER"
|
||
for report_line in "${REPORT[@]}"; do
|
||
echo "$report_line"
|
||
done
|
||
echo ""
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — notification not sent"
|
||
else
|
||
if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then
|
||
NOTIFY_SCRIPT="/usr/local/emhttp/plugins/dynamix/scripts/notify"
|
||
if [[ -x "$NOTIFY_SCRIPT" ]]; then
|
||
"$NOTIFY_SCRIPT" -s "☕ Weekly Report — $MY_ID" -d "$BODY" -i "normal" 2>/dev/null
|
||
echo "unRAID notification sent"
|
||
fi
|
||
fi
|
||
if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then
|
||
# Escape body for JSON
|
||
ESCAPED_BODY=$(echo "$BODY" | python3 -c \
|
||
'import sys,json; print(json.dumps(sys.stdin.read()))' 2>/dev/null || \
|
||
echo "\"$BODY\"")
|
||
PAYLOAD="{\"content\": ${ESCAPED_BODY}}"
|
||
curl -sf -H "Content-Type: application/json" \
|
||
-d "$PAYLOAD" "$DISCORD_WEBHOOK" >/dev/null 2>&1 && \
|
||
echo "Discord notification sent" || \
|
||
warn "Discord notification failed"
|
||
fi
|
||
echo "Report generated — $MY_ID — ${#ISSUES[@]} issue(s) ${#FINDINGS[@]} finding(s)"
|
||
fi |