diff --git a/Master.conf b/Master.conf index 226a14e..a9c0f58 100644 --- a/Master.conf +++ b/Master.conf @@ -167,7 +167,7 @@ ARRAY_START_SCRIPTS=( "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop - "Failover/failover.sh" # mutual failover — continuous loop +# "Failover/failover.sh" # mutual failover — continuous loop ) # ━━━ Daily Sync Maintenance ━━━ @@ -255,9 +255,9 @@ MEDIA_MANAGEMENT_JOBS=( "Media/media_shares_permissions.sh" # apply permissions — runs first "Media/media_cleaner.sh anime" # remove junk from anime shares "Media/media_cleaner.sh media" # remove junk from media shares - "Media/lidarr_cleanup.sh" # remove orphaned music files - "Media/sonarr_cleanup.sh" # remove orphaned TV files - "Media/radarr_cleanup.sh" # remove orphaned movie files +# "Media/lidarr_cleanup.sh" # remove orphaned music files +# "Media/sonarr_cleanup.sh" # remove orphaned TV files +# "Media/radarr_cleanup.sh" # remove orphaned movie files "Docker_Essentials/downloaders_reset.sh" # clear stuck states + purge old history ) @@ -438,6 +438,8 @@ declare -A PROFILE_SKIP_DISK_CHECK=( FAILOVER_CHECK_INTERVAL=120 FAILOVER_HANDBACK_STRIKES=2 FAILOVER_STATE_FILE="/boot/config/failover_state.db" + FAILOVER_ENABLED=false # set true when HOST2 is back online and tested + # false = suppresses "not running" warnings in status scripts # ━━━ Failover Test ━━━ FAILOVER_TEST_BLOCK_WAIT=150 diff --git a/Monitors/continuous_scripts_status.sh b/Monitors/continuous_scripts_status.sh index 7aa866a..494b44c 100644 --- a/Monitors/continuous_scripts_status.sh +++ b/Monitors/continuous_scripts_status.sh @@ -375,8 +375,12 @@ if is_watchdog_running "failover"; then FAILOVER_UPTIME=$(format_uptime "$FAILOVER_AGE") echo " ✅ Running │ PID: $FAILOVER_PID │ Uptime: $FAILOVER_UPTIME" else - echo " ❌ NOT RUNNING — failover.sh is not active" - echo " Start via: bash Orchestrators/array_start.sh" + if [[ "${FAILOVER_ENABLED:-true}" == false ]]; then + echo " ⏸️ Disabled — FAILOVER_ENABLED=false in Master.conf" + else + echo " ❌ NOT RUNNING — failover.sh is not active" + echo " Start via: bash Orchestrators/array_start.sh" + fi fi echo "" @@ -482,7 +486,7 @@ echo "━━━━━━━━━━━━━━━━━━━━━━━━ ISSUES=0 [[ "$SYS_RUNNING" == false ]] && ((ISSUES++)) [[ "$DOCKER_RUNNING" == false ]] && ((ISSUES++)) -[[ "$FAILOVER_RUNNING" == false ]] && ((ISSUES++)) +[[ "$FAILOVER_RUNNING" == false ]] && [[ "${FAILOVER_ENABLED:-true}" != false ]] && ((ISSUES++)) [[ -n "$ACTIVE_STRIKES" ]] && ((ISSUES++)) [[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && ((ISSUES++)) [[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && ((ISSUES++)) diff --git a/Orchestrators/sunday_morning_coffee_report.sh b/Orchestrators/sunday_morning_coffee_report.sh index e883f49..9341fbb 100644 --- a/Orchestrators/sunday_morning_coffee_report.sh +++ b/Orchestrators/sunday_morning_coffee_report.sh @@ -156,59 +156,85 @@ fi # ----------------------------------------------------------------------------------------------- section "📀 ARRAY" -# unRAID array info +# unRAID array info — parse var/local/emhttp correctly if [[ -f /var/local/emhttp/disks.ini ]]; then - DISK_COUNT=$(grep -c "^\[disk" /var/local/emhttp/disks.ini 2>/dev/null || echo "?") - line "Array disks: $DISK_COUNT" + # Count data disks only (disk1, disk2 etc — not parity, cache, flash) + DISK_COUNT=$(grep -c "^\[disk[0-9]" /var/local/emhttp/disks.ini 2>/dev/null || echo "?") + PARITY_COUNT=$(grep -c "^\[parity" /var/local/emhttp/disks.ini 2>/dev/null || echo 0) + line "Array disks: $DISK_COUNT data + $PARITY_COUNT parity" +else + line "Array: disks.ini not found — array may not be started" fi -# Parity -if [[ -f /var/local/emhttp/parity-date.txt ]]; then - PARITY_INFO=$(cat /var/local/emhttp/parity-date.txt 2>/dev/null) +# Parity status from var/local/emhttp +PARITY_FILE="/var/local/emhttp/parity-date.txt" +if [[ -f "$PARITY_FILE" ]]; then + PARITY_INFO=$(cat "$PARITY_FILE" 2>/dev/null) if echo "$PARITY_INFO" | grep -q "progress"; then finding "Parity check in progress" else PARITY_DATE=$(echo "$PARITY_INFO" | grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -1) PARITY_ERRORS=$(echo "$PARITY_INFO" | grep -oE 'errors=[0-9]+' | cut -d= -f2 || echo 0) - if [[ "${PARITY_ERRORS:-0}" -gt 0 ]]; then - issue "Parity: $PARITY_ERRORS errors on last check ($PARITY_DATE)" + PARITY_ERRORS="${PARITY_ERRORS:-0}" + if [[ "$PARITY_ERRORS" -gt 0 ]]; then + issue "Parity: $PARITY_ERRORS errors on last check (${PARITY_DATE:-unknown})" else - line "Parity: OK (last check: ${PARITY_DATE:-unknown}, 0 errors)" + line "Parity: OK — last check: ${PARITY_DATE:-unknown}, 0 errors ✅" fi fi +else + line "Parity: no check recorded yet" fi -# ZFS pool health +# ZFS pool health — filter ignored pools if command -v zpool >/dev/null 2>&1; then - POOLS=$(zpool list -H -o name,health 2>/dev/null) while IFS=$'\t' read -r pool health; do [[ -z "$pool" ]] && continue - # Skip single-disk unRAID array pools + # Skip pools in ZFS_REPORT_IGNORE_POOLS + SKIP=false + for ignore in "${ZFS_REPORT_IGNORE_POOLS[@]:-}"; do + [[ "$pool" == "$ignore" ]] && SKIP=true && break + done + [[ "$SKIP" == true ]] && continue if [[ "$health" == "ONLINE" ]]; then line "ZFS $pool: ONLINE ✅" else issue "ZFS $pool: $health — check immediately" fi - done <<< "$POOLS" + done < <(zpool list -H -o name,health 2>/dev/null) fi -# Drive temperatures from SMART +# Drive temperatures from SMART — fixed parsing if command -v smartctl >/dev/null 2>&1; then TEMP_WARN=false TEMP_SUMMARY="" for disk in /dev/sd? /dev/nvme?; do [[ ! -e "$disk" ]] && continue DISK_NAME=$(basename "$disk") - # Skip ignored drives SKIP=false for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do [[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break done [[ "$SKIP" == true ]] && continue - TEMP=$(smartctl -A "$disk" 2>/dev/null | \ - awk '/Temperature_Celsius|Temperature:/ {print $NF; exit}' | tr -d '°C') + + # Try multiple SMART temp attribute names — different drives use different fields + TEMP=$(smartctl -A "$disk" 2>/dev/null | awk ' + /^190 / || /^194 / { print $10; exit } + /Temperature_Celsius/ { print $10; exit } + /Airflow_Temperature/ { print $10; exit } + ') + + # NVMe uses different output format + if [[ -z "$TEMP" ]]; then + TEMP=$(smartctl -A "$disk" 2>/dev/null | \ + awk '/^Temperature:/ { print $2; exit }') + fi + + # Strip any non-numeric chars + TEMP="${TEMP//[^0-9]/}" [[ -z "$TEMP" ]] && continue - TEMP_INT=$(printf "%.0f" "$TEMP" 2>/dev/null || echo 0) + TEMP_INT="$TEMP" + if [[ "$TEMP_INT" -ge "${SMART_TEMP_CRIT:-55}" ]]; then issue "Drive $DISK_NAME: ${TEMP_INT}°C — CRITICAL" TEMP_WARN=true @@ -220,7 +246,7 @@ if command -v smartctl >/dev/null 2>&1; then TEMP_SUMMARY+="${DISK_NAME}:${TEMP_INT}°C" done if [[ "$TEMP_WARN" == false ]]; then - line "Drive temps: all normal" + line "Drive temps: all normal ✅" fi [[ -n "$TEMP_SUMMARY" ]] && line "Temps: $TEMP_SUMMARY" fi @@ -314,10 +340,11 @@ for arr_name in "Sonarr|${HOST1_SONARR_URL}|${HOST1_SONARR_API_KEY}|v3" \ if [[ "$url" == *"your-"* ]] || [[ -z "$key" ]]; then continue; fi QUEUE=$(curl -sf --max-time 5 -H "X-Api-Key: $key" \ "${url}/api/${ver}/queue?pageSize=1" 2>/dev/null | \ - python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('totalRecords',0))" \ - 2>/dev/null || echo "?") + grep -o '"totalRecords":[0-9]*' | grep -o '[0-9]*' || echo "?") if [[ "$QUEUE" == "0" ]] || [[ -z "$QUEUE" ]]; then line "$name queue: empty ✅" + elif [[ "$QUEUE" == "?" ]]; then + finding "$name queue: API unavailable" else line "$name queue: $QUEUE items" fi @@ -467,7 +494,8 @@ MEM_AVAIL_CR=$(awk '/MemAvailable/ {printf "%.1f", $2/1048576}' /proc/meminfo) MEM_TOTAL_CR=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo) LOAD_CR=$(awk '{print $1}' /proc/loadavg) ZOMBIE_CR=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0) -ZOMBIE_CR="${ZOMBIE_CR//[^0-9]/}"; ZOMBIE_CR="${ZOMBIE_CR:-0}" +ZOMBIE_CR="${ZOMBIE_CR//[^0-9]/}" +ZOMBIE_CR="${ZOMBIE_CR:-0}" ARC_GB_CR="n/a" if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then ARC_B=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) @@ -518,16 +546,26 @@ if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then fi fi -# Docker overview +# Docker overview — filter WATCHDOG_SCAN_IGNORE from stopped list if command -v docker >/dev/null 2>&1; then RUNNING_CR=$(docker ps -q 2>/dev/null | wc -l) TOTAL_CR=$(docker ps -aq 2>/dev/null | wc -l) UNHEALTHY_CR=$(docker ps --filter health=unhealthy -q 2>/dev/null | wc -l) - STOPPED_CR=$(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null | head -5 | tr '\n' ' ') - STOPPED_COUNT_CR=$(docker ps -af "status=exited" -q 2>/dev/null | wc -l) + + STOPPED_FILTERED_CR=() + while IFS= read -r name; do + [[ -z "$name" ]] && continue + SKIP=false + for ignore in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do + [[ "$name" == "$ignore" ]] && SKIP=true && break + done + [[ "$SKIP" == false ]] && STOPPED_FILTERED_CR+=("$name") + done < <(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null) + + STOPPED_COUNT_CR="${#STOPPED_FILTERED_CR[@]}" line " 📦 $RUNNING_CR/$TOTAL_CR running │ unhealthy: $UNHEALTHY_CR" - if [[ "${STOPPED_COUNT_CR:-0}" -gt 0 ]]; then - issue " ⚠️ Stopped: $STOPPED_CR" + if [[ "$STOPPED_COUNT_CR" -gt 0 ]]; then + issue " ⚠️ Stopped (unexpected): ${STOPPED_FILTERED_CR[*]}" else line " ✅ All containers running" fi @@ -544,7 +582,11 @@ if is_loop_running "failover"; then FAILOVER_UP_CR=$(format_uptime_cr "$FAILOVER_AGE_CR") line " ✅ Running │ PID: $FAILOVER_PID_CR │ Uptime: $FAILOVER_UP_CR" else - issue "failover NOT RUNNING" + if [[ "${FAILOVER_ENABLED:-true}" == false ]]; then + line " ⏸️ Not running — disabled in Master.conf (FAILOVER_ENABLED=false)" + else + issue "failover NOT RUNNING" + fi fi FAILOVER_STATE_CR="UNKNOWN" @@ -642,44 +684,50 @@ else fi if [[ "$EMBY_KEY" != *"your-"* ]] && [[ -n "$EMBY_KEY" ]]; then - WEEK_MS=$(( 7 * 24 * 3600 * 10000000 )) - ACTIVITY=$(curl -sf --max-time 10 \ + + # Check if Emby is reachable + EMBY_SYSTEM=$(curl -sf --max-time 5 \ -H "X-Emby-Token: $EMBY_KEY" \ - "${EMBY_URL}/Sessions?ControllableByUserId=&api_key=$EMBY_KEY" \ - 2>/dev/null) + "${EMBY_URL}/System/Info?api_key=$EMBY_KEY" 2>/dev/null) - # Use activity log for weekly stats - ACTIVITY_LOG=$(curl -sf --max-time 10 \ - -H "X-Emby-Token: $EMBY_KEY" \ - "${EMBY_URL}/user_usage_stats/user_activity?days=7&api_key=$EMBY_KEY" \ - 2>/dev/null) - - if [[ -n "$ACTIVITY_LOG" ]]; then - TOTAL_PLAYS=$(echo "$ACTIVITY_LOG" | \ - python3 -c "import sys,json; d=json.load(sys.stdin); \ - print(sum(u.get('total_plays',0) for u in d))" 2>/dev/null || echo "?") - line "Streams this week: $TOTAL_PLAYS total plays" - - # Top 3 users - TOP_USERS=$(echo "$ACTIVITY_LOG" | \ - python3 -c " -import sys,json -d=json.load(sys.stdin) -users=sorted(d,key=lambda x:x.get('total_plays',0),reverse=True)[:3] -for u in users: - print(f\" → {u.get('user_name','?')}: {u.get('total_plays',0)} plays\") -" 2>/dev/null) - [[ -n "$TOP_USERS" ]] && echo "$TOP_USERS" | while IFS= read -r l; do line "$l"; done + if [[ -z "$EMBY_SYSTEM" ]]; then + line "Emby: API unavailable — check if Emby is running" else - # Fallback — just show active sessions - ACTIVE=$(curl -sf --max-time 5 \ + EMBY_VERSION=$(echo "$EMBY_SYSTEM" | grep -o '"Version":"[^"]*"' | cut -d'"' -f4) + line "Emby: v${EMBY_VERSION:-unknown} — reachable ✅" + + # Active sessions right now + SESSIONS=$(curl -sf --max-time 5 \ -H "X-Emby-Token: $EMBY_KEY" \ - "${EMBY_URL}/Sessions?api_key=$EMBY_KEY" 2>/dev/null | \ - python3 -c "import sys,json; \ - d=json.load(sys.stdin); \ - active=[s for s in d if s.get('NowPlayingItem')]; \ - print(f'{len(active)} active streams now')" 2>/dev/null || echo "API unavailable") - line "Emby: $ACTIVE" + "${EMBY_URL}/Sessions?api_key=$EMBY_KEY" 2>/dev/null) + ACTIVE_COUNT=$(echo "$SESSIONS" | grep -o '"NowPlayingItem"' | wc -l) + ACTIVE_COUNT="${ACTIVE_COUNT//[^0-9]/}"; ACTIVE_COUNT="${ACTIVE_COUNT:-0}" + line "Active streams now: $ACTIVE_COUNT" + + # Weekly stats via user_usage_stats plugin if available + ACTIVITY_LOG=$(curl -sf --max-time 10 \ + -H "X-Emby-Token: $EMBY_KEY" \ + "${EMBY_URL}/user_usage_stats/user_activity?days=7&api_key=$EMBY_KEY" \ + 2>/dev/null) + + if [[ -n "$ACTIVITY_LOG" ]] && echo "$ACTIVITY_LOG" | grep -q "user_name"; then + # Parse total plays — sum all total_plays fields + TOTAL_PLAYS=$(echo "$ACTIVITY_LOG" | \ + grep -o '"total_plays":[0-9]*' | \ + awk -F: '{sum+=$2} END {print sum+0}') + line "Streams this week: $TOTAL_PLAYS total plays" + + # Top 3 users by play count + echo "$ACTIVITY_LOG" | \ + grep -o '"user_name":"[^"]*","total_plays":[0-9]*' | \ + awk -F'"' '{name=$4; plays=$NF; gsub(/.*:/,"",plays); print plays, name}' | \ + sort -rn | head -3 | \ + while read -r plays name; do + line " → $name: $plays plays" + done + else + line "Weekly stats: user_usage_stats plugin not available" + fi fi else line "Emby: API key not configured" @@ -811,6 +859,7 @@ REPORT+=("━━━━━━━━━━━━━━━━━━━━━━━ # ----------------------------------------------------------------------------------------------- HEADER="☕ SUNDAY MORNING COFFEE REPORT — $REPORT_DATE" DIVIDER="$(printf '%.0s━' {1..50})" +BODY=$(printf '%s\n' "$HEADER" "${REPORT[@]}") echo "" echo "$DIVIDER" @@ -824,11 +873,24 @@ done echo "" # Send notification -BODY=$(printf '%s\n' "$HEADER" "${REPORT[@]}") - if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — notification not sent" else - notify "$BODY" "☕ Weekly Report" "normal" + # Notify with full body — suppress body from terminal log to avoid duplication + if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then + NOTIFY_SCRIPT="/usr/local/emhttp/plugins/dynamix/scripts/notify" + if [[ -x "$NOTIFY_SCRIPT" ]]; then + "$NOTIFY_SCRIPT" -s "☕ Weekly Report" -d "$BODY" -i "normal" 2>/dev/null + log "$ICON_NOTIFY unRAID notification sent" + fi + fi + if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then + PAYLOAD=$(printf '{"content": "%s — **%s**\\n%s"}' \ + "$ICON_NOTIFY" "☕ Weekly Report" "$BODY") + curl -s -H "Content-Type: application/json" \ + -d "$PAYLOAD" "$DISCORD_WEBHOOK" >/dev/null 2>&1 && \ + log "$ICON_NOTIFY Discord notification sent" || \ + warn "Discord notification failed" + fi success "Report sent" fi \ No newline at end of file