From b3d53b792b30d23effa12f0919a40af34befd41c Mon Sep 17 00:00:00 2001 From: FailedProxy Date: Sun, 26 Apr 2026 02:30:12 -0400 Subject: [PATCH] added /mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh --- Master.conf | 6 +- Monitors/continuous_scripts_staus.sh | 486 ++++++++++++++++++ Orchestrators/sunday_morning_coffee_report.sh | 272 +++++++--- user_script_plug-in.sh | 73 +-- 4 files changed, 736 insertions(+), 101 deletions(-) create mode 100644 Monitors/continuous_scripts_staus.sh diff --git a/Master.conf b/Master.conf index 2887b77..77a3ddf 100644 --- a/Master.conf +++ b/Master.conf @@ -944,7 +944,7 @@ LIDARR_PROTECTED_PATTERNS=( # NEVER deleted — cover art, metadata, lyrics # Lidarr generates these but doesn't include them in trackFile API # Without this protection cleanup would delete all your artwork -LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this +LIDARR_MAX_DELETE_GB=5 # require --i-know-what-im-doing if deletion exceeds this LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" @@ -975,7 +975,7 @@ declare -A HOST2_SONARR_PATH_MAP=( ) SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion -SONARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this +SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_PROTECTED_PATTERNS=( # Subtitles @@ -1020,7 +1020,7 @@ declare -A HOST2_RADARR_PATH_MAP=( ) RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion -RADARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this +RADARR_MAX_DELETE_GB=15 # require --i-know-what-im-doing if deletion exceeds this RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") RADARR_PROTECTED_PATTERNS=( # Subtitles diff --git a/Monitors/continuous_scripts_staus.sh b/Monitors/continuous_scripts_staus.sh new file mode 100644 index 0000000..5ed71df --- /dev/null +++ b/Monitors/continuous_scripts_staus.sh @@ -0,0 +1,486 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# ----------------------------- Continuous Scripts Status -------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Live status dashboard for all continuously running scripts in the ecosystem. +# Run manually anytime — no schedule, no cron. +# +# Covers all scripts started by array_start.sh that run until array stops: +# system_watchdog.sh — system health monitor +# docker_watchdog.sh — container health monitor +# failover.sh — mutual failover monitor +# +# Shows for each: +# Running state, PID, uptime, current cycle +# Active strikes, skip list, recent actions +# Live system/container health snapshot +# Failover state, tier status, Tailscale connectivity +# +# If a script is mid-cycle state files are read as-is — reflects last completed cycle. +# Run: bash Monitors/continuous_scripts_status.sh +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# HELPERS +# ----------------------------------------------------------------------------------------------- + +get_lock_pid() { + local script_name="$1" + local lockfile="$LOCK_DIR/${script_name}.lock" + if [[ -f "$lockfile" ]]; then + local content + content=$(cat "$lockfile" 2>/dev/null) + echo "${content%%:*}" + fi +} + +get_lock_name() { + local script_name="$1" + local lockfile="$LOCK_DIR/${script_name}.lock" + if [[ -f "$lockfile" ]]; then + local content + content=$(cat "$lockfile" 2>/dev/null) + echo "${content##*:}" + fi +} + +is_watchdog_running() { + local script_name="$1" + local pid + pid=$(get_lock_pid "$script_name") + local locked_name + locked_name=$(get_lock_name "$script_name") + [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$locked_name" == "$script_name" ]] +} + +get_lock_age() { + local script_name="$1" + local lockfile="$LOCK_DIR/${script_name}.lock" + if [[ -f "$lockfile" ]]; then + local mtime now + mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0) + now=$(date +%s) + echo $(( now - mtime )) + else + echo 0 + fi +} + +format_uptime() { + local seconds=$1 + local days=$(( seconds / 86400 )) + local hours=$(( (seconds % 86400) / 3600 )) + local mins=$(( (seconds % 3600) / 60 )) + if (( days > 0 )); then + echo "${days}d ${hours}h ${mins}m" + elif (( hours > 0 )); then + echo "${hours}h ${mins}m" + else + echo "${mins}m" + fi +} + +get_strikes() { + local state_file="$1" + local key="$2" + grep -E "^${key}:" "$state_file" 2>/dev/null | cut -d: -f2 +} + +divider() { printf '%.0s─' {1..55}; echo; } +header() { echo ""; echo " $1"; divider; } + +# ----------------------------------------------------------------------------------------------- +# ━━━ HEADER ━━━ +# ----------------------------------------------------------------------------------------------- +clear +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " 🛡️ WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')" +echo " 🖥️ $(hostname)" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +# ----------------------------------------------------------------------------------------------- +# ━━━ System Watchdog ━━━ +# ----------------------------------------------------------------------------------------------- +header "⚙️ SYSTEM WATCHDOG" + +SYS_PID=$(get_lock_pid "system_watchdog") +SYS_RUNNING=false + +if is_watchdog_running "system_watchdog"; then + SYS_RUNNING=true + SYS_AGE=$(get_lock_age "system_watchdog") + SYS_UPTIME=$(format_uptime "$SYS_AGE") + SYS_CYCLE=$(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL )) + echo " ✅ Running │ PID: $SYS_PID │ Uptime: $SYS_UPTIME │ ~Cycle: $SYS_CYCLE" + echo " ⏱️ Interval: ${SYSTEM_WATCHDOG_INTERVAL}s │ Heartbeat every: ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS}hr" +else + echo " ❌ NOT RUNNING — system_watchdog.sh is not active" + echo " Start via: bash Orchestrators/array_start.sh" +fi + +echo "" + +# System watchdog strikes +if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then + ACTIVE_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") + if [[ -n "$ACTIVE_STRIKES" ]]; then + echo " ⚠️ Active strikes:" + while IFS=: read -r key count; do + [[ -z "$key" ]] && continue + echo " → $key: $count/$SYS_WATCHDOG_STRIKE_LIMIT" + done <<< "$ACTIVE_STRIKES" + else + echo " ✅ Strikes: none" + fi +else + echo " ℹ️ Strike state file not found (watchdog may not have run yet)" +fi + +# Reboot log +if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then + TOTAL_REBOOTS=$(grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0) + WEEK_EPOCH=$(date -d "7 days ago" +%s) + WEEK_REBOOTS=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \ + "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l) + echo " 🔄 Watchdog reboots: $WEEK_REBOOTS this week / $TOTAL_REBOOTS total" +fi + +# Container skip list +if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then + SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE") + echo "" + echo " ⛔ Skip list ($SKIP_COUNT containers — manual intervention needed):" + while IFS= read -r container; do + [[ -z "$container" ]] && continue + echo " → $container" + done < "$SYS_WATCHDOG_FAILED_FILE" +else + echo " ✅ Skip list: empty" +fi + +# Current system health snapshot +echo "" +echo " 📊 Current system state:" + +# rootfs +ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') +[[ "${ROOTFS_PCT:-0}" -ge "${SYS_WATCHDOG_ROOTFS_PCT:-95}" ]] && \ + ROOTFS_ICON="⚠️ " || ROOTFS_ICON="✅" +echo " ${ROOTFS_ICON} rootfs: ${ROOTFS_PCT}% (threshold: ${SYS_WATCHDOG_ROOTFS_PCT}%)" + +# RAM +MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) +MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}") +MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo) +[[ $(printf "%.0f" "$MEM_FREE_GB") -lt "${SYS_WATCHDOG_MEM_GB:-4}" ]] && \ + MEM_ICON="⚠️ " || MEM_ICON="✅" +echo " ${MEM_ICON} RAM: ${MEM_FREE_GB}GB free / ${MEM_TOTAL_GB}GB total (threshold: ${SYS_WATCHDOG_MEM_GB}GB free)" + +# ARC +if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then + ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX )) + ARC_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}") + [[ "$ARC_PCT" -ge "${SYS_WATCHDOG_ARC_PINNED_PCT:-98}" ]] && \ + ARC_ICON="⚠️ " || ARC_ICON="✅" + echo " ${ARC_ICON} ZFS ARC: ${ARC_GB}GB (${ARC_PCT}% of max, threshold: ${SYS_WATCHDOG_ARC_PINNED_PCT}%)" +fi + +# Load +LOAD=$(awk '{print $1}' /proc/loadavg) +CORES=$(nproc) +LOAD_THRESH=$(( CORES * ${SYS_WATCHDOG_LOAD_MULTIPLIER:-3} )) +LOAD_INT=$(printf "%.0f" "$LOAD") +[[ "$LOAD_INT" -ge "$LOAD_THRESH" ]] && LOAD_ICON="⚠️ " || LOAD_ICON="✅" +echo " ${LOAD_ICON} Load avg: $LOAD (threshold: ${LOAD_THRESH} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}x ${CORES} cores)" + +# Zombies +ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0) +ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}" +ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}" +[[ "$ZOMBIE_COUNT" -ge "${SYS_WATCHDOG_ZOMBIE_LIMIT:-50}" ]] && \ + ZOMBIE_ICON="⚠️ " || ZOMBIE_ICON="✅" +echo " ${ZOMBIE_ICON} Zombies: $ZOMBIE_COUNT (threshold: ${SYS_WATCHDOG_ZOMBIE_LIMIT})" + +# CPU temp +if command -v sensors >/dev/null 2>&1; then + CPU_TEMP=$(sensors 2>/dev/null | \ + grep -i "Package id 0\|Tctl\|CPU Temp" | \ + awk '{print $NF}' | tr -d '+°C' | head -1) + if [[ -n "$CPU_TEMP" ]]; then + CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP") + [[ "$CPU_TEMP_INT" -ge "${SYS_WATCHDOG_CPU_TEMP_MAX:-95}" ]] && \ + TEMP_ICON="⚠️ " || TEMP_ICON="✅" + echo " ${TEMP_ICON} CPU temp: ${CPU_TEMP_INT}°C (threshold: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C)" + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ Docker Watchdog ━━━ +# ----------------------------------------------------------------------------------------------- +header "🐳 DOCKER WATCHDOG" + +DOCKER_PID=$(get_lock_pid "docker_watchdog") +DOCKER_RUNNING=false + +if is_watchdog_running "docker_watchdog"; then + DOCKER_RUNNING=true + DOCKER_AGE=$(get_lock_age "docker_watchdog") + DOCKER_UPTIME=$(format_uptime "$DOCKER_AGE") + DOCKER_CYCLE=$(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL )) + echo " ✅ Running │ PID: $DOCKER_PID │ Uptime: $DOCKER_UPTIME │ ~Cycle: $DOCKER_CYCLE" + echo " ⏱️ Interval: ${DOCKER_WATCHDOG_INTERVAL}s │ Heartbeat every: ${DOCKER_WATCHDOG_HEARTBEAT_HOURS}hr" +else + echo " ❌ NOT RUNNING — docker_watchdog.sh is not active" + echo " Start via: bash Orchestrators/array_start.sh" +fi + +echo "" + +# Container watchdog strikes +if [[ -f "$WATCHDOG_STATE_FILE" ]]; then + ACTIVE_CONTAINER_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") + if [[ -n "$ACTIVE_CONTAINER_STRIKES" ]]; then + echo " ⚠️ Active container strikes:" + while IFS=: read -r key count; do + [[ -z "$key" ]] && continue + echo " → $key: $count" + done <<< "$ACTIVE_CONTAINER_STRIKES" + else + echo " ✅ Container strikes: none" + fi +fi + +# Container restart history this week +if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then + WEEK_EPOCH=$(date -d "7 days ago" +%s 2>/dev/null || date -v-7d +%s 2>/dev/null) + WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_EPOCH" \ + 'NR>0 {if ($2 >= cutoff) count++} END {print count+0}' \ + "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null) + if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then + echo "" + echo " 🔄 Container restarts this week: $WEEK_RESTARTS" + awk -F'|' -v cutoff="$WEEK_EPOCH" \ + '$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \ + sort | uniq -c | sort -rn | head -5 | \ + while read -r count name; do + echo " → $name: $count restart(s)" + done + else + echo " ✅ Container restarts this week: none" + fi +fi + +# Docker container overview +echo "" +echo " 📦 Container overview:" + +if command -v docker >/dev/null 2>&1; then + RUNNING=$(docker ps -q 2>/dev/null | wc -l) + TOTAL=$(docker ps -aq 2>/dev/null | wc -l) + UNHEALTHY=$(docker ps --filter health=unhealthy -q 2>/dev/null | wc -l) + STOPPED=$(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null) + STOPPED_COUNT=$(echo "$STOPPED" | grep -c . 2>/dev/null || echo 0) + + echo " Running: $RUNNING / $TOTAL total" + [[ "$UNHEALTHY" -gt 0 ]] && echo " ⚠️ Unhealthy: $UNHEALTHY" + if [[ "$STOPPED_COUNT" -gt 0 ]]; then + echo " ⚠️ Stopped containers:" + echo "$STOPPED" | head -10 | while IFS= read -r name; do + [[ -z "$name" ]] && continue + echo " → $name" + done + else + echo " ✅ All containers running" + fi + + # Check required containers + detect_hosts 2>/dev/null + if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + REQUIRED=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}") + else + REQUIRED=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}") + fi + + REQUIRED_ISSUES=0 + if [[ ${#REQUIRED[@]} -gt 0 ]]; then + echo "" + echo " 🔐 Required containers:" + for container in "${REQUIRED[@]}"; do + [[ -z "$container" ]] && continue + STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "not found") + if [[ "$STATUS" == "true" ]]; then + echo " ✅ $container" + else + echo " ❌ $container — $STATUS" + ((REQUIRED_ISSUES++)) + fi + done + fi + + # Tier 1 monitored containers from WATCHDOG_CONTAINERS + if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then + echo "" + echo " 📊 Monitored containers (memory):" + for container in "${!WATCHDOG_CONTAINERS[@]}"; do + LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}" + LIMIT_GB=$(awk "BEGIN {printf \"%.0f\", $LIMIT_MB / 1024}") + USAGE=$(docker stats --no-stream --format "{{.MemUsage}}" "$container" \ + 2>/dev/null | awk '{print $1}') + STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "not found") + if [[ "$STATUS" == "true" ]]; then + echo " ✅ $container: ${USAGE:-?} (limit: ${LIMIT_GB}GB)" + else + echo " ❌ $container: not running (limit: ${LIMIT_GB}GB)" + fi + done + fi +else + echo " Docker not available" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ Failover ━━━ +# ----------------------------------------------------------------------------------------------- +header "🔀 FAILOVER" + +FAILOVER_PID=$(get_lock_pid "failover") +FAILOVER_RUNNING=false + +if is_watchdog_running "failover"; then + FAILOVER_RUNNING=true + FAILOVER_AGE=$(get_lock_age "failover") + FAILOVER_UPTIME=$(format_uptime "$FAILOVER_AGE") + echo " ✅ Running │ PID: $FAILOVER_PID │ Uptime: $FAILOVER_UPTIME" +else + echo " ❌ NOT RUNNING — failover.sh is not active" + echo " Start via: bash Orchestrators/array_start.sh" +fi + +echo "" + +# Failover state +FAILOVER_STATE="UNKNOWN" +FAILOVER_LAST_CHANGE="" +FAILOVER_STATE_SECONDS=0 + +if [[ -f "$FAILOVER_STATE_FILE" ]]; then + FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + FAILOVER_LAST_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + FAILOVER_LAST_EPOCH=$(grep "^last_change_epoch=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + if [[ -n "$FAILOVER_LAST_EPOCH" ]]; then + FAILOVER_STATE_SECONDS=$(( $(date +%s) - FAILOVER_LAST_EPOCH )) + fi +fi + +STATE_DURATION=$(format_uptime "${FAILOVER_STATE_SECONDS:-0}") + +case "$FAILOVER_STATE" in + NORMAL) + echo " ✅ State: NORMAL" + echo " 📅 In NORMAL state for: $STATE_DURATION" + ;; + FAILOVER) + echo " ⚠️ State: FAILOVER — remote server down" + echo " ⏱️ Duration: $STATE_DURATION" + # Show which tiers are active + TIER1_DELAY=0 + if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + TIER2_DELAY=$HOST2_TIER2_DELAY + TIER3_DELAY=$HOST2_TIER3_DELAY + TIER4_DELAY=$HOST2_TIER4_DELAY + else + TIER2_DELAY=$HOST1_TIER2_DELAY + TIER3_DELAY=$HOST1_TIER3_DELAY + TIER4_DELAY=$HOST1_TIER4_DELAY + fi + FAILOVER_MINS=$(( FAILOVER_STATE_SECONDS / 60 )) + echo "" + echo " 🔄 Tier status:" + echo " Tier 1 (immediate): ✅ active" + if (( FAILOVER_MINS >= TIER2_DELAY )); then + echo " Tier 2 (${TIER2_DELAY}min): ✅ active" + else + REMAINING=$(( TIER2_DELAY - FAILOVER_MINS )) + echo " Tier 2 (${TIER2_DELAY}min): ⏳ activates in ${REMAINING}min" + fi + if (( FAILOVER_MINS >= TIER3_DELAY )); then + echo " Tier 3 (${TIER3_DELAY}min): ✅ active" + else + REMAINING=$(( TIER3_DELAY - FAILOVER_MINS )) + echo " Tier 3 (${TIER3_DELAY}min): ⏳ activates in ${REMAINING}min" + fi + if (( FAILOVER_MINS >= TIER4_DELAY )); then + echo " Tier 4 (${TIER4_DELAY}min): ✅ active" + else + REMAINING=$(( TIER4_DELAY - FAILOVER_MINS )) + echo " Tier 4 (${TIER4_DELAY}min): ⏳ activates in ${REMAINING}min" + fi + ;; + NO_INTERNET) + echo " ❌ State: NO_INTERNET — DDNS stopped" + echo " ⏱️ Down for: $STATE_DURATION" + ;; + DARK) + echo " ❌ State: DARK — remote down AND no internet" + echo " ⏱️ Duration: $STATE_DURATION" + ;; + *) + echo " ❓ State: ${FAILOVER_STATE:-unknown}" + ;; +esac + +# Tailscale remote visibility +echo "" +if command -v tailscale >/dev/null 2>&1; then + REMOTE_IP=$(tailscale ip -4 "$HOST2" 2>/dev/null) + if [[ -n "$REMOTE_IP" ]]; then + # Try a quick ping to see last seen + if ping -c 1 -W 2 "$REMOTE_IP" >/dev/null 2>&1; then + echo " 🌐 Remote: $REMOTE_IP │ reachable ✅" + else + echo " 🌐 Remote: $REMOTE_IP │ not responding ⚠️" + fi + else + echo " 🌐 Remote: $HOST2 not visible on Tailscale ❌" + fi +else + echo " 🌐 Tailscale: not available" +fi + +echo " 📡 Check interval: ${FAILOVER_CHECK_INTERVAL}s │ Handback strikes: ${FAILOVER_HANDBACK_STRIKES}" + +# ----------------------------------------------------------------------------------------------- +# ━━━ Footer ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +# Overall status +ISSUES=0 +[[ "$SYS_RUNNING" == false ]] && ((ISSUES++)) +[[ "$DOCKER_RUNNING" == false ]] && ((ISSUES++)) +[[ "$FAILOVER_RUNNING" == false ]] && ((ISSUES++)) +[[ -n "$ACTIVE_STRIKES" ]] && ((ISSUES++)) +[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && ((ISSUES++)) +[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && ((ISSUES++)) +[[ "$FAILOVER_STATE" != "NORMAL" ]] && [[ "$FAILOVER_STATE" != "UNKNOWN" ]] && ((ISSUES++)) + +if [[ "$ISSUES" -eq 0 ]]; then + echo " ✅ All continuous scripts healthy — no issues detected" +else + echo " ⚠️ $ISSUES issue(s) detected — review above" +fi + +echo " 🕐 Checked at: $(date '+%H:%M:%S')" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" \ No newline at end of file diff --git a/Orchestrators/sunday_morning_coffee_report.sh b/Orchestrators/sunday_morning_coffee_report.sh index 8b8cad3..9030253 100644 --- a/Orchestrators/sunday_morning_coffee_report.sh +++ b/Orchestrators/sunday_morning_coffee_report.sh @@ -11,11 +11,11 @@ # Sections: # 🖥️ System — uptime, memory, boot drive, cache drive, reboots # 📀 Array — disk count, parity status, ZFS health, drive temps -# 🔀 Failover — state, last change, Tailscale connectivity # 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split # 🎵 Media Activity — arr cleanup stats, arr recovery stats, library health # 🌐 Rsync — weekly transfer totals, per-share breakdown -# 🛡️ Watchdog — container strikes, restarts, system strikes, skip list +# 🛡️ Watchdog — system watchdog, docker watchdog, failover (all continuous loops) +# strikes, skip list, restarts, system snapshot, container overview # 🔐 Security — SSL cert expiry per domain # 📊 Emby — weekly stream count, top users, top content # ⚙️ Health — SMART summary, docker container count, Gitea sync status @@ -225,36 +225,6 @@ if command -v smartctl >/dev/null 2>&1; then [[ -n "$TEMP_SUMMARY" ]] && line "Temps: $TEMP_SUMMARY" fi -# ----------------------------------------------------------------------------------------------- -# ━━━ 🔀 Failover ━━━ -# ----------------------------------------------------------------------------------------------- -section "🔀 FAILOVER" - -if [[ -f "$FAILOVER_STATE_FILE" ]]; then - FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) - FAILOVER_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) - if [[ "$FAILOVER_STATE" == "NORMAL" ]]; then - line "State: NORMAL ✅" - else - issue "Failover state: $FAILOVER_STATE — not normal" - fi - line "Last change: ${FAILOVER_CHANGE:-unknown}" -else - issue "Failover state file not found" -fi - -# Tailscale connectivity -if command -v tailscale >/dev/null 2>&1; then - TS_STATUS=$(tailscale status 2>/dev/null) - REMOTE_VISIBLE=$(echo "$TS_STATUS" | grep -c "$HOST2" 2>/dev/null || echo 0) - if [[ "$REMOTE_VISIBLE" -gt 0 ]]; then - REMOTE_IP=$(tailscale ip -4 "$HOST2" 2>/dev/null || echo "unknown") - line "Tailscale: $HOST2 visible at $REMOTE_IP ✅" - else - issue "Tailscale: $HOST2 not visible — check connectivity" - fi -fi - # ----------------------------------------------------------------------------------------------- # ━━━ 🎬 Transcodes ━━━ # ----------------------------------------------------------------------------------------------- @@ -394,35 +364,143 @@ fi # ----------------------------------------------------------------------------------------------- section "🛡️ WATCHDOG" -# Container watchdog strikes -if [[ -f "$WATCHDOG_STATE_FILE" ]]; then - ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$" | wc -l) - if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then - STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ') - issue "Container watchdog: $ACTIVE_STRIKES active strikes — $STRIKE_LIST" - else - line "Container watchdog: no active strikes ✅" +get_lock_pid_cr() { + local script_name="$1" + local lockfile="$LOCK_DIR/${script_name}.lock" + if [[ -f "$lockfile" ]]; then + local content + content=$(cat "$lockfile" 2>/dev/null) + echo "${content%%:*}" fi +} + +get_lock_name_cr() { + local script_name="$1" + local lockfile="$LOCK_DIR/${script_name}.lock" + if [[ -f "$lockfile" ]]; then + local content + content=$(cat "$lockfile" 2>/dev/null) + echo "${content##*:}" + fi +} + +is_loop_running() { + local script_name="$1" + local pid name + pid=$(get_lock_pid_cr "$script_name") + name=$(get_lock_name_cr "$script_name") + [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$name" == "$script_name" ]] +} + +get_lock_age_cr() { + local script_name="$1" + local lockfile="$LOCK_DIR/${script_name}.lock" + if [[ -f "$lockfile" ]]; then + local mtime + mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0) + echo $(( $(date +%s) - mtime )) + else + echo 0 + fi +} + +format_uptime_cr() { + local seconds=$1 + local days=$(( seconds / 86400 )) + local hours=$(( (seconds % 86400) / 3600 )) + local mins=$(( (seconds % 3600) / 60 )) + if (( days > 0 )); then echo "${days}d ${hours}h ${mins}m" + elif (( hours > 0 )); then echo "${hours}h ${mins}m" + else echo "${mins}m"; fi +} + +# ── System Watchdog ── +line "⚙️ System Watchdog" + +SYS_PID_CR=$(get_lock_pid_cr "system_watchdog") +if is_loop_running "system_watchdog"; then + SYS_AGE_CR=$(get_lock_age_cr "system_watchdog") + SYS_UP_CR=$(format_uptime_cr "$SYS_AGE_CR") + SYS_CYCLE_CR=$(( SYS_AGE_CR / SYSTEM_WATCHDOG_INTERVAL )) + line " ✅ Running │ PID: $SYS_PID_CR │ Uptime: $SYS_UP_CR │ ~Cycle: $SYS_CYCLE_CR" +else + issue "system_watchdog NOT RUNNING" fi -# System watchdog strikes +# System strikes if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then - SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$" | wc -l) - if [[ "$SYS_STRIKES" -gt 0 ]]; then - SYS_LIST=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ') - issue "System watchdog: $SYS_STRIKES active strikes — $SYS_LIST" + SYS_ACTIVE=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") + if [[ -n "$SYS_ACTIVE" ]]; then + while IFS=: read -r key count; do + [[ -z "$key" ]] && continue + issue " ⚠️ Strike: $key — $count/$SYS_WATCHDOG_STRIKE_LIMIT" + done <<< "$SYS_ACTIVE" else - line "System watchdog: no active strikes ✅" + line " ✅ Strikes: none" fi fi -# Container skip list +# Reboots this week +if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then + WEEK_EPOCH=$(date -d "$WEEK_START" +%s) + REBOOT_COUNT=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \ + "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l) + if [[ "${REBOOT_COUNT:-0}" -gt 0 ]]; then + issue " 🔄 Reboots this week: $REBOOT_COUNT (watchdog triggered)" + else + line " ✅ Reboots this week: 0" + fi +fi + +# Skip list if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE") - SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ') - issue "Skip list: $SKIP_COUNT containers — $SKIP_LIST — manual intervention needed" + SKIP_LIST=$(tr '\n' ' ' < "$SYS_WATCHDOG_FAILED_FILE") + issue " ⛔ Skip list ($SKIP_COUNT): $SKIP_LIST" else - line "Skip list: empty ✅" + line " ✅ Skip list: empty" +fi + +# System snapshot +ROOTFS_PCT_CR=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') +MEM_AVAIL_CR=$(awk '/MemAvailable/ {printf "%.1f", $2/1048576}' /proc/meminfo) +MEM_TOTAL_CR=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo) +LOAD_CR=$(awk '{print $1}' /proc/loadavg) +ZOMBIE_CR=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0) +ZOMBIE_CR="${ZOMBIE_CR//[^0-9]/}"; ZOMBIE_CR="${ZOMBIE_CR:-0}" +ARC_GB_CR="n/a" +if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then + ARC_B=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_GB_CR=$(awk "BEGIN {printf \"%.1f\", $ARC_B / 1073741824}") +fi +line " 📊 rootfs: ${ROOTFS_PCT_CR}% │ RAM: ${MEM_AVAIL_CR}GB free/${MEM_TOTAL_CR}GB │ ARC: ${ARC_GB_CR}GB │ load: ${LOAD_CR} │ zombies: ${ZOMBIE_CR}" + +REPORT+=("") + +# ── Docker Watchdog ── +line "🐳 Docker Watchdog" + +DOCKER_PID_CR=$(get_lock_pid_cr "docker_watchdog") +if is_loop_running "docker_watchdog"; then + DOCKER_AGE_CR=$(get_lock_age_cr "docker_watchdog") + DOCKER_UP_CR=$(format_uptime_cr "$DOCKER_AGE_CR") + DOCKER_CYCLE_CR=$(( DOCKER_AGE_CR / DOCKER_WATCHDOG_INTERVAL )) + line " ✅ Running │ PID: $DOCKER_PID_CR │ Uptime: $DOCKER_UP_CR │ ~Cycle: $DOCKER_CYCLE_CR" +else + issue "docker_watchdog NOT RUNNING" +fi + +# Container strikes +if [[ -f "$WATCHDOG_STATE_FILE" ]]; then + DOCK_ACTIVE=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") + if [[ -n "$DOCK_ACTIVE" ]]; then + while IFS=: read -r key count; do + [[ -z "$key" ]] && continue + issue " ⚠️ Strike: $key — $count" + done <<< "$DOCK_ACTIVE" + else + line " ✅ Container strikes: none" + fi fi # Container restarts this week @@ -433,24 +511,94 @@ if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then RESTARTED=$(awk -F'|' -v cutoff="$WEEK_START" \ '$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \ sort | uniq -c | sort -rn | head -5 | \ - awk '{print $2 "(" $1 ")"}' | tr '\n' ' ') - finding "Watchdog restarted $WEEK_RESTARTS containers this week: $RESTARTED" + awk '{print $2"("$1")"}' | tr '\n' ' ') + finding " 🔄 Restarts this week: $WEEK_RESTARTS — $RESTARTED" else - line "Watchdog restarts: none this week ✅" + line " ✅ Container restarts this week: none" fi fi -# Docker container count +# Docker overview if command -v docker >/dev/null 2>&1; then - RUNNING=$(docker ps -q 2>/dev/null | wc -l) - TOTAL=$(docker ps -aq 2>/dev/null | wc -l) - STOPPED=$(( TOTAL - RUNNING )) - if [[ "$STOPPED" -gt 0 ]]; then - STOPPED_NAMES=$(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null | \ - head -5 | tr '\n' ' ') - finding "Docker: $RUNNING/$TOTAL running — $STOPPED stopped: $STOPPED_NAMES" + RUNNING_CR=$(docker ps -q 2>/dev/null | wc -l) + TOTAL_CR=$(docker ps -aq 2>/dev/null | wc -l) + UNHEALTHY_CR=$(docker ps --filter health=unhealthy -q 2>/dev/null | wc -l) + STOPPED_CR=$(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null | head -5 | tr '\n' ' ') + STOPPED_COUNT_CR=$(docker ps -af "status=exited" -q 2>/dev/null | wc -l) + line " 📦 $RUNNING_CR/$TOTAL_CR running │ unhealthy: $UNHEALTHY_CR" + if [[ "${STOPPED_COUNT_CR:-0}" -gt 0 ]]; then + issue " ⚠️ Stopped: $STOPPED_CR" else - line "Docker: $RUNNING/$TOTAL containers running ✅" + line " ✅ All containers running" + fi +fi + +REPORT+=("") + +# ── Failover ── +line "🔀 Failover" + +FAILOVER_PID_CR=$(get_lock_pid_cr "failover") +if is_loop_running "failover"; then + FAILOVER_AGE_CR=$(get_lock_age_cr "failover") + FAILOVER_UP_CR=$(format_uptime_cr "$FAILOVER_AGE_CR") + line " ✅ Running │ PID: $FAILOVER_PID_CR │ Uptime: $FAILOVER_UP_CR" +else + issue "failover NOT RUNNING" +fi + +FAILOVER_STATE_CR="UNKNOWN" +FAILOVER_STATE_SECS_CR=0 +if [[ -f "$FAILOVER_STATE_FILE" ]]; then + FAILOVER_STATE_CR=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + FAILOVER_LAST_EPOCH_CR=$(grep "^last_change_epoch=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + [[ -n "$FAILOVER_LAST_EPOCH_CR" ]] && \ + FAILOVER_STATE_SECS_CR=$(( $(date +%s) - FAILOVER_LAST_EPOCH_CR )) +fi + +STATE_DUR_CR=$(format_uptime_cr "${FAILOVER_STATE_SECS_CR:-0}") + +case "$FAILOVER_STATE_CR" in + NORMAL) + line " ✅ State: NORMAL │ Duration: $STATE_DUR_CR" + ;; + FAILOVER) + issue " ⚠️ State: FAILOVER — remote down for $STATE_DUR_CR" + FAILOVER_MINS_CR=$(( FAILOVER_STATE_SECS_CR / 60 )) + if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + T2=$HOST2_TIER2_DELAY; T3=$HOST2_TIER3_DELAY; T4=$HOST2_TIER4_DELAY + else + T2=$HOST1_TIER2_DELAY; T3=$HOST1_TIER3_DELAY; T4=$HOST1_TIER4_DELAY + fi + (( FAILOVER_MINS_CR >= T2 )) && line " 🔄 Tier 2: ✅ active" || \ + line " 🔄 Tier 2: ⏳ in $(( T2 - FAILOVER_MINS_CR ))min" + (( FAILOVER_MINS_CR >= T3 )) && line " 🔄 Tier 3: ✅ active" || \ + line " 🔄 Tier 3: ⏳ in $(( T3 - FAILOVER_MINS_CR ))min" + (( FAILOVER_MINS_CR >= T4 )) && line " 🔄 Tier 4: ✅ active" || \ + line " 🔄 Tier 4: ⏳ in $(( T4 - FAILOVER_MINS_CR ))min" + ;; + NO_INTERNET) + issue " ❌ State: NO_INTERNET — DDNS stopped │ Duration: $STATE_DUR_CR" + ;; + DARK) + issue " ❌ State: DARK — remote down AND no internet │ Duration: $STATE_DUR_CR" + ;; + *) + finding " ❓ State: ${FAILOVER_STATE_CR:-unknown}" + ;; +esac + +# Tailscale +if command -v tailscale >/dev/null 2>&1; then + REMOTE_IP_CR=$(tailscale ip -4 "$HOST2" 2>/dev/null) + if [[ -n "$REMOTE_IP_CR" ]]; then + if ping -c 1 -W 2 "$REMOTE_IP_CR" >/dev/null 2>&1; then + line " 🌐 $HOST2: $REMOTE_IP_CR — reachable ✅" + else + issue " 🌐 $HOST2: $REMOTE_IP_CR — not responding" + fi + else + issue " 🌐 $HOST2 not visible on Tailscale" fi fi diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index 44151e9..1797709 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -49,20 +49,13 @@ # ├── user_script_plug-in.sh # This file — copy into User Scripts plugin # ├── git_pull_execute.sh # Pulls latest scripts from Gitea repo # │ -# ├── Orchestrators/ -# │ ├── array_start.sh # Single entry point — launches all array-start scripts -# │ ├── daily_sync_maintenance.sh # Daily — git pull, media sync, media mgmt, docker restart -# │ ├── weekly_sync_maintenance.sh # Weekly — critical sync + updates, docker weekly restart -# │ ├── media_management.sh # Permissions + cleaners + arr cleanup — run manually -# │ ├── transcode_management.sh # Cleanup then manager every 3min + daily stats -# │ └── README-Orchestrators.md -# │ # ├── Failover/ # │ ├── failover.sh # Mutual container failover — runs continuously # │ ├── failover_test.sh # Controlled failover simulation — run manually # │ └── README-Failover.md # │ # ├── Monitors/ +# │ ├── continuous_scripts_status.sh # Live status of all continuous loop scripts — run manually # │ ├── backup_verify.sh # Random sample checksum verification vs remote # │ ├── bandwidth_monitor.sh # Daily transfer logging + weekly summary # │ ├── cert_monitor.sh # SSL certificate expiry — direct openssl check @@ -72,6 +65,16 @@ # │ ├── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report # │ └── README-Monitors.md # │ +# ├── Orchestrators/ +# │ ├── array_start.sh # Single entry point — launches all array-start scripts +# │ ├── daily_sync_maintenance.sh # Daily — git pull, media sync, media mgmt, docker restart +# │ ├── weekly_sync_maintenance.sh # Weekly — critical sync + updates, docker weekly restart +# │ ├── daily_sync_maintenance.sh # Media shares sync both directions +# │ ├── weekly_sync_maintenance.sh # Clean sync + container updates (Emby + auth stack) +# │ ├── media_management.sh # Permissions + cleaners + arr cleanup — run manually +# │ ├── transcode_management.sh # Cleanup then manager every 3min + daily stats +# │ └── README-Orchestrators.md +# │ # ├── Rsync/ # │ ├── rsync.sh # Core rsync script — called per share or profile # │ └── README-Rsync_Setup.md # Rsync-specific setup guide @@ -135,8 +138,22 @@ # in real time — symlink flips work correctly for the lifetime of the container. # # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ 🚀 Orchestrators — Uncomment the one below you want to run ━━━ +# ━━━ 🚀 Script Commands — Uncomment the one you want to run ━━━ # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# +# ━━━ Failover ━━━ +# Runs continuously as background task — set schedule to "At Startup of Array (Background)". +# Both servers must have this running for mutual failover to work. +# +#/mnt/user/appdata/unraid_scripts/Failover/failover.sh +# +# ━━━ Failover Test ━━━ +# Run manually during a maintenance window — starts and stops real containers. +# Always --dry-run first to walk through phases without making changes. +# +#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run +#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh +# # ━━━ Orchestrators ━━━ # Orchestrators are the main scripts — they handle most if not all needed scripts # in a specific order of operations. Each orchestrator owns a domain and runs @@ -166,7 +183,6 @@ # arrs_failed_stalled_recovery.sh — 0 */6 * * * (every 6 hours) # blocklist + re-search failed imports and stalled downloads # across Sonarr, Radarr, and Lidarr automatically -# ━━━━━━━━━━━━━━━━━━━━━━ # #/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh #/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh @@ -175,32 +191,6 @@ #/mnt/user/appdata/unraid_scripts/Orchestrators/sunday_morning_coffee_report.sh #/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh # -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# ━━━ 🚀 Script Commands — Uncomment the one you want to run ━━━ -# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -# -# ━━━ Failover ━━━ -# Runs continuously as background task — set schedule to "At Startup of Array (Background)". -# Both servers must have this running for mutual failover to work. -# -#/mnt/user/appdata/unraid_scripts/Failover/failover.sh -# -# ━━━ Failover Test ━━━ -# Run manually during a maintenance window — starts and stops real containers. -# Always --dry-run first to walk through phases without making changes. -# -#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run -#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh -# -# ━━━ Monitors ━━━ -#/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh -#/mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report -#/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh -#/mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh -#/mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh -#/mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh -#/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh -# # ━━━ Rsync — Appdata Profiles ━━━ #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Critical-Data @@ -253,6 +243,17 @@ #/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh # ^^ schedule: 0 5 * * * (5am daily) — always dry-run first # +# ━━━ Monitors ━━━ +#/mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh +# ^^ run manually anytime — live status of system_watchdog, docker_watchdog, failover +#/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh +#/mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report +#/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh +#/mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh +#/mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh +#/mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh +#/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh +# # ━━━ Transcodes ━━━ # transcode_management.sh runs cleanup then manager — schedule that, not the individuals. # ramdisk_setup.sh runs once at array start.