added /mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh
This commit is contained in:
+3
-3
@@ -944,7 +944,7 @@ LIDARR_PROTECTED_PATTERNS=(
|
||||
# NEVER deleted — cover art, metadata, lyrics
|
||||
# Lidarr generates these but doesn't include them in trackFile API
|
||||
# Without this protection cleanup would delete all your artwork
|
||||
LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this
|
||||
LIDARR_MAX_DELETE_GB=5 # require --i-know-what-im-doing if deletion exceeds this
|
||||
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||
# protects against API returning partial data on a bad day
|
||||
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
|
||||
@@ -975,7 +975,7 @@ declare -A HOST2_SONARR_PATH_MAP=(
|
||||
)
|
||||
|
||||
SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion
|
||||
SONARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this
|
||||
SONARR_MAX_DELETE_GB=10 # require --i-know-what-im-doing if deletion exceeds this
|
||||
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
|
||||
SONARR_PROTECTED_PATTERNS=(
|
||||
# Subtitles
|
||||
@@ -1020,7 +1020,7 @@ declare -A HOST2_RADARR_PATH_MAP=(
|
||||
)
|
||||
|
||||
RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion
|
||||
RADARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this
|
||||
RADARR_MAX_DELETE_GB=15 # require --i-know-what-im-doing if deletion exceeds this
|
||||
RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
|
||||
RADARR_PROTECTED_PATTERNS=(
|
||||
# Subtitles
|
||||
|
||||
@@ -0,0 +1,486 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------------------- Continuous Scripts Status --------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Live status dashboard for all continuously running scripts in the ecosystem.
|
||||
# Run manually anytime — no schedule, no cron.
|
||||
#
|
||||
# Covers all scripts started by array_start.sh that run until array stops:
|
||||
# system_watchdog.sh — system health monitor
|
||||
# docker_watchdog.sh — container health monitor
|
||||
# failover.sh — mutual failover monitor
|
||||
#
|
||||
# Shows for each:
|
||||
# Running state, PID, uptime, current cycle
|
||||
# Active strikes, skip list, recent actions
|
||||
# Live system/container health snapshot
|
||||
# Failover state, tier status, Tailscale connectivity
|
||||
#
|
||||
# If a script is mid-cycle state files are read as-is — reflects last completed cycle.
|
||||
# Run: bash Monitors/continuous_scripts_status.sh
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# HELPERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
get_lock_pid() {
|
||||
local script_name="$1"
|
||||
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||||
if [[ -f "$lockfile" ]]; then
|
||||
local content
|
||||
content=$(cat "$lockfile" 2>/dev/null)
|
||||
echo "${content%%:*}"
|
||||
fi
|
||||
}
|
||||
|
||||
get_lock_name() {
|
||||
local script_name="$1"
|
||||
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||||
if [[ -f "$lockfile" ]]; then
|
||||
local content
|
||||
content=$(cat "$lockfile" 2>/dev/null)
|
||||
echo "${content##*:}"
|
||||
fi
|
||||
}
|
||||
|
||||
is_watchdog_running() {
|
||||
local script_name="$1"
|
||||
local pid
|
||||
pid=$(get_lock_pid "$script_name")
|
||||
local locked_name
|
||||
locked_name=$(get_lock_name "$script_name")
|
||||
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$locked_name" == "$script_name" ]]
|
||||
}
|
||||
|
||||
get_lock_age() {
|
||||
local script_name="$1"
|
||||
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||||
if [[ -f "$lockfile" ]]; then
|
||||
local mtime now
|
||||
mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0)
|
||||
now=$(date +%s)
|
||||
echo $(( now - mtime ))
|
||||
else
|
||||
echo 0
|
||||
fi
|
||||
}
|
||||
|
||||
format_uptime() {
|
||||
local seconds=$1
|
||||
local days=$(( seconds / 86400 ))
|
||||
local hours=$(( (seconds % 86400) / 3600 ))
|
||||
local mins=$(( (seconds % 3600) / 60 ))
|
||||
if (( days > 0 )); then
|
||||
echo "${days}d ${hours}h ${mins}m"
|
||||
elif (( hours > 0 )); then
|
||||
echo "${hours}h ${mins}m"
|
||||
else
|
||||
echo "${mins}m"
|
||||
fi
|
||||
}
|
||||
|
||||
get_strikes() {
|
||||
local state_file="$1"
|
||||
local key="$2"
|
||||
grep -E "^${key}:" "$state_file" 2>/dev/null | cut -d: -f2
|
||||
}
|
||||
|
||||
divider() { printf '%.0s─' {1..55}; echo; }
|
||||
header() { echo ""; echo " $1"; divider; }
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ HEADER ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
clear
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " 🛡️ WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')"
|
||||
echo " 🖥️ $(hostname)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ System Watchdog ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
header "⚙️ SYSTEM WATCHDOG"
|
||||
|
||||
SYS_PID=$(get_lock_pid "system_watchdog")
|
||||
SYS_RUNNING=false
|
||||
|
||||
if is_watchdog_running "system_watchdog"; then
|
||||
SYS_RUNNING=true
|
||||
SYS_AGE=$(get_lock_age "system_watchdog")
|
||||
SYS_UPTIME=$(format_uptime "$SYS_AGE")
|
||||
SYS_CYCLE=$(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL ))
|
||||
echo " ✅ Running │ PID: $SYS_PID │ Uptime: $SYS_UPTIME │ ~Cycle: $SYS_CYCLE"
|
||||
echo " ⏱️ Interval: ${SYSTEM_WATCHDOG_INTERVAL}s │ Heartbeat every: ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS}hr"
|
||||
else
|
||||
echo " ❌ NOT RUNNING — system_watchdog.sh is not active"
|
||||
echo " Start via: bash Orchestrators/array_start.sh"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# System watchdog strikes
|
||||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||||
ACTIVE_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||||
if [[ -n "$ACTIVE_STRIKES" ]]; then
|
||||
echo " ⚠️ Active strikes:"
|
||||
while IFS=: read -r key count; do
|
||||
[[ -z "$key" ]] && continue
|
||||
echo " → $key: $count/$SYS_WATCHDOG_STRIKE_LIMIT"
|
||||
done <<< "$ACTIVE_STRIKES"
|
||||
else
|
||||
echo " ✅ Strikes: none"
|
||||
fi
|
||||
else
|
||||
echo " ℹ️ Strike state file not found (watchdog may not have run yet)"
|
||||
fi
|
||||
|
||||
# Reboot log
|
||||
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
|
||||
TOTAL_REBOOTS=$(grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0)
|
||||
WEEK_EPOCH=$(date -d "7 days ago" +%s)
|
||||
WEEK_REBOOTS=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \
|
||||
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
|
||||
echo " 🔄 Watchdog reboots: $WEEK_REBOOTS this week / $TOTAL_REBOOTS total"
|
||||
fi
|
||||
|
||||
# Container skip list
|
||||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||||
echo ""
|
||||
echo " ⛔ Skip list ($SKIP_COUNT containers — manual intervention needed):"
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
echo " → $container"
|
||||
done < "$SYS_WATCHDOG_FAILED_FILE"
|
||||
else
|
||||
echo " ✅ Skip list: empty"
|
||||
fi
|
||||
|
||||
# Current system health snapshot
|
||||
echo ""
|
||||
echo " 📊 Current system state:"
|
||||
|
||||
# rootfs
|
||||
ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||||
[[ "${ROOTFS_PCT:-0}" -ge "${SYS_WATCHDOG_ROOTFS_PCT:-95}" ]] && \
|
||||
ROOTFS_ICON="⚠️ " || ROOTFS_ICON="✅"
|
||||
echo " ${ROOTFS_ICON} rootfs: ${ROOTFS_PCT}% (threshold: ${SYS_WATCHDOG_ROOTFS_PCT}%)"
|
||||
|
||||
# RAM
|
||||
MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}")
|
||||
MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
|
||||
[[ $(printf "%.0f" "$MEM_FREE_GB") -lt "${SYS_WATCHDOG_MEM_GB:-4}" ]] && \
|
||||
MEM_ICON="⚠️ " || MEM_ICON="✅"
|
||||
echo " ${MEM_ICON} RAM: ${MEM_FREE_GB}GB free / ${MEM_TOTAL_GB}GB total (threshold: ${SYS_WATCHDOG_MEM_GB}GB free)"
|
||||
|
||||
# ARC
|
||||
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||||
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX ))
|
||||
ARC_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}")
|
||||
[[ "$ARC_PCT" -ge "${SYS_WATCHDOG_ARC_PINNED_PCT:-98}" ]] && \
|
||||
ARC_ICON="⚠️ " || ARC_ICON="✅"
|
||||
echo " ${ARC_ICON} ZFS ARC: ${ARC_GB}GB (${ARC_PCT}% of max, threshold: ${SYS_WATCHDOG_ARC_PINNED_PCT}%)"
|
||||
fi
|
||||
|
||||
# Load
|
||||
LOAD=$(awk '{print $1}' /proc/loadavg)
|
||||
CORES=$(nproc)
|
||||
LOAD_THRESH=$(( CORES * ${SYS_WATCHDOG_LOAD_MULTIPLIER:-3} ))
|
||||
LOAD_INT=$(printf "%.0f" "$LOAD")
|
||||
[[ "$LOAD_INT" -ge "$LOAD_THRESH" ]] && LOAD_ICON="⚠️ " || LOAD_ICON="✅"
|
||||
echo " ${LOAD_ICON} Load avg: $LOAD (threshold: ${LOAD_THRESH} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}x ${CORES} cores)"
|
||||
|
||||
# Zombies
|
||||
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0)
|
||||
ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"
|
||||
ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
|
||||
[[ "$ZOMBIE_COUNT" -ge "${SYS_WATCHDOG_ZOMBIE_LIMIT:-50}" ]] && \
|
||||
ZOMBIE_ICON="⚠️ " || ZOMBIE_ICON="✅"
|
||||
echo " ${ZOMBIE_ICON} Zombies: $ZOMBIE_COUNT (threshold: ${SYS_WATCHDOG_ZOMBIE_LIMIT})"
|
||||
|
||||
# CPU temp
|
||||
if command -v sensors >/dev/null 2>&1; then
|
||||
CPU_TEMP=$(sensors 2>/dev/null | \
|
||||
grep -i "Package id 0\|Tctl\|CPU Temp" | \
|
||||
awk '{print $NF}' | tr -d '+°C' | head -1)
|
||||
if [[ -n "$CPU_TEMP" ]]; then
|
||||
CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP")
|
||||
[[ "$CPU_TEMP_INT" -ge "${SYS_WATCHDOG_CPU_TEMP_MAX:-95}" ]] && \
|
||||
TEMP_ICON="⚠️ " || TEMP_ICON="✅"
|
||||
echo " ${TEMP_ICON} CPU temp: ${CPU_TEMP_INT}°C (threshold: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
header "🐳 DOCKER WATCHDOG"
|
||||
|
||||
DOCKER_PID=$(get_lock_pid "docker_watchdog")
|
||||
DOCKER_RUNNING=false
|
||||
|
||||
if is_watchdog_running "docker_watchdog"; then
|
||||
DOCKER_RUNNING=true
|
||||
DOCKER_AGE=$(get_lock_age "docker_watchdog")
|
||||
DOCKER_UPTIME=$(format_uptime "$DOCKER_AGE")
|
||||
DOCKER_CYCLE=$(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL ))
|
||||
echo " ✅ Running │ PID: $DOCKER_PID │ Uptime: $DOCKER_UPTIME │ ~Cycle: $DOCKER_CYCLE"
|
||||
echo " ⏱️ Interval: ${DOCKER_WATCHDOG_INTERVAL}s │ Heartbeat every: ${DOCKER_WATCHDOG_HEARTBEAT_HOURS}hr"
|
||||
else
|
||||
echo " ❌ NOT RUNNING — docker_watchdog.sh is not active"
|
||||
echo " Start via: bash Orchestrators/array_start.sh"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Container watchdog strikes
|
||||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
ACTIVE_CONTAINER_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||||
if [[ -n "$ACTIVE_CONTAINER_STRIKES" ]]; then
|
||||
echo " ⚠️ Active container strikes:"
|
||||
while IFS=: read -r key count; do
|
||||
[[ -z "$key" ]] && continue
|
||||
echo " → $key: $count"
|
||||
done <<< "$ACTIVE_CONTAINER_STRIKES"
|
||||
else
|
||||
echo " ✅ Container strikes: none"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Container restart history this week
|
||||
if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
|
||||
WEEK_EPOCH=$(date -d "7 days ago" +%s 2>/dev/null || date -v-7d +%s 2>/dev/null)
|
||||
WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_EPOCH" \
|
||||
'NR>0 {if ($2 >= cutoff) count++} END {print count+0}' \
|
||||
"$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null)
|
||||
if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " 🔄 Container restarts this week: $WEEK_RESTARTS"
|
||||
awk -F'|' -v cutoff="$WEEK_EPOCH" \
|
||||
'$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
|
||||
sort | uniq -c | sort -rn | head -5 | \
|
||||
while read -r count name; do
|
||||
echo " → $name: $count restart(s)"
|
||||
done
|
||||
else
|
||||
echo " ✅ Container restarts this week: none"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Docker container overview
|
||||
echo ""
|
||||
echo " 📦 Container overview:"
|
||||
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
RUNNING=$(docker ps -q 2>/dev/null | wc -l)
|
||||
TOTAL=$(docker ps -aq 2>/dev/null | wc -l)
|
||||
UNHEALTHY=$(docker ps --filter health=unhealthy -q 2>/dev/null | wc -l)
|
||||
STOPPED=$(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null)
|
||||
STOPPED_COUNT=$(echo "$STOPPED" | grep -c . 2>/dev/null || echo 0)
|
||||
|
||||
echo " Running: $RUNNING / $TOTAL total"
|
||||
[[ "$UNHEALTHY" -gt 0 ]] && echo " ⚠️ Unhealthy: $UNHEALTHY"
|
||||
if [[ "$STOPPED_COUNT" -gt 0 ]]; then
|
||||
echo " ⚠️ Stopped containers:"
|
||||
echo "$STOPPED" | head -10 | while IFS= read -r name; do
|
||||
[[ -z "$name" ]] && continue
|
||||
echo " → $name"
|
||||
done
|
||||
else
|
||||
echo " ✅ All containers running"
|
||||
fi
|
||||
|
||||
# Check required containers
|
||||
detect_hosts 2>/dev/null
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
REQUIRED=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
else
|
||||
REQUIRED=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
fi
|
||||
|
||||
REQUIRED_ISSUES=0
|
||||
if [[ ${#REQUIRED[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " 🔐 Required containers:"
|
||||
for container in "${REQUIRED[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
echo " ✅ $container"
|
||||
else
|
||||
echo " ❌ $container — $STATUS"
|
||||
((REQUIRED_ISSUES++))
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Tier 1 monitored containers from WATCHDOG_CONTAINERS
|
||||
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " 📊 Monitored containers (memory):"
|
||||
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
|
||||
LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
|
||||
LIMIT_GB=$(awk "BEGIN {printf \"%.0f\", $LIMIT_MB / 1024}")
|
||||
USAGE=$(docker stats --no-stream --format "{{.MemUsage}}" "$container" \
|
||||
2>/dev/null | awk '{print $1}')
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
echo " ✅ $container: ${USAGE:-?} (limit: ${LIMIT_GB}GB)"
|
||||
else
|
||||
echo " ❌ $container: not running (limit: ${LIMIT_GB}GB)"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
else
|
||||
echo " Docker not available"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ Failover ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
header "🔀 FAILOVER"
|
||||
|
||||
FAILOVER_PID=$(get_lock_pid "failover")
|
||||
FAILOVER_RUNNING=false
|
||||
|
||||
if is_watchdog_running "failover"; then
|
||||
FAILOVER_RUNNING=true
|
||||
FAILOVER_AGE=$(get_lock_age "failover")
|
||||
FAILOVER_UPTIME=$(format_uptime "$FAILOVER_AGE")
|
||||
echo " ✅ Running │ PID: $FAILOVER_PID │ Uptime: $FAILOVER_UPTIME"
|
||||
else
|
||||
echo " ❌ NOT RUNNING — failover.sh is not active"
|
||||
echo " Start via: bash Orchestrators/array_start.sh"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Failover state
|
||||
FAILOVER_STATE="UNKNOWN"
|
||||
FAILOVER_LAST_CHANGE=""
|
||||
FAILOVER_STATE_SECONDS=0
|
||||
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
FAILOVER_LAST_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
FAILOVER_LAST_EPOCH=$(grep "^last_change_epoch=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
if [[ -n "$FAILOVER_LAST_EPOCH" ]]; then
|
||||
FAILOVER_STATE_SECONDS=$(( $(date +%s) - FAILOVER_LAST_EPOCH ))
|
||||
fi
|
||||
fi
|
||||
|
||||
STATE_DURATION=$(format_uptime "${FAILOVER_STATE_SECONDS:-0}")
|
||||
|
||||
case "$FAILOVER_STATE" in
|
||||
NORMAL)
|
||||
echo " ✅ State: NORMAL"
|
||||
echo " 📅 In NORMAL state for: $STATE_DURATION"
|
||||
;;
|
||||
FAILOVER)
|
||||
echo " ⚠️ State: FAILOVER — remote server down"
|
||||
echo " ⏱️ Duration: $STATE_DURATION"
|
||||
# Show which tiers are active
|
||||
TIER1_DELAY=0
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
TIER2_DELAY=$HOST2_TIER2_DELAY
|
||||
TIER3_DELAY=$HOST2_TIER3_DELAY
|
||||
TIER4_DELAY=$HOST2_TIER4_DELAY
|
||||
else
|
||||
TIER2_DELAY=$HOST1_TIER2_DELAY
|
||||
TIER3_DELAY=$HOST1_TIER3_DELAY
|
||||
TIER4_DELAY=$HOST1_TIER4_DELAY
|
||||
fi
|
||||
FAILOVER_MINS=$(( FAILOVER_STATE_SECONDS / 60 ))
|
||||
echo ""
|
||||
echo " 🔄 Tier status:"
|
||||
echo " Tier 1 (immediate): ✅ active"
|
||||
if (( FAILOVER_MINS >= TIER2_DELAY )); then
|
||||
echo " Tier 2 (${TIER2_DELAY}min): ✅ active"
|
||||
else
|
||||
REMAINING=$(( TIER2_DELAY - FAILOVER_MINS ))
|
||||
echo " Tier 2 (${TIER2_DELAY}min): ⏳ activates in ${REMAINING}min"
|
||||
fi
|
||||
if (( FAILOVER_MINS >= TIER3_DELAY )); then
|
||||
echo " Tier 3 (${TIER3_DELAY}min): ✅ active"
|
||||
else
|
||||
REMAINING=$(( TIER3_DELAY - FAILOVER_MINS ))
|
||||
echo " Tier 3 (${TIER3_DELAY}min): ⏳ activates in ${REMAINING}min"
|
||||
fi
|
||||
if (( FAILOVER_MINS >= TIER4_DELAY )); then
|
||||
echo " Tier 4 (${TIER4_DELAY}min): ✅ active"
|
||||
else
|
||||
REMAINING=$(( TIER4_DELAY - FAILOVER_MINS ))
|
||||
echo " Tier 4 (${TIER4_DELAY}min): ⏳ activates in ${REMAINING}min"
|
||||
fi
|
||||
;;
|
||||
NO_INTERNET)
|
||||
echo " ❌ State: NO_INTERNET — DDNS stopped"
|
||||
echo " ⏱️ Down for: $STATE_DURATION"
|
||||
;;
|
||||
DARK)
|
||||
echo " ❌ State: DARK — remote down AND no internet"
|
||||
echo " ⏱️ Duration: $STATE_DURATION"
|
||||
;;
|
||||
*)
|
||||
echo " ❓ State: ${FAILOVER_STATE:-unknown}"
|
||||
;;
|
||||
esac
|
||||
|
||||
# Tailscale remote visibility
|
||||
echo ""
|
||||
if command -v tailscale >/dev/null 2>&1; then
|
||||
REMOTE_IP=$(tailscale ip -4 "$HOST2" 2>/dev/null)
|
||||
if [[ -n "$REMOTE_IP" ]]; then
|
||||
# Try a quick ping to see last seen
|
||||
if ping -c 1 -W 2 "$REMOTE_IP" >/dev/null 2>&1; then
|
||||
echo " 🌐 Remote: $REMOTE_IP │ reachable ✅"
|
||||
else
|
||||
echo " 🌐 Remote: $REMOTE_IP │ not responding ⚠️"
|
||||
fi
|
||||
else
|
||||
echo " 🌐 Remote: $HOST2 not visible on Tailscale ❌"
|
||||
fi
|
||||
else
|
||||
echo " 🌐 Tailscale: not available"
|
||||
fi
|
||||
|
||||
echo " 📡 Check interval: ${FAILOVER_CHECK_INTERVAL}s │ Handback strikes: ${FAILOVER_HANDBACK_STRIKES}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ Footer ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Overall status
|
||||
ISSUES=0
|
||||
[[ "$SYS_RUNNING" == false ]] && ((ISSUES++))
|
||||
[[ "$DOCKER_RUNNING" == false ]] && ((ISSUES++))
|
||||
[[ "$FAILOVER_RUNNING" == false ]] && ((ISSUES++))
|
||||
[[ -n "$ACTIVE_STRIKES" ]] && ((ISSUES++))
|
||||
[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && ((ISSUES++))
|
||||
[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && ((ISSUES++))
|
||||
[[ "$FAILOVER_STATE" != "NORMAL" ]] && [[ "$FAILOVER_STATE" != "UNKNOWN" ]] && ((ISSUES++))
|
||||
|
||||
if [[ "$ISSUES" -eq 0 ]]; then
|
||||
echo " ✅ All continuous scripts healthy — no issues detected"
|
||||
else
|
||||
echo " ⚠️ $ISSUES issue(s) detected — review above"
|
||||
fi
|
||||
|
||||
echo " 🕐 Checked at: $(date '+%H:%M:%S')"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
@@ -11,11 +11,11 @@
|
||||
# Sections:
|
||||
# 🖥️ System — uptime, memory, boot drive, cache drive, reboots
|
||||
# 📀 Array — disk count, parity status, ZFS health, drive temps
|
||||
# 🔀 Failover — state, last change, Tailscale connectivity
|
||||
# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split
|
||||
# 🎵 Media Activity — arr cleanup stats, arr recovery stats, library health
|
||||
# 🌐 Rsync — weekly transfer totals, per-share breakdown
|
||||
# 🛡️ Watchdog — container strikes, restarts, system strikes, skip list
|
||||
# 🛡️ Watchdog — system watchdog, docker watchdog, failover (all continuous loops)
|
||||
# strikes, skip list, restarts, system snapshot, container overview
|
||||
# 🔐 Security — SSL cert expiry per domain
|
||||
# 📊 Emby — weekly stream count, top users, top content
|
||||
# ⚙️ Health — SMART summary, docker container count, Gitea sync status
|
||||
@@ -225,36 +225,6 @@ if command -v smartctl >/dev/null 2>&1; then
|
||||
[[ -n "$TEMP_SUMMARY" ]] && line "Temps: $TEMP_SUMMARY"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ 🔀 Failover ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
section "🔀 FAILOVER"
|
||||
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
FAILOVER_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
if [[ "$FAILOVER_STATE" == "NORMAL" ]]; then
|
||||
line "State: NORMAL ✅"
|
||||
else
|
||||
issue "Failover state: $FAILOVER_STATE — not normal"
|
||||
fi
|
||||
line "Last change: ${FAILOVER_CHANGE:-unknown}"
|
||||
else
|
||||
issue "Failover state file not found"
|
||||
fi
|
||||
|
||||
# Tailscale connectivity
|
||||
if command -v tailscale >/dev/null 2>&1; then
|
||||
TS_STATUS=$(tailscale status 2>/dev/null)
|
||||
REMOTE_VISIBLE=$(echo "$TS_STATUS" | grep -c "$HOST2" 2>/dev/null || echo 0)
|
||||
if [[ "$REMOTE_VISIBLE" -gt 0 ]]; then
|
||||
REMOTE_IP=$(tailscale ip -4 "$HOST2" 2>/dev/null || echo "unknown")
|
||||
line "Tailscale: $HOST2 visible at $REMOTE_IP ✅"
|
||||
else
|
||||
issue "Tailscale: $HOST2 not visible — check connectivity"
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ 🎬 Transcodes ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
@@ -394,35 +364,143 @@ fi
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
section "🛡️ WATCHDOG"
|
||||
|
||||
# Container watchdog strikes
|
||||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$" | wc -l)
|
||||
if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then
|
||||
STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
|
||||
issue "Container watchdog: $ACTIVE_STRIKES active strikes — $STRIKE_LIST"
|
||||
else
|
||||
line "Container watchdog: no active strikes ✅"
|
||||
get_lock_pid_cr() {
|
||||
local script_name="$1"
|
||||
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||||
if [[ -f "$lockfile" ]]; then
|
||||
local content
|
||||
content=$(cat "$lockfile" 2>/dev/null)
|
||||
echo "${content%%:*}"
|
||||
fi
|
||||
}
|
||||
|
||||
get_lock_name_cr() {
|
||||
local script_name="$1"
|
||||
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||||
if [[ -f "$lockfile" ]]; then
|
||||
local content
|
||||
content=$(cat "$lockfile" 2>/dev/null)
|
||||
echo "${content##*:}"
|
||||
fi
|
||||
}
|
||||
|
||||
is_loop_running() {
|
||||
local script_name="$1"
|
||||
local pid name
|
||||
pid=$(get_lock_pid_cr "$script_name")
|
||||
name=$(get_lock_name_cr "$script_name")
|
||||
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$name" == "$script_name" ]]
|
||||
}
|
||||
|
||||
get_lock_age_cr() {
|
||||
local script_name="$1"
|
||||
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||||
if [[ -f "$lockfile" ]]; then
|
||||
local mtime
|
||||
mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0)
|
||||
echo $(( $(date +%s) - mtime ))
|
||||
else
|
||||
echo 0
|
||||
fi
|
||||
}
|
||||
|
||||
format_uptime_cr() {
|
||||
local seconds=$1
|
||||
local days=$(( seconds / 86400 ))
|
||||
local hours=$(( (seconds % 86400) / 3600 ))
|
||||
local mins=$(( (seconds % 3600) / 60 ))
|
||||
if (( days > 0 )); then echo "${days}d ${hours}h ${mins}m"
|
||||
elif (( hours > 0 )); then echo "${hours}h ${mins}m"
|
||||
else echo "${mins}m"; fi
|
||||
}
|
||||
|
||||
# ── System Watchdog ──
|
||||
line "⚙️ System Watchdog"
|
||||
|
||||
SYS_PID_CR=$(get_lock_pid_cr "system_watchdog")
|
||||
if is_loop_running "system_watchdog"; then
|
||||
SYS_AGE_CR=$(get_lock_age_cr "system_watchdog")
|
||||
SYS_UP_CR=$(format_uptime_cr "$SYS_AGE_CR")
|
||||
SYS_CYCLE_CR=$(( SYS_AGE_CR / SYSTEM_WATCHDOG_INTERVAL ))
|
||||
line " ✅ Running │ PID: $SYS_PID_CR │ Uptime: $SYS_UP_CR │ ~Cycle: $SYS_CYCLE_CR"
|
||||
else
|
||||
issue "system_watchdog NOT RUNNING"
|
||||
fi
|
||||
|
||||
# System watchdog strikes
|
||||
# System strikes
|
||||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||||
SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$" | wc -l)
|
||||
if [[ "$SYS_STRIKES" -gt 0 ]]; then
|
||||
SYS_LIST=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
|
||||
issue "System watchdog: $SYS_STRIKES active strikes — $SYS_LIST"
|
||||
SYS_ACTIVE=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||||
if [[ -n "$SYS_ACTIVE" ]]; then
|
||||
while IFS=: read -r key count; do
|
||||
[[ -z "$key" ]] && continue
|
||||
issue " ⚠️ Strike: $key — $count/$SYS_WATCHDOG_STRIKE_LIMIT"
|
||||
done <<< "$SYS_ACTIVE"
|
||||
else
|
||||
line "System watchdog: no active strikes ✅"
|
||||
line " ✅ Strikes: none"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Container skip list
|
||||
# Reboots this week
|
||||
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
|
||||
WEEK_EPOCH=$(date -d "$WEEK_START" +%s)
|
||||
REBOOT_COUNT=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \
|
||||
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
|
||||
if [[ "${REBOOT_COUNT:-0}" -gt 0 ]]; then
|
||||
issue " 🔄 Reboots this week: $REBOOT_COUNT (watchdog triggered)"
|
||||
else
|
||||
line " ✅ Reboots this week: 0"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Skip list
|
||||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||||
SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ')
|
||||
issue "Skip list: $SKIP_COUNT containers — $SKIP_LIST — manual intervention needed"
|
||||
SKIP_LIST=$(tr '\n' ' ' < "$SYS_WATCHDOG_FAILED_FILE")
|
||||
issue " ⛔ Skip list ($SKIP_COUNT): $SKIP_LIST"
|
||||
else
|
||||
line "Skip list: empty ✅"
|
||||
line " ✅ Skip list: empty"
|
||||
fi
|
||||
|
||||
# System snapshot
|
||||
ROOTFS_PCT_CR=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||||
MEM_AVAIL_CR=$(awk '/MemAvailable/ {printf "%.1f", $2/1048576}' /proc/meminfo)
|
||||
MEM_TOTAL_CR=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
|
||||
LOAD_CR=$(awk '{print $1}' /proc/loadavg)
|
||||
ZOMBIE_CR=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0)
|
||||
ZOMBIE_CR="${ZOMBIE_CR//[^0-9]/}"; ZOMBIE_CR="${ZOMBIE_CR:-0}"
|
||||
ARC_GB_CR="n/a"
|
||||
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||||
ARC_B=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_GB_CR=$(awk "BEGIN {printf \"%.1f\", $ARC_B / 1073741824}")
|
||||
fi
|
||||
line " 📊 rootfs: ${ROOTFS_PCT_CR}% │ RAM: ${MEM_AVAIL_CR}GB free/${MEM_TOTAL_CR}GB │ ARC: ${ARC_GB_CR}GB │ load: ${LOAD_CR} │ zombies: ${ZOMBIE_CR}"
|
||||
|
||||
REPORT+=("")
|
||||
|
||||
# ── Docker Watchdog ──
|
||||
line "🐳 Docker Watchdog"
|
||||
|
||||
DOCKER_PID_CR=$(get_lock_pid_cr "docker_watchdog")
|
||||
if is_loop_running "docker_watchdog"; then
|
||||
DOCKER_AGE_CR=$(get_lock_age_cr "docker_watchdog")
|
||||
DOCKER_UP_CR=$(format_uptime_cr "$DOCKER_AGE_CR")
|
||||
DOCKER_CYCLE_CR=$(( DOCKER_AGE_CR / DOCKER_WATCHDOG_INTERVAL ))
|
||||
line " ✅ Running │ PID: $DOCKER_PID_CR │ Uptime: $DOCKER_UP_CR │ ~Cycle: $DOCKER_CYCLE_CR"
|
||||
else
|
||||
issue "docker_watchdog NOT RUNNING"
|
||||
fi
|
||||
|
||||
# Container strikes
|
||||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
DOCK_ACTIVE=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||||
if [[ -n "$DOCK_ACTIVE" ]]; then
|
||||
while IFS=: read -r key count; do
|
||||
[[ -z "$key" ]] && continue
|
||||
issue " ⚠️ Strike: $key — $count"
|
||||
done <<< "$DOCK_ACTIVE"
|
||||
else
|
||||
line " ✅ Container strikes: none"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Container restarts this week
|
||||
@@ -433,24 +511,94 @@ if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
|
||||
RESTARTED=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
|
||||
sort | uniq -c | sort -rn | head -5 | \
|
||||
awk '{print $2 "(" $1 ")"}' | tr '\n' ' ')
|
||||
finding "Watchdog restarted $WEEK_RESTARTS containers this week: $RESTARTED"
|
||||
awk '{print $2"("$1")"}' | tr '\n' ' ')
|
||||
finding " 🔄 Restarts this week: $WEEK_RESTARTS — $RESTARTED"
|
||||
else
|
||||
line "Watchdog restarts: none this week ✅"
|
||||
line " ✅ Container restarts this week: none"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Docker container count
|
||||
# Docker overview
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
RUNNING=$(docker ps -q 2>/dev/null | wc -l)
|
||||
TOTAL=$(docker ps -aq 2>/dev/null | wc -l)
|
||||
STOPPED=$(( TOTAL - RUNNING ))
|
||||
if [[ "$STOPPED" -gt 0 ]]; then
|
||||
STOPPED_NAMES=$(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null | \
|
||||
head -5 | tr '\n' ' ')
|
||||
finding "Docker: $RUNNING/$TOTAL running — $STOPPED stopped: $STOPPED_NAMES"
|
||||
RUNNING_CR=$(docker ps -q 2>/dev/null | wc -l)
|
||||
TOTAL_CR=$(docker ps -aq 2>/dev/null | wc -l)
|
||||
UNHEALTHY_CR=$(docker ps --filter health=unhealthy -q 2>/dev/null | wc -l)
|
||||
STOPPED_CR=$(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null | head -5 | tr '\n' ' ')
|
||||
STOPPED_COUNT_CR=$(docker ps -af "status=exited" -q 2>/dev/null | wc -l)
|
||||
line " 📦 $RUNNING_CR/$TOTAL_CR running │ unhealthy: $UNHEALTHY_CR"
|
||||
if [[ "${STOPPED_COUNT_CR:-0}" -gt 0 ]]; then
|
||||
issue " ⚠️ Stopped: $STOPPED_CR"
|
||||
else
|
||||
line "Docker: $RUNNING/$TOTAL containers running ✅"
|
||||
line " ✅ All containers running"
|
||||
fi
|
||||
fi
|
||||
|
||||
REPORT+=("")
|
||||
|
||||
# ── Failover ──
|
||||
line "🔀 Failover"
|
||||
|
||||
FAILOVER_PID_CR=$(get_lock_pid_cr "failover")
|
||||
if is_loop_running "failover"; then
|
||||
FAILOVER_AGE_CR=$(get_lock_age_cr "failover")
|
||||
FAILOVER_UP_CR=$(format_uptime_cr "$FAILOVER_AGE_CR")
|
||||
line " ✅ Running │ PID: $FAILOVER_PID_CR │ Uptime: $FAILOVER_UP_CR"
|
||||
else
|
||||
issue "failover NOT RUNNING"
|
||||
fi
|
||||
|
||||
FAILOVER_STATE_CR="UNKNOWN"
|
||||
FAILOVER_STATE_SECS_CR=0
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
FAILOVER_STATE_CR=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
FAILOVER_LAST_EPOCH_CR=$(grep "^last_change_epoch=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
[[ -n "$FAILOVER_LAST_EPOCH_CR" ]] && \
|
||||
FAILOVER_STATE_SECS_CR=$(( $(date +%s) - FAILOVER_LAST_EPOCH_CR ))
|
||||
fi
|
||||
|
||||
STATE_DUR_CR=$(format_uptime_cr "${FAILOVER_STATE_SECS_CR:-0}")
|
||||
|
||||
case "$FAILOVER_STATE_CR" in
|
||||
NORMAL)
|
||||
line " ✅ State: NORMAL │ Duration: $STATE_DUR_CR"
|
||||
;;
|
||||
FAILOVER)
|
||||
issue " ⚠️ State: FAILOVER — remote down for $STATE_DUR_CR"
|
||||
FAILOVER_MINS_CR=$(( FAILOVER_STATE_SECS_CR / 60 ))
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
T2=$HOST2_TIER2_DELAY; T3=$HOST2_TIER3_DELAY; T4=$HOST2_TIER4_DELAY
|
||||
else
|
||||
T2=$HOST1_TIER2_DELAY; T3=$HOST1_TIER3_DELAY; T4=$HOST1_TIER4_DELAY
|
||||
fi
|
||||
(( FAILOVER_MINS_CR >= T2 )) && line " 🔄 Tier 2: ✅ active" || \
|
||||
line " 🔄 Tier 2: ⏳ in $(( T2 - FAILOVER_MINS_CR ))min"
|
||||
(( FAILOVER_MINS_CR >= T3 )) && line " 🔄 Tier 3: ✅ active" || \
|
||||
line " 🔄 Tier 3: ⏳ in $(( T3 - FAILOVER_MINS_CR ))min"
|
||||
(( FAILOVER_MINS_CR >= T4 )) && line " 🔄 Tier 4: ✅ active" || \
|
||||
line " 🔄 Tier 4: ⏳ in $(( T4 - FAILOVER_MINS_CR ))min"
|
||||
;;
|
||||
NO_INTERNET)
|
||||
issue " ❌ State: NO_INTERNET — DDNS stopped │ Duration: $STATE_DUR_CR"
|
||||
;;
|
||||
DARK)
|
||||
issue " ❌ State: DARK — remote down AND no internet │ Duration: $STATE_DUR_CR"
|
||||
;;
|
||||
*)
|
||||
finding " ❓ State: ${FAILOVER_STATE_CR:-unknown}"
|
||||
;;
|
||||
esac
|
||||
|
||||
# Tailscale
|
||||
if command -v tailscale >/dev/null 2>&1; then
|
||||
REMOTE_IP_CR=$(tailscale ip -4 "$HOST2" 2>/dev/null)
|
||||
if [[ -n "$REMOTE_IP_CR" ]]; then
|
||||
if ping -c 1 -W 2 "$REMOTE_IP_CR" >/dev/null 2>&1; then
|
||||
line " 🌐 $HOST2: $REMOTE_IP_CR — reachable ✅"
|
||||
else
|
||||
issue " 🌐 $HOST2: $REMOTE_IP_CR — not responding"
|
||||
fi
|
||||
else
|
||||
issue " 🌐 $HOST2 not visible on Tailscale"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
+37
-36
@@ -49,20 +49,13 @@
|
||||
# ├── user_script_plug-in.sh # This file — copy into User Scripts plugin
|
||||
# ├── git_pull_execute.sh # Pulls latest scripts from Gitea repo
|
||||
# │
|
||||
# ├── Orchestrators/
|
||||
# │ ├── array_start.sh # Single entry point — launches all array-start scripts
|
||||
# │ ├── daily_sync_maintenance.sh # Daily — git pull, media sync, media mgmt, docker restart
|
||||
# │ ├── weekly_sync_maintenance.sh # Weekly — critical sync + updates, docker weekly restart
|
||||
# │ ├── media_management.sh # Permissions + cleaners + arr cleanup — run manually
|
||||
# │ ├── transcode_management.sh # Cleanup then manager every 3min + daily stats
|
||||
# │ └── README-Orchestrators.md
|
||||
# │
|
||||
# ├── Failover/
|
||||
# │ ├── failover.sh # Mutual container failover — runs continuously
|
||||
# │ ├── failover_test.sh # Controlled failover simulation — run manually
|
||||
# │ └── README-Failover.md
|
||||
# │
|
||||
# ├── Monitors/
|
||||
# │ ├── continuous_scripts_status.sh # Live status of all continuous loop scripts — run manually
|
||||
# │ ├── backup_verify.sh # Random sample checksum verification vs remote
|
||||
# │ ├── bandwidth_monitor.sh # Daily transfer logging + weekly summary
|
||||
# │ ├── cert_monitor.sh # SSL certificate expiry — direct openssl check
|
||||
@@ -72,6 +65,16 @@
|
||||
# │ ├── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report
|
||||
# │ └── README-Monitors.md
|
||||
# │
|
||||
# ├── Orchestrators/
|
||||
# │ ├── array_start.sh # Single entry point — launches all array-start scripts
|
||||
# │ ├── daily_sync_maintenance.sh # Daily — git pull, media sync, media mgmt, docker restart
|
||||
# │ ├── weekly_sync_maintenance.sh # Weekly — critical sync + updates, docker weekly restart
|
||||
# │ ├── daily_sync_maintenance.sh # Media shares sync both directions
|
||||
# │ ├── weekly_sync_maintenance.sh # Clean sync + container updates (Emby + auth stack)
|
||||
# │ ├── media_management.sh # Permissions + cleaners + arr cleanup — run manually
|
||||
# │ ├── transcode_management.sh # Cleanup then manager every 3min + daily stats
|
||||
# │ └── README-Orchestrators.md
|
||||
# │
|
||||
# ├── Rsync/
|
||||
# │ ├── rsync.sh # Core rsync script — called per share or profile
|
||||
# │ └── README-Rsync_Setup.md # Rsync-specific setup guide
|
||||
@@ -135,8 +138,22 @@
|
||||
# in real time — symlink flips work correctly for the lifetime of the container.
|
||||
#
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# ━━━ 🚀 Orchestrators — Uncomment the one below you want to run ━━━
|
||||
# ━━━ 🚀 Script Commands — Uncomment the one you want to run ━━━
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
#
|
||||
# ━━━ Failover ━━━
|
||||
# Runs continuously as background task — set schedule to "At Startup of Array (Background)".
|
||||
# Both servers must have this running for mutual failover to work.
|
||||
#
|
||||
#/mnt/user/appdata/unraid_scripts/Failover/failover.sh
|
||||
#
|
||||
# ━━━ Failover Test ━━━
|
||||
# Run manually during a maintenance window — starts and stops real containers.
|
||||
# Always --dry-run first to walk through phases without making changes.
|
||||
#
|
||||
#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run
|
||||
#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh
|
||||
#
|
||||
# ━━━ Orchestrators ━━━
|
||||
# Orchestrators are the main scripts — they handle most if not all needed scripts
|
||||
# in a specific order of operations. Each orchestrator owns a domain and runs
|
||||
@@ -166,7 +183,6 @@
|
||||
# arrs_failed_stalled_recovery.sh — 0 */6 * * * (every 6 hours)
|
||||
# blocklist + re-search failed imports and stalled downloads
|
||||
# across Sonarr, Radarr, and Lidarr automatically
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━
|
||||
#
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh
|
||||
@@ -175,32 +191,6 @@
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/sunday_morning_coffee_report.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh
|
||||
#
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# ━━━ 🚀 Script Commands — Uncomment the one you want to run ━━━
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
#
|
||||
# ━━━ Failover ━━━
|
||||
# Runs continuously as background task — set schedule to "At Startup of Array (Background)".
|
||||
# Both servers must have this running for mutual failover to work.
|
||||
#
|
||||
#/mnt/user/appdata/unraid_scripts/Failover/failover.sh
|
||||
#
|
||||
# ━━━ Failover Test ━━━
|
||||
# Run manually during a maintenance window — starts and stops real containers.
|
||||
# Always --dry-run first to walk through phases without making changes.
|
||||
#
|
||||
#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh --dry-run
|
||||
#/mnt/user/appdata/unraid_scripts/Failover/failover_test.sh
|
||||
#
|
||||
# ━━━ Monitors ━━━
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh
|
||||
#
|
||||
# ━━━ Rsync — Appdata Profiles ━━━
|
||||
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack
|
||||
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Critical-Data
|
||||
@@ -253,6 +243,17 @@
|
||||
#/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh
|
||||
# ^^ schedule: 0 5 * * * (5am daily) — always dry-run first
|
||||
#
|
||||
# ━━━ Monitors ━━━
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh
|
||||
# ^^ run manually anytime — live status of system_watchdog, docker_watchdog, failover
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/backup_verify.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/bandwidth_monitor.sh --report
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/cert_monitor.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/emby_session_report.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/smart_health.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh
|
||||
#
|
||||
# ━━━ Transcodes ━━━
|
||||
# transcode_management.sh runs cleanup then manager — schedule that, not the individuals.
|
||||
# ramdisk_setup.sh runs once at array start.
|
||||
|
||||
Reference in New Issue
Block a user