massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
@@ -1,35 +1,58 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------------------- Continuous Scripts Status --------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ========================= Continuous Scripts Status ==========================================
|
||||
# ==============================================================================================
|
||||
# Live status dashboard for all continuously running scripts in the ecosystem.
|
||||
# Run manually anytime — no schedule, no cron.
|
||||
#
|
||||
# Covers all scripts started by array_start.sh that run until array stops:
|
||||
# system_watchdog.sh — system health monitor
|
||||
# docker_watchdog.sh — container health monitor
|
||||
# failover.sh — mutual failover monitor
|
||||
# ── WHAT IT SHOWS ─────────────────────────────────────────────────────────────────────────────
|
||||
# For each continuous script (system_watchdog, docker_watchdog, failover):
|
||||
# Running state, PID, uptime, approximate cycle count
|
||||
# Active strikes and skip list
|
||||
# Recent restart history
|
||||
# Live health snapshot
|
||||
#
|
||||
# Shows for each:
|
||||
# Running state, PID, uptime, current cycle
|
||||
# Active strikes, skip list, recent actions
|
||||
# Live system/container health snapshot
|
||||
# Failover state, tier status, Tailscale connectivity
|
||||
# system_watchdog — rootfs, RAM, ZFS ARC, load, zombie count, CPU temp
|
||||
# docker_watchdog — running/stopped/unhealthy containers, required containers,
|
||||
# monitored memory containers, recent restart history
|
||||
# failover — current state, tier status, remote Tailscale visibility
|
||||
#
|
||||
# If a script is mid-cycle state files are read as-is — reflects last completed cycle.
|
||||
# Run: bash Monitors/continuous_scripts_status.sh
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# If a script is mid-cycle, state files are read as-is — reflects last completed cycle.
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays.
|
||||
# Required containers, tier delays, and Tailscale checks use MY_ID/REMOTE_ID correctly.
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# continuous_scripts_status.sh — show dashboard
|
||||
# continuous_scripts_status.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Dashboard script — output is the point
|
||||
SILENT_MODE=false
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# HELPERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
|
||||
detect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
get_lock_pid() {
|
||||
local script_name="$1"
|
||||
@@ -51,11 +74,10 @@ get_lock_name() {
|
||||
fi
|
||||
}
|
||||
|
||||
is_watchdog_running() {
|
||||
is_script_running() {
|
||||
local script_name="$1"
|
||||
local pid
|
||||
local pid locked_name
|
||||
pid=$(get_lock_pid "$script_name")
|
||||
local locked_name
|
||||
locked_name=$(get_lock_name "$script_name")
|
||||
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$locked_name" == "$script_name" ]]
|
||||
}
|
||||
@@ -73,6 +95,7 @@ get_lock_age() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Human readable uptime — days/hours/mins
|
||||
format_uptime() {
|
||||
local seconds=$1
|
||||
local days=$(( seconds / 86400 ))
|
||||
@@ -87,34 +110,29 @@ format_uptime() {
|
||||
fi
|
||||
}
|
||||
|
||||
get_strikes() {
|
||||
local state_file="$1"
|
||||
local key="$2"
|
||||
grep -E "^${key}:" "$state_file" 2>/dev/null | cut -d: -f2
|
||||
}
|
||||
divider() { printf '%.0s─' {1..57}; echo; }
|
||||
section() { echo ""; echo " $1"; divider; }
|
||||
|
||||
divider() { printf '%.0s─' {1..55}; echo; }
|
||||
header() { echo ""; echo " $1"; divider; }
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ HEADER ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Header ━━━
|
||||
# ==============================================================================================
|
||||
clear
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " 🛡️ WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')"
|
||||
echo " 🖥️ $(hostname)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME"
|
||||
echo " $ICON_HOST Remote: $REMOTE_ID — $REMOTE_SERVER_NAME"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ System Watchdog ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
header "⚙️ SYSTEM WATCHDOG"
|
||||
# ==============================================================================================
|
||||
section "⚙️ SYSTEM WATCHDOG"
|
||||
|
||||
SYS_PID=$(get_lock_pid "system_watchdog")
|
||||
SYS_RUNNING=false
|
||||
|
||||
if is_watchdog_running "system_watchdog"; then
|
||||
if is_script_running "system_watchdog"; then
|
||||
SYS_RUNNING=true
|
||||
SYS_AGE=$(get_lock_age "system_watchdog")
|
||||
SYS_UPTIME=$(format_uptime "$SYS_AGE")
|
||||
@@ -128,7 +146,7 @@ fi
|
||||
|
||||
echo ""
|
||||
|
||||
# System watchdog strikes
|
||||
# System strikes
|
||||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||||
ACTIVE_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||||
if [[ -n "$ACTIVE_STRIKES" ]]; then
|
||||
@@ -149,8 +167,8 @@ if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
|
||||
TOTAL_REBOOTS=$(grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0)
|
||||
TOTAL_REBOOTS="${TOTAL_REBOOTS//[^0-9]/}"
|
||||
TOTAL_REBOOTS="${TOTAL_REBOOTS:-0}"
|
||||
WEEK_EPOCH=$(date -d "7 days ago" +%s)
|
||||
WEEK_REBOOTS=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \
|
||||
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
|
||||
WEEK_REBOOTS=$(awk -v cutoff="$WEEK_CUTOFF" '$0 >= cutoff' \
|
||||
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
|
||||
echo " 🔄 Watchdog reboots: $WEEK_REBOOTS this week / $TOTAL_REBOOTS total"
|
||||
fi
|
||||
@@ -159,7 +177,7 @@ fi
|
||||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||||
echo ""
|
||||
echo " ⛔ Skip list ($SKIP_COUNT containers — manual intervention needed):"
|
||||
echo " ⛔ Skip list ($SKIP_COUNT — manual intervention needed):"
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
echo " → $container"
|
||||
@@ -168,17 +186,15 @@ else
|
||||
echo " ✅ Skip list: empty"
|
||||
fi
|
||||
|
||||
# Current system health snapshot
|
||||
# Live system health snapshot
|
||||
echo ""
|
||||
echo " 📊 Current system state:"
|
||||
|
||||
# rootfs
|
||||
ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||||
[[ "${ROOTFS_PCT:-0}" -ge "${SYS_WATCHDOG_ROOTFS_PCT:-95}" ]] && \
|
||||
ROOTFS_ICON="⚠️ " || ROOTFS_ICON="✅"
|
||||
echo " ${ROOTFS_ICON} rootfs: ${ROOTFS_PCT}% (threshold: ${SYS_WATCHDOG_ROOTFS_PCT}%)"
|
||||
|
||||
# RAM
|
||||
MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}")
|
||||
MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
|
||||
@@ -186,7 +202,6 @@ MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
|
||||
MEM_ICON="⚠️ " || MEM_ICON="✅"
|
||||
echo " ${MEM_ICON} RAM: ${MEM_FREE_GB}GB free / ${MEM_TOTAL_GB}GB total (threshold: ${SYS_WATCHDOG_MEM_GB}GB free)"
|
||||
|
||||
# ARC
|
||||
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||||
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
@@ -197,7 +212,6 @@ if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||||
echo " ${ARC_ICON} ZFS ARC: ${ARC_GB}GB (${ARC_PCT}% of max, threshold: ${SYS_WATCHDOG_ARC_PINNED_PCT}%)"
|
||||
fi
|
||||
|
||||
# Load
|
||||
LOAD=$(awk '{print $1}' /proc/loadavg)
|
||||
CORES=$(nproc)
|
||||
LOAD_THRESH=$(( CORES * ${SYS_WATCHDOG_LOAD_MULTIPLIER:-3} ))
|
||||
@@ -205,17 +219,13 @@ LOAD_INT=$(printf "%.0f" "$LOAD")
|
||||
[[ "$LOAD_INT" -ge "$LOAD_THRESH" ]] && LOAD_ICON="⚠️ " || LOAD_ICON="✅"
|
||||
echo " ${LOAD_ICON} Load avg: $LOAD (threshold: ${LOAD_THRESH} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}x ${CORES} cores)"
|
||||
|
||||
# Zombies
|
||||
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0)
|
||||
ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"
|
||||
ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
|
||||
ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"
|
||||
ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
|
||||
[[ "$ZOMBIE_COUNT" -ge "${SYS_WATCHDOG_ZOMBIE_LIMIT:-50}" ]] && \
|
||||
ZOMBIE_ICON="⚠️ " || ZOMBIE_ICON="✅"
|
||||
echo " ${ZOMBIE_ICON} Zombies: $ZOMBIE_COUNT (threshold: ${SYS_WATCHDOG_ZOMBIE_LIMIT})"
|
||||
|
||||
# CPU temp
|
||||
if command -v sensors >/dev/null 2>&1; then
|
||||
CPU_TEMP=$(sensors 2>/dev/null | \
|
||||
grep -i "Package id 0\|Tctl\|CPU Temp" | \
|
||||
@@ -228,15 +238,15 @@ if command -v sensors >/dev/null 2>&1; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
header "🐳 DOCKER WATCHDOG"
|
||||
# ==============================================================================================
|
||||
section "🐳 DOCKER WATCHDOG"
|
||||
|
||||
DOCKER_PID=$(get_lock_pid "docker_watchdog")
|
||||
DOCKER_RUNNING=false
|
||||
|
||||
if is_watchdog_running "docker_watchdog"; then
|
||||
if is_script_running "docker_watchdog"; then
|
||||
DOCKER_RUNNING=true
|
||||
DOCKER_AGE=$(get_lock_age "docker_watchdog")
|
||||
DOCKER_UPTIME=$(format_uptime "$DOCKER_AGE")
|
||||
@@ -250,7 +260,7 @@ fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Container watchdog strikes
|
||||
# Container strikes
|
||||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
ACTIVE_CONTAINER_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||||
if [[ -n "$ACTIVE_CONTAINER_STRIKES" ]]; then
|
||||
@@ -264,16 +274,15 @@ if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# Container restart history this week
|
||||
# Container restart history
|
||||
if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
|
||||
WEEK_EPOCH=$(date -d "7 days ago" +%s 2>/dev/null || date -v-7d +%s 2>/dev/null)
|
||||
WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_EPOCH" \
|
||||
'NR>0 {if ($2 >= cutoff) count++} END {print count+0}' \
|
||||
"$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null)
|
||||
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
|
||||
WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||||
'$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l)
|
||||
if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " 🔄 Container restarts this week: $WEEK_RESTARTS"
|
||||
awk -F'|' -v cutoff="$WEEK_EPOCH" \
|
||||
awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||||
'$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
|
||||
sort | uniq -c | sort -rn | head -5 | \
|
||||
while read -r count name; do
|
||||
@@ -284,31 +293,34 @@ if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# Docker container overview
|
||||
# Container overview
|
||||
echo ""
|
||||
echo " 📦 Container overview:"
|
||||
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
RUNNING=$(docker ps -q 2>/dev/null | wc -l)
|
||||
TOTAL=$(docker ps -aq 2>/dev/null | wc -l)
|
||||
UNHEALTHY=$(docker ps --filter health=unhealthy -q 2>/dev/null | wc -l)
|
||||
RUNNING=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l)
|
||||
TOTAL=$(timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l)
|
||||
UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
|
||||
--filter health=unhealthy -q 2>/dev/null | wc -l)
|
||||
|
||||
# Filter intentionally stopped containers from the stopped list
|
||||
# Stopped containers — filter intentionally ignored ones
|
||||
STOPPED_FILTERED=()
|
||||
while IFS= read -r name; do
|
||||
[[ -z "$name" ]] && continue
|
||||
SKIP=false
|
||||
local SKIP=false
|
||||
for ignore in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do
|
||||
[[ "$name" == "$ignore" ]] && SKIP=true && break
|
||||
done
|
||||
[[ "$SKIP" == false ]] && STOPPED_FILTERED+=("$name")
|
||||
done < <(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null)
|
||||
STOPPED_COUNT="${#STOPPED_FILTERED[@]}"
|
||||
done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \
|
||||
--format "{{.Names}}" 2>/dev/null)
|
||||
|
||||
STOPPED_COUNT="${#STOPPED_FILTERED[@]}"
|
||||
echo " Running: $RUNNING / $TOTAL total"
|
||||
[[ "$UNHEALTHY" -gt 0 ]] && echo " ⚠️ Unhealthy: $UNHEALTHY"
|
||||
|
||||
if [[ "$STOPPED_COUNT" -gt 0 ]]; then
|
||||
echo " ⚠️ Stopped containers (unexpected):"
|
||||
echo " ⚠️ Stopped (unexpected):"
|
||||
for name in "${STOPPED_FILTERED[@]}"; do
|
||||
echo " → $name"
|
||||
done
|
||||
@@ -316,40 +328,35 @@ if command -v docker >/dev/null 2>&1; then
|
||||
echo " ✅ All containers running"
|
||||
fi
|
||||
|
||||
# Check required containers
|
||||
detect_hosts 2>/dev/null
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
REQUIRED=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
else
|
||||
REQUIRED=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
fi
|
||||
|
||||
# Required containers — aliased by detect_hosts() → WATCHDOG_REQUIRED_CONTAINERS
|
||||
REQUIRED_ISSUES=0
|
||||
if [[ ${#REQUIRED[@]} -gt 0 ]]; then
|
||||
if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " 🔐 Required containers:"
|
||||
for container in "${REQUIRED[@]}"; do
|
||||
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
echo " ✅ $container"
|
||||
else
|
||||
echo " ❌ $container — $STATUS"
|
||||
((REQUIRED_ISSUES++))
|
||||
(( REQUIRED_ISSUES++ ))
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Tier 1 monitored containers from WATCHDOG_CONTAINERS
|
||||
# Memory-monitored containers — aliased by detect_hosts() → WATCHDOG_CONTAINERS
|
||||
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " 📊 Monitored containers (memory):"
|
||||
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
|
||||
LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
|
||||
LIMIT_GB=$(awk "BEGIN {printf \"%.0f\", $LIMIT_MB / 1024}")
|
||||
USAGE=$(docker stats --no-stream --format "{{.MemUsage}}" "$container" \
|
||||
2>/dev/null | awk '{print $1}')
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
|
||||
USAGE=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
|
||||
--format "{{.MemUsage}}" "$container" 2>/dev/null | awk '{print $1}')
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
echo " ✅ $container: ${USAGE:-?} (limit: ${LIMIT_GB}GB)"
|
||||
else
|
||||
@@ -361,22 +368,22 @@ else
|
||||
echo " Docker not available"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Failover ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
header "🔀 FAILOVER"
|
||||
# ==============================================================================================
|
||||
section "🔀 FAILOVER"
|
||||
|
||||
FAILOVER_PID=$(get_lock_pid "failover")
|
||||
FAILOVER_RUNNING=false
|
||||
|
||||
if is_watchdog_running "failover"; then
|
||||
if is_script_running "failover"; then
|
||||
FAILOVER_RUNNING=true
|
||||
FAILOVER_AGE=$(get_lock_age "failover")
|
||||
FAILOVER_UPTIME=$(format_uptime "$FAILOVER_AGE")
|
||||
echo " ✅ Running │ PID: $FAILOVER_PID │ Uptime: $FAILOVER_UPTIME"
|
||||
else
|
||||
if [[ "${FAILOVER_ENABLED:-true}" == false ]]; then
|
||||
echo " ⏸️ Disabled — FAILOVER_ENABLED=false in Master.conf"
|
||||
echo " ⏸️ Disabled — FAILOVER_ENABLED=false in master.conf"
|
||||
else
|
||||
echo " ❌ NOT RUNNING — failover.sh is not active"
|
||||
echo " Start via: bash Orchestrators/array_start.sh"
|
||||
@@ -387,60 +394,55 @@ echo ""
|
||||
|
||||
# Failover state
|
||||
FAILOVER_STATE="UNKNOWN"
|
||||
FAILOVER_LAST_CHANGE=""
|
||||
FAILOVER_STATE_SECONDS=0
|
||||
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
FAILOVER_LAST_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
FAILOVER_LAST_EPOCH=$(grep "^last_change_epoch=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
if [[ -n "$FAILOVER_LAST_EPOCH" ]]; then
|
||||
FAILOVER_LAST_EPOCH=$(grep "^failover_start=" "$FAILOVER_STATE_FILE" \
|
||||
2>/dev/null | cut -d= -f2)
|
||||
if [[ -n "$FAILOVER_LAST_EPOCH" && "$FAILOVER_LAST_EPOCH" -gt 0 ]]; then
|
||||
FAILOVER_STATE_SECONDS=$(( $(date +%s) - FAILOVER_LAST_EPOCH ))
|
||||
fi
|
||||
fi
|
||||
|
||||
STATE_DURATION=$(format_uptime "${FAILOVER_STATE_SECONDS:-0}")
|
||||
|
||||
# Tier delays via REMOTE_ID — same logic as failover.sh
|
||||
REMOTE_TIER2_VAR="${REMOTE_ID}_TIER2_DELAY"
|
||||
REMOTE_TIER3_VAR="${REMOTE_ID}_TIER3_DELAY"
|
||||
REMOTE_TIER4_VAR="${REMOTE_ID}_TIER4_DELAY"
|
||||
TIER2_DELAY="${!REMOTE_TIER2_VAR:-240}"
|
||||
TIER3_DELAY="${!REMOTE_TIER3_VAR:-720}"
|
||||
TIER4_DELAY="${!REMOTE_TIER4_VAR:-1440}"
|
||||
|
||||
case "$FAILOVER_STATE" in
|
||||
NORMAL)
|
||||
echo " ✅ State: NORMAL"
|
||||
echo " 📅 In NORMAL state for: $STATE_DURATION"
|
||||
;;
|
||||
FAILOVER)
|
||||
echo " ⚠️ State: FAILOVER — remote server down"
|
||||
echo " ⚠️ State: FAILOVER — $REMOTE_SERVER_NAME is down"
|
||||
echo " ⏱️ Duration: $STATE_DURATION"
|
||||
# Show which tiers are active
|
||||
TIER1_DELAY=0
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
TIER2_DELAY=$HOST2_TIER2_DELAY
|
||||
TIER3_DELAY=$HOST2_TIER3_DELAY
|
||||
TIER4_DELAY=$HOST2_TIER4_DELAY
|
||||
else
|
||||
TIER2_DELAY=$HOST1_TIER2_DELAY
|
||||
TIER3_DELAY=$HOST1_TIER3_DELAY
|
||||
TIER4_DELAY=$HOST1_TIER4_DELAY
|
||||
fi
|
||||
FAILOVER_MINS=$(( FAILOVER_STATE_SECONDS / 60 ))
|
||||
echo ""
|
||||
echo " 🔄 Tier status:"
|
||||
echo " Tier 1 (immediate): ✅ active"
|
||||
echo " Tier 1 (immediate): ✅ active"
|
||||
if (( FAILOVER_MINS >= TIER2_DELAY )); then
|
||||
echo " Tier 2 (${TIER2_DELAY}min): ✅ active"
|
||||
echo " Tier 2 (${TIER2_DELAY}min): ✅ active"
|
||||
else
|
||||
REMAINING=$(( TIER2_DELAY - FAILOVER_MINS ))
|
||||
echo " Tier 2 (${TIER2_DELAY}min): ⏳ activates in ${REMAINING}min"
|
||||
echo " Tier 2 (${TIER2_DELAY}min): ⏳ in ${REMAINING}min"
|
||||
fi
|
||||
if (( FAILOVER_MINS >= TIER3_DELAY )); then
|
||||
echo " Tier 3 (${TIER3_DELAY}min): ✅ active"
|
||||
echo " Tier 3 (${TIER3_DELAY}min): ✅ active"
|
||||
else
|
||||
REMAINING=$(( TIER3_DELAY - FAILOVER_MINS ))
|
||||
echo " Tier 3 (${TIER3_DELAY}min): ⏳ activates in ${REMAINING}min"
|
||||
echo " Tier 3 (${TIER3_DELAY}min): ⏳ in ${REMAINING}min"
|
||||
fi
|
||||
if (( FAILOVER_MINS >= TIER4_DELAY )); then
|
||||
echo " Tier 4 (${TIER4_DELAY}min): ✅ active"
|
||||
echo " Tier 4 (${TIER4_DELAY}min): ✅ active"
|
||||
else
|
||||
REMAINING=$(( TIER4_DELAY - FAILOVER_MINS ))
|
||||
echo " Tier 4 (${TIER4_DELAY}min): ⏳ activates in ${REMAINING}min"
|
||||
echo " Tier 4 (${TIER4_DELAY}min): ⏳ in ${REMAINING}min"
|
||||
fi
|
||||
;;
|
||||
NO_INTERNET)
|
||||
@@ -448,7 +450,7 @@ case "$FAILOVER_STATE" in
|
||||
echo " ⏱️ Down for: $STATE_DURATION"
|
||||
;;
|
||||
DARK)
|
||||
echo " ❌ State: DARK — remote down AND no internet"
|
||||
echo " ❌ State: DARK — $REMOTE_SERVER_NAME down AND no internet"
|
||||
echo " ⏱️ Duration: $STATE_DURATION"
|
||||
;;
|
||||
*)
|
||||
@@ -456,19 +458,18 @@ case "$FAILOVER_STATE" in
|
||||
;;
|
||||
esac
|
||||
|
||||
# Tailscale remote visibility
|
||||
# Tailscale remote visibility — uses REMOTE_SERVER_NAME from detect_hosts()
|
||||
echo ""
|
||||
if command -v tailscale >/dev/null 2>&1; then
|
||||
REMOTE_IP=$(tailscale ip -4 "$HOST2" 2>/dev/null)
|
||||
REMOTE_IP=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
|
||||
if [[ -n "$REMOTE_IP" ]]; then
|
||||
# Try a quick ping to see last seen
|
||||
if ping -c 1 -W 2 "$REMOTE_IP" >/dev/null 2>&1; then
|
||||
echo " 🌐 Remote: $REMOTE_IP │ reachable ✅"
|
||||
echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_IP — reachable ✅"
|
||||
else
|
||||
echo " 🌐 Remote: $REMOTE_IP │ not responding ⚠️"
|
||||
echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_IP — not responding ⚠️"
|
||||
fi
|
||||
else
|
||||
echo " 🌐 Remote: $HOST2 not visible on Tailscale ❌"
|
||||
echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME) not visible on Tailscale ❌"
|
||||
fi
|
||||
else
|
||||
echo " 🌐 Tailscale: not available"
|
||||
@@ -476,28 +477,27 @@ fi
|
||||
|
||||
echo " 📡 Check interval: ${FAILOVER_CHECK_INTERVAL}s │ Handback strikes: ${FAILOVER_HANDBACK_STRIKES}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Footer ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Overall status
|
||||
ISSUES=0
|
||||
[[ "$SYS_RUNNING" == false ]] && ((ISSUES++))
|
||||
[[ "$DOCKER_RUNNING" == false ]] && ((ISSUES++))
|
||||
[[ "$FAILOVER_RUNNING" == false ]] && [[ "${FAILOVER_ENABLED:-true}" != false ]] && ((ISSUES++))
|
||||
[[ -n "$ACTIVE_STRIKES" ]] && ((ISSUES++))
|
||||
[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && ((ISSUES++))
|
||||
[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && ((ISSUES++))
|
||||
[[ "$FAILOVER_STATE" != "NORMAL" ]] && [[ "$FAILOVER_STATE" != "UNKNOWN" ]] && ((ISSUES++))
|
||||
[[ "$SYS_RUNNING" == false ]] && (( ISSUES++ ))
|
||||
[[ "$DOCKER_RUNNING" == false ]] && (( ISSUES++ ))
|
||||
[[ "$FAILOVER_RUNNING" == false && "${FAILOVER_ENABLED:-true}" != false ]] && (( ISSUES++ ))
|
||||
[[ -n "$ACTIVE_STRIKES" ]] && (( ISSUES++ ))
|
||||
[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && (( ISSUES++ ))
|
||||
[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && (( ISSUES++ ))
|
||||
[[ "$FAILOVER_STATE" != "NORMAL" && "$FAILOVER_STATE" != "UNKNOWN" ]] && (( ISSUES++ ))
|
||||
|
||||
if [[ "$ISSUES" -eq 0 ]]; then
|
||||
echo " ✅ All continuous scripts healthy — no issues detected"
|
||||
echo " ✅ $MY_ID — all continuous scripts healthy"
|
||||
else
|
||||
echo " ⚠️ $ISSUES issue(s) detected — review above"
|
||||
fi
|
||||
|
||||
echo " 🕐 Checked at: $(date '+%H:%M:%S')"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " 🕐 Checked: $(date '+%H:%M:%S')"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
Reference in New Issue
Block a user