Files
Varaverk/Tools/continuous_scripts_status.sh
T
Gmer4LfeandClaude Sonnet 4.6 309546e615 feat: watchdog architecture v2 — resource manager + single-pass orchestrator
Introduce a four-layer self-healing stack replacing the continuous-loop watchdogs:

- resource_manager.sh (new): single-pass pressure reduction layer; throttles
  SABnzbd/qBit at level 1, docker-pauses background containers at level 2,
  docker-stops optional containers and signals docker_watchdog to defer at
  level 3; graduated recovery with hysteresis

- watchdog_orchestrator.sh (new, Orchestrators/): runs resource_manager →
  docker_watchdog → system_watchdog in sequence; intended for per-minute cron
  via User Scripts; startup grace, acquire_lock to prevent pile-up, heartbeat

- docker_watchdog.sh: de-looped to single-pass; daemon strikes persisted to
  state file across runs; cross-script coordination reads RM_STATE_FILE instead
  of SYS_WATCHDOG_STATE_FILE

- system_watchdog.sh: de-looped to single-pass; stripped of all container
  management (shutdown_non_essential_containers removed); reboot-only last resort

- master.conf: removed system_watchdog and docker_watchdog from
  ARRAY_START_SCRIPTS; added WATCHDOG ORCHESTRATOR and RESOURCE MANAGER sections

- master_host1.conf: added RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS arrays

- common.sh: aliased RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS via detect_hosts()

- continuous_scripts_status.sh: moved to Tools/ (preserved for future use)

- sunday_morning_coffee_report.sh: watchdog section updated to use state file
  mtime checks instead of is_running; added Resource Manager subsection;
  fixed mem_shutdown grep filter pointing to wrong state file

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-12 17:41:59 -04:00

501 lines
20 KiB
Bash
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/bash
# ==============================================================================================
# ========================= Continuous Scripts Status ==========================================
# ==============================================================================================
# Live status dashboard for all continuously running scripts in the ecosystem.
# Run manually anytime — no schedule, no cron.
#
# ── WHAT IT SHOWS ─────────────────────────────────────────────────────────────────────────────
# For each continuous script (system_watchdog, docker_watchdog, failover):
# Running state, PID, uptime, approximate cycle count
# Active strikes and skip list
# Recent restart history
# Live health snapshot
#
# system_watchdog — rootfs, RAM, ZFS ARC, load, zombie count, CPU temp
# docker_watchdog — running/stopped/unhealthy containers, required containers,
# monitored memory containers, recent restart history
# failover — current state, tier status, remote Tailscale visibility
#
# If a script is mid-cycle, state files are read as-is — reflects last completed cycle.
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays.
# Required containers and tier delays use MY_ID/REMOTE_ID correctly.
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# continuous_scripts_status.sh — show dashboard
# continuous_scripts_status.sh --log — verbose output
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
# Dashboard script — output is the point
SILENT_MODE=false
parse_args "$@"
DOCKER_TIMEOUT=15
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
detect_hosts
# ==============================================================================================
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
# ==============================================================================================
get_lock_pid() {
local script_name="$1"
local lockfile="$LOCK_DIR/${script_name}.lock"
if [[ -f "$lockfile" ]]; then
local content
content=$(cat "$lockfile" 2>/dev/null)
echo "${content%%:*}"
fi
}
get_lock_name() {
local script_name="$1"
local lockfile="$LOCK_DIR/${script_name}.lock"
if [[ -f "$lockfile" ]]; then
local content
content=$(cat "$lockfile" 2>/dev/null)
echo "${content##*:}"
fi
}
is_script_running() {
local script_name="$1"
local pid locked_name
pid=$(get_lock_pid "$script_name")
locked_name=$(get_lock_name "$script_name")
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$locked_name" == "$script_name" ]]
}
get_lock_age() {
local script_name="$1"
local lockfile="$LOCK_DIR/${script_name}.lock"
if [[ -f "$lockfile" ]]; then
local mtime now
mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0)
now=$(date +%s)
echo $(( now - mtime ))
else
echo 0
fi
}
# Human readable uptime — days/hours/mins
format_uptime() {
local seconds=$1
local days=$(( seconds / 86400 ))
local hours=$(( (seconds % 86400) / 3600 ))
local mins=$(( (seconds % 3600) / 60 ))
if (( days > 0 )); then
echo "${days}d ${hours}h ${mins}m"
elif (( hours > 0 )); then
echo "${hours}h ${mins}m"
else
echo "${mins}m"
fi
}
divider() { printf '%.0s─' {1..57}; echo; }
section() { echo ""; echo " $1"; divider; }
# ==============================================================================================
# ━━━ Header ━━━
# ==============================================================================================
clear
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " 🛡️ WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')"
echo " $ICON_HOST $MY_ID$LOCAL_SERVER_NAME"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
# ==============================================================================================
# ━━━ System Watchdog ━━━
# ==============================================================================================
section "⚙️ SYSTEM WATCHDOG"
SYS_PID=$(get_lock_pid "system_watchdog")
SYS_RUNNING=false
if is_script_running "system_watchdog"; then
SYS_RUNNING=true
SYS_AGE=$(get_lock_age "system_watchdog")
SYS_UPTIME=$(format_uptime "$SYS_AGE")
SYS_CYCLE=$(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL ))
echo " ✅ Running │ PID: $SYS_PID │ Uptime: $SYS_UPTIME │ ~Cycle: $SYS_CYCLE"
echo " ⏱️ Interval: ${SYSTEM_WATCHDOG_INTERVAL}s │ Heartbeat every: ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS}hr"
else
echo " ❌ NOT RUNNING — system_watchdog.sh is not active"
echo " Start via: bash Orchestrators/array_started.sh"
fi
echo ""
# System strikes
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
ACTIVE_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
if [[ -n "$ACTIVE_STRIKES" ]]; then
echo " ⚠️ Active strikes:"
while IFS=: read -r key count; do
[[ -z "$key" ]] && continue
echo " → $key: $count/$SYS_WATCHDOG_STRIKE_LIMIT"
done <<< "$ACTIVE_STRIKES"
else
echo " ✅ Strikes: none"
fi
else
echo " ️ Strike state file not found (watchdog may not have run yet)"
fi
# Reboot log
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
TOTAL_REBOOTS=$(grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0)
TOTAL_REBOOTS="${TOTAL_REBOOTS//[^0-9]/}"
TOTAL_REBOOTS="${TOTAL_REBOOTS:-0}"
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
WEEK_REBOOTS=$(awk -v cutoff="$WEEK_CUTOFF" '$0 >= cutoff' \
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
echo " 🔄 Watchdog reboots: $WEEK_REBOOTS this week / $TOTAL_REBOOTS total"
fi
# Container skip list
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
echo ""
echo " ⛔ Skip list ($SKIP_COUNT — manual intervention needed):"
while IFS= read -r container; do
[[ -z "$container" ]] && continue
echo " → $container"
done < "$SYS_WATCHDOG_FAILED_FILE"
else
echo " ✅ Skip list: empty"
fi
# Live system health snapshot
echo ""
echo " 📊 Current system state:"
ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
[[ "${ROOTFS_PCT:-0}" -ge "${SYS_WATCHDOG_ROOTFS_PCT:-95}" ]] && \
ROOTFS_ICON="⚠️ " || ROOTFS_ICON="✅"
echo " ${ROOTFS_ICON} rootfs: ${ROOTFS_PCT}% (threshold: ${SYS_WATCHDOG_ROOTFS_PCT}%)"
MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}")
MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
[[ $(printf "%.0f" "$MEM_FREE_GB") -lt "${SYS_WATCHDOG_MEM_GB:-4}" ]] && \
MEM_ICON="⚠️ " || MEM_ICON="✅"
echo " ${MEM_ICON} RAM: ${MEM_FREE_GB}GB free / ${MEM_TOTAL_GB}GB total (threshold: ${SYS_WATCHDOG_MEM_GB}GB free)"
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX ))
ARC_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}")
[[ "$ARC_PCT" -ge "${SYS_WATCHDOG_ARC_PINNED_PCT:-98}" ]] && \
ARC_ICON="⚠️ " || ARC_ICON="✅"
echo " ${ARC_ICON} ZFS ARC: ${ARC_GB}GB (${ARC_PCT}% of max, threshold: ${SYS_WATCHDOG_ARC_PINNED_PCT}%)"
fi
LOAD=$(awk '{print $1}' /proc/loadavg)
CORES=$(nproc)
LOAD_THRESH=$(( CORES * ${SYS_WATCHDOG_LOAD_MULTIPLIER:-3} ))
LOAD_INT=$(printf "%.0f" "$LOAD")
[[ "$LOAD_INT" -ge "$LOAD_THRESH" ]] && LOAD_ICON="⚠️ " || LOAD_ICON="✅"
echo " ${LOAD_ICON} Load avg: $LOAD (threshold: ${LOAD_THRESH} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}x ${CORES} cores)"
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0)
ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"
ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
[[ "$ZOMBIE_COUNT" -ge "${SYS_WATCHDOG_ZOMBIE_LIMIT:-50}" ]] && \
ZOMBIE_ICON="⚠️ " || ZOMBIE_ICON="✅"
echo " ${ZOMBIE_ICON} Zombies: $ZOMBIE_COUNT (threshold: ${SYS_WATCHDOG_ZOMBIE_LIMIT})"
if command -v sensors >/dev/null 2>&1; then
CPU_TEMP=$(sensors 2>/dev/null | \
grep -i "Package id 0\|Tctl\|CPU Temp" | \
awk '{print $NF}' | tr -d '+°C' | head -1)
if [[ -n "$CPU_TEMP" ]]; then
CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP")
[[ "$CPU_TEMP_INT" -ge "${SYS_WATCHDOG_CPU_TEMP_MAX:-95}" ]] && \
TEMP_ICON="⚠️ " || TEMP_ICON="✅"
echo " ${TEMP_ICON} CPU temp: ${CPU_TEMP_INT}°C (threshold: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C)"
fi
fi
# ==============================================================================================
# ━━━ Docker Watchdog ━━━
# ==============================================================================================
section "🐳 DOCKER WATCHDOG"
DOCKER_PID=$(get_lock_pid "docker_watchdog")
DOCKER_RUNNING=false
if is_script_running "docker_watchdog"; then
DOCKER_RUNNING=true
DOCKER_AGE=$(get_lock_age "docker_watchdog")
DOCKER_UPTIME=$(format_uptime "$DOCKER_AGE")
DOCKER_CYCLE=$(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL ))
echo " ✅ Running │ PID: $DOCKER_PID │ Uptime: $DOCKER_UPTIME │ ~Cycle: $DOCKER_CYCLE"
echo " ⏱️ Interval: ${DOCKER_WATCHDOG_INTERVAL}s │ Heartbeat every: ${DOCKER_WATCHDOG_HEARTBEAT_HOURS}hr"
else
echo " ❌ NOT RUNNING — docker_watchdog.sh is not active"
echo " Start via: bash Orchestrators/array_started.sh"
fi
echo ""
# Container strikes
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
ACTIVE_CONTAINER_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
if [[ -n "$ACTIVE_CONTAINER_STRIKES" ]]; then
echo " ⚠️ Active container strikes:"
while IFS=: read -r key count; do
[[ -z "$key" ]] && continue
echo " → $key: $count"
done <<< "$ACTIVE_CONTAINER_STRIKES"
else
echo " ✅ Container strikes: none"
fi
fi
# Container restart history
if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
'$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l)
if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then
echo ""
echo " 🔄 Container restarts this week: $WEEK_RESTARTS"
awk -F'|' -v cutoff="$WEEK_CUTOFF" \
'$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
sort | uniq -c | sort -rn | head -5 | \
while read -r count name; do
echo " → $name: $count restart(s)"
done
else
echo " ✅ Container restarts this week: none"
fi
fi
# Container overview
echo ""
echo " 📦 Container overview:"
if command -v docker >/dev/null 2>&1; then
RUNNING=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l)
TOTAL=$(timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l)
UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
--filter health=unhealthy -q 2>/dev/null | wc -l)
# Stopped containers — bucket into clean vs unexpected, skip SCAN_IGNORE entirely
CLEAN_STOPPED=()
UNEXPECTED_STOPPED=()
while IFS= read -r name; do
[[ -z "$name" ]] && continue
SKIP=false
for ignore in "${WATCHDOG_SCAN_IGNORE[@]}"; do
[[ "$name" == "$ignore" ]] && SKIP=true && break
done
[[ "$SKIP" == true ]] && continue
exit_code=$(docker inspect --format '{{.State.ExitCode}}' "$name" 2>/dev/null)
if [[ "$exit_code" == "0" || "$exit_code" == "143" ]]; then
CLEAN_STOPPED+=("$name")
else
UNEXPECTED_STOPPED+=("$name")
fi
done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \
--format "{{.Names}}" 2>/dev/null)
echo " Running: $RUNNING / $TOTAL total"
[[ "$UNHEALTHY" -gt 0 ]] && echo " ⚠️ Unhealthy: $UNHEALTHY"
if [[ "${#UNEXPECTED_STOPPED[@]}" -gt 0 ]]; then
echo " ⚠️ Stopped (unexpected):"
for name in "${UNEXPECTED_STOPPED[@]}"; do
echo " → $name"
done
fi
if [[ "${#CLEAN_STOPPED[@]}" -gt 0 ]]; then
echo " ⏸️ Stopped (clean):"
for name in "${CLEAN_STOPPED[@]}"; do
echo " → $name"
done
fi
if [[ "${#UNEXPECTED_STOPPED[@]}" -eq 0 && "${#CLEAN_STOPPED[@]}" -eq 0 ]]; then
echo " ✅ All containers running"
fi
# Required containers — aliased by detect_hosts() → WATCHDOG_REQUIRED_CONTAINERS
REQUIRED_ISSUES=0
if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
echo ""
echo " 🔐 Required containers:"
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
[[ -z "$container" ]] && continue
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
if [[ "$STATUS" == "true" ]]; then
echo " ✅ $container"
else
echo " ❌ $container$STATUS"
(( REQUIRED_ISSUES++ ))
fi
done
fi
# Memory-monitored containers — aliased by detect_hosts() → WATCHDOG_CONTAINERS
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
echo ""
echo " 📊 Monitored containers (memory):"
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
LIMIT_GB=$(awk "BEGIN {printf \"%.0f\", $LIMIT_MB / 1024}")
USAGE=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
--format "{{.MemUsage}}" "$container" 2>/dev/null | awk '{print $1}')
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
if [[ "$STATUS" == "true" ]]; then
echo " ✅ $container: ${USAGE:-?} (limit: ${LIMIT_GB}GB)"
else
echo " ❌ $container: not running (limit: ${LIMIT_GB}GB)"
fi
done
fi
else
echo " Docker not available"
fi
# ==============================================================================================
# ━━━ Failover ━━━
# ==============================================================================================
section "🔀 FALLBACK"
FALLBACK_PID=$(get_lock_pid "fallback")
FALLBACK_RUNNING=false
if is_script_running "fallback"; then
FALLBACK_RUNNING=true
FALLBACK_AGE=$(get_lock_age "fallback")
FALLBACK_UPTIME=$(format_uptime "$FALLBACK_AGE")
echo " ✅ Running │ PID: $FALLBACK_PID │ Uptime: $FALLBACK_UPTIME"
else
if [[ "${FALLBACK_ENABLED:-true}" == false ]]; then
echo " ⏸️ Disabled — FALLBACK_ENABLED=false in master.conf"
else
echo " ❌ NOT RUNNING — fallback.sh is not active"
echo " Start via: bash Orchestrators/array_started.sh"
fi
fi
echo ""
# Fallback state
FALLBACK_STATE="UNKNOWN"
FALLBACK_STATE_SECONDS=0
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
FALLBACK_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
FALLBACK_LAST_EPOCH=$(grep "^fallback_start=" "$FALLBACK_STATE_FILE" \
2>/dev/null | cut -d= -f2)
if [[ -n "$FALLBACK_LAST_EPOCH" && "$FALLBACK_LAST_EPOCH" -gt 0 ]]; then
FALLBACK_STATE_SECONDS=$(( $(date +%s) - FALLBACK_LAST_EPOCH ))
fi
fi
STATE_DURATION=$(format_uptime "${FALLBACK_STATE_SECONDS:-0}")
# Tier delays via REMOTE_ID — same logic as fallback.sh
REMOTE_TIER2_VAR="${REMOTE_ID}_TIER2_DELAY"
REMOTE_TIER3_VAR="${REMOTE_ID}_TIER3_DELAY"
REMOTE_TIER4_VAR="${REMOTE_ID}_TIER4_DELAY"
TIER2_DELAY="${!REMOTE_TIER2_VAR:-240}"
TIER3_DELAY="${!REMOTE_TIER3_VAR:-720}"
TIER4_DELAY="${!REMOTE_TIER4_VAR:-1440}"
case "$FALLBACK_STATE" in
NORMAL)
echo " ✅ State: NORMAL"
;;
FALLBACK)
echo " ⚠️ State: FALLBACK — $REMOTE_SERVER_NAME is down"
echo " ⏱️ Duration: $STATE_DURATION"
FALLBACK_MINS=$(( FALLBACK_STATE_SECONDS / 60 ))
echo ""
echo " 🔄 Tier status:"
echo " Tier 1 (immediate): ✅ active"
if (( FALLBACK_MINS >= TIER2_DELAY )); then
echo " Tier 2 (${TIER2_DELAY}min): ✅ active"
else
REMAINING=$(( TIER2_DELAY - FALLBACK_MINS ))
echo " Tier 2 (${TIER2_DELAY}min): ⏳ in ${REMAINING}min"
fi
if (( FALLBACK_MINS >= TIER3_DELAY )); then
echo " Tier 3 (${TIER3_DELAY}min): ✅ active"
else
REMAINING=$(( TIER3_DELAY - FALLBACK_MINS ))
echo " Tier 3 (${TIER3_DELAY}min): ⏳ in ${REMAINING}min"
fi
if (( FALLBACK_MINS >= TIER4_DELAY )); then
echo " Tier 4 (${TIER4_DELAY}min): ✅ active"
else
REMAINING=$(( TIER4_DELAY - FALLBACK_MINS ))
echo " Tier 4 (${TIER4_DELAY}min): ⏳ in ${REMAINING}min"
fi
;;
NO_INTERNET)
echo " ❌ State: NO_INTERNET — DDNS stopped"
echo " ⏱️ Down for: $STATE_DURATION"
;;
DARK)
echo " ❌ State: DARK — $REMOTE_SERVER_NAME down AND no internet"
echo " ⏱️ Duration: $STATE_DURATION"
;;
*)
echo " ❓ State: ${FALLBACK_STATE:-unknown}"
;;
esac
echo " 📡 Check interval: ${FALLBACK_CHECK_INTERVAL}s │ Handback strikes: ${FALLBACK_HANDBACK_STRIKES}"
# ==============================================================================================
# ━━━ Footer ━━━
# ==============================================================================================
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
ISSUES=0
[[ "$SYS_RUNNING" == false ]] && (( ISSUES++ ))
[[ "$DOCKER_RUNNING" == false ]] && (( ISSUES++ ))
[[ "$FALLBACK_RUNNING" == false && "${FALLBACK_ENABLED:-true}" != false ]] && (( ISSUES++ ))
[[ -n "$ACTIVE_STRIKES" ]] && (( ISSUES++ ))
[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && (( ISSUES++ ))
[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && (( ISSUES++ ))
[[ "$FALLBACK_STATE" != "NORMAL" && "$FALLBACK_STATE" != "UNKNOWN" ]] && (( ISSUES++ ))
if [[ "$ISSUES" -eq 0 ]]; then
echo " ✅ $MY_ID — all continuous scripts healthy"
else
echo " ⚠️ $ISSUES issue(s) detected — review above"
fi
echo " 🕐 Checked: $(date '+%H:%M:%S')"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""