Gmer4LfeandClaude Sonnet 4.6
070f9caf7f
Skip list belongs to docker_watchdog, not stability_watchdog (legacy naming from when docker+system were the only two watchdogs). Rename variable and physical file system_watchdog_failed.db → docker_watchdog_failed.db across all files. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
517 lines
20 KiB
Bash
517 lines
20 KiB
Bash
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ========================= Continuous Scripts Status ==========================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Live status dashboard for all continuously running scripts in the ecosystem.
|
||
# Run manually at any time — no schedule, no cron.
|
||
#
|
||
# For each script shows: running state, PID, uptime, approximate cycle count,
|
||
# active strikes, skip list, recent restart history, and a live health snapshot.
|
||
#
|
||
# system_watchdog — rootfs, RAM, ZFS ARC, load, zombie count, CPU temp
|
||
# docker_watchdog — running/stopped/unhealthy containers, required containers,
|
||
# memory-monitored containers, recent restart history
|
||
# failover — current state, tier status, remote Tailscale visibility
|
||
#
|
||
# If a script is mid-cycle, state files are read as-is — reflects last completed cycle.
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Host-Aware Output
|
||
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays.
|
||
# Required containers and tier delays are shown for the correct host.
|
||
#
|
||
# Read-Only
|
||
# Reads state files and docker inspect output only — makes no changes to any
|
||
# running script, container, or state file.
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# continuous_scripts_status.sh
|
||
# Show the full dashboard for all continuous scripts.
|
||
#
|
||
# continuous_scripts_status.sh --log
|
||
# Verbose output with additional detail per script section.
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../load_config.sh"
|
||
|
||
# Dashboard script — output is the point
|
||
SILENT_MODE=false
|
||
|
||
parse_args "$@"
|
||
|
||
DOCKER_TIMEOUT=15
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
acquire_lock
|
||
|
||
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
|
||
detect_hosts
|
||
|
||
# ==============================================================================================
|
||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
get_lock_pid() {
|
||
local script_name="$1"
|
||
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||
if [[ -f "$lockfile" ]]; then
|
||
local content
|
||
content=$(cat "$lockfile" 2>/dev/null)
|
||
echo "${content%%:*}"
|
||
fi
|
||
}
|
||
|
||
get_lock_name() {
|
||
local script_name="$1"
|
||
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||
if [[ -f "$lockfile" ]]; then
|
||
local content
|
||
content=$(cat "$lockfile" 2>/dev/null)
|
||
echo "${content##*:}"
|
||
fi
|
||
}
|
||
|
||
is_script_running() {
|
||
local script_name="$1"
|
||
local pid locked_name
|
||
pid=$(get_lock_pid "$script_name")
|
||
locked_name=$(get_lock_name "$script_name")
|
||
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$locked_name" == "$script_name" ]]
|
||
}
|
||
|
||
get_lock_age() {
|
||
local script_name="$1"
|
||
local lockfile="$LOCK_DIR/${script_name}.lock"
|
||
if [[ -f "$lockfile" ]]; then
|
||
local mtime now
|
||
mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0)
|
||
now=$(date +%s)
|
||
echo $(( now - mtime ))
|
||
else
|
||
echo 0
|
||
fi
|
||
}
|
||
|
||
# Human readable uptime — days/hours/mins
|
||
format_uptime() {
|
||
local seconds=$1
|
||
local days=$(( seconds / 86400 ))
|
||
local hours=$(( (seconds % 86400) / 3600 ))
|
||
local mins=$(( (seconds % 3600) / 60 ))
|
||
if (( days > 0 )); then
|
||
echo "${days}d ${hours}h ${mins}m"
|
||
elif (( hours > 0 )); then
|
||
echo "${hours}h ${mins}m"
|
||
else
|
||
echo "${mins}m"
|
||
fi
|
||
}
|
||
|
||
divider() { printf '%.0s─' {1..57}; echo; }
|
||
section() { echo ""; echo " $1"; divider; }
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Header ━━━
|
||
# ==============================================================================================
|
||
clear
|
||
echo ""
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " 🛡️ WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')"
|
||
echo " $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ System Watchdog ━━━
|
||
# ==============================================================================================
|
||
section "⚙️ SYSTEM WATCHDOG"
|
||
|
||
SYS_PID=$(get_lock_pid "system_watchdog")
|
||
SYS_RUNNING=false
|
||
|
||
if is_script_running "system_watchdog"; then
|
||
SYS_RUNNING=true
|
||
SYS_AGE=$(get_lock_age "system_watchdog")
|
||
SYS_UPTIME=$(format_uptime "$SYS_AGE")
|
||
SYS_CYCLE=$(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL ))
|
||
echo " ✅ Running │ PID: $SYS_PID │ Uptime: $SYS_UPTIME │ ~Cycle: $SYS_CYCLE"
|
||
echo " ⏱️ Interval: ${SYSTEM_WATCHDOG_INTERVAL}s │ Heartbeat every: ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS}hr"
|
||
else
|
||
echo " ❌ NOT RUNNING — system_watchdog.sh is not active"
|
||
echo " Start via: bash Orchestrators/array_started.sh"
|
||
fi
|
||
|
||
echo ""
|
||
|
||
# System strikes
|
||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||
ACTIVE_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||
if [[ -n "$ACTIVE_STRIKES" ]]; then
|
||
echo " ⚠️ Active strikes:"
|
||
while IFS=: read -r key count; do
|
||
[[ -z "$key" ]] && continue
|
||
echo " → $key: $count/$SYS_WATCHDOG_STRIKE_LIMIT"
|
||
done <<< "$ACTIVE_STRIKES"
|
||
else
|
||
echo " ✅ Strikes: none"
|
||
fi
|
||
else
|
||
echo " ℹ️ Strike state file not found (watchdog may not have run yet)"
|
||
fi
|
||
|
||
# Reboot log
|
||
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
|
||
TOTAL_REBOOTS=$(grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0)
|
||
TOTAL_REBOOTS="${TOTAL_REBOOTS//[^0-9]/}"
|
||
TOTAL_REBOOTS="${TOTAL_REBOOTS:-0}"
|
||
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
|
||
WEEK_REBOOTS=$(awk -v cutoff="$WEEK_CUTOFF" '$0 >= cutoff' \
|
||
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
|
||
echo " 🔄 Watchdog reboots: $WEEK_REBOOTS this week / $TOTAL_REBOOTS total"
|
||
fi
|
||
|
||
# Container skip list
|
||
if [[ -f "$DOCKER_WATCHDOG_FAILED_FILE" ]] && [[ -s "$DOCKER_WATCHDOG_FAILED_FILE" ]]; then
|
||
SKIP_COUNT=$(wc -l < "$DOCKER_WATCHDOG_FAILED_FILE")
|
||
echo ""
|
||
echo " ⛔ Skip list ($SKIP_COUNT — manual intervention needed):"
|
||
while IFS= read -r container; do
|
||
[[ -z "$container" ]] && continue
|
||
echo " → $container"
|
||
done < "$DOCKER_WATCHDOG_FAILED_FILE"
|
||
else
|
||
echo " ✅ Skip list: empty"
|
||
fi
|
||
|
||
# Live system health snapshot
|
||
echo ""
|
||
echo " 📊 Current system state:"
|
||
|
||
ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||
[[ "${ROOTFS_PCT:-0}" -ge "${SYS_WATCHDOG_ROOTFS_PCT:-95}" ]] && \
|
||
ROOTFS_ICON="⚠️ " || ROOTFS_ICON="✅"
|
||
echo " ${ROOTFS_ICON} rootfs: ${ROOTFS_PCT}% (threshold: ${SYS_WATCHDOG_ROOTFS_PCT}%)"
|
||
|
||
MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||
MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}")
|
||
MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
|
||
[[ $(printf "%.0f" "$MEM_FREE_GB") -lt "${SYS_WATCHDOG_MEM_GB:-4}" ]] && \
|
||
MEM_ICON="⚠️ " || MEM_ICON="✅"
|
||
echo " ${MEM_ICON} RAM: ${MEM_FREE_GB}GB free / ${MEM_TOTAL_GB}GB total (threshold: ${SYS_WATCHDOG_MEM_GB}GB free)"
|
||
|
||
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||
ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX ))
|
||
ARC_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}")
|
||
[[ "$ARC_PCT" -ge "${SYS_WATCHDOG_ARC_PINNED_PCT:-98}" ]] && \
|
||
ARC_ICON="⚠️ " || ARC_ICON="✅"
|
||
echo " ${ARC_ICON} ZFS ARC: ${ARC_GB}GB (${ARC_PCT}% of max, threshold: ${SYS_WATCHDOG_ARC_PINNED_PCT}%)"
|
||
fi
|
||
|
||
LOAD=$(awk '{print $1}' /proc/loadavg)
|
||
CORES=$(nproc)
|
||
LOAD_THRESH=$(( CORES * ${SYS_WATCHDOG_LOAD_MULTIPLIER:-3} ))
|
||
LOAD_INT=$(printf "%.0f" "$LOAD")
|
||
[[ "$LOAD_INT" -ge "$LOAD_THRESH" ]] && LOAD_ICON="⚠️ " || LOAD_ICON="✅"
|
||
echo " ${LOAD_ICON} Load avg: $LOAD (threshold: ${LOAD_THRESH} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}x ${CORES} cores)"
|
||
|
||
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0)
|
||
ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"
|
||
ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
|
||
[[ "$ZOMBIE_COUNT" -ge "${SYS_WATCHDOG_ZOMBIE_LIMIT:-50}" ]] && \
|
||
ZOMBIE_ICON="⚠️ " || ZOMBIE_ICON="✅"
|
||
echo " ${ZOMBIE_ICON} Zombies: $ZOMBIE_COUNT (threshold: ${SYS_WATCHDOG_ZOMBIE_LIMIT})"
|
||
|
||
if command -v sensors >/dev/null 2>&1; then
|
||
CPU_TEMP=$(sensors 2>/dev/null | \
|
||
grep -i "Package id 0\|Tctl\|CPU Temp" | \
|
||
awk '{print $NF}' | tr -d '+°C' | head -1)
|
||
if [[ -n "$CPU_TEMP" ]]; then
|
||
CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP")
|
||
[[ "$CPU_TEMP_INT" -ge "${SYS_WATCHDOG_CPU_TEMP_MAX:-95}" ]] && \
|
||
TEMP_ICON="⚠️ " || TEMP_ICON="✅"
|
||
echo " ${TEMP_ICON} CPU temp: ${CPU_TEMP_INT}°C (threshold: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C)"
|
||
fi
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Docker Watchdog ━━━
|
||
# ==============================================================================================
|
||
section "🐳 DOCKER WATCHDOG"
|
||
|
||
DOCKER_PID=$(get_lock_pid "docker_watchdog")
|
||
DOCKER_RUNNING=false
|
||
|
||
if is_script_running "docker_watchdog"; then
|
||
DOCKER_RUNNING=true
|
||
DOCKER_AGE=$(get_lock_age "docker_watchdog")
|
||
DOCKER_UPTIME=$(format_uptime "$DOCKER_AGE")
|
||
DOCKER_CYCLE=$(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL ))
|
||
echo " ✅ Running │ PID: $DOCKER_PID │ Uptime: $DOCKER_UPTIME │ ~Cycle: $DOCKER_CYCLE"
|
||
echo " ⏱️ Interval: ${DOCKER_WATCHDOG_INTERVAL}s │ Heartbeat every: ${DOCKER_WATCHDOG_HEARTBEAT_HOURS}hr"
|
||
else
|
||
echo " ❌ NOT RUNNING — docker_watchdog.sh is not active"
|
||
echo " Start via: bash Orchestrators/array_started.sh"
|
||
fi
|
||
|
||
echo ""
|
||
|
||
# Container strikes
|
||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||
ACTIVE_CONTAINER_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||
if [[ -n "$ACTIVE_CONTAINER_STRIKES" ]]; then
|
||
echo " ⚠️ Active container strikes:"
|
||
while IFS=: read -r key count; do
|
||
[[ -z "$key" ]] && continue
|
||
echo " → $key: $count"
|
||
done <<< "$ACTIVE_CONTAINER_STRIKES"
|
||
else
|
||
echo " ✅ Container strikes: none"
|
||
fi
|
||
fi
|
||
|
||
# Container restart history
|
||
if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
|
||
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
|
||
WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||
'$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l)
|
||
if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then
|
||
echo ""
|
||
echo " 🔄 Container restarts this week: $WEEK_RESTARTS"
|
||
awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||
'$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
|
||
sort | uniq -c | sort -rn | head -5 | \
|
||
while read -r count name; do
|
||
echo " → $name: $count restart(s)"
|
||
done
|
||
else
|
||
echo " ✅ Container restarts this week: none"
|
||
fi
|
||
fi
|
||
|
||
# Container overview
|
||
echo ""
|
||
echo " 📦 Container overview:"
|
||
|
||
if command -v docker >/dev/null 2>&1; then
|
||
RUNNING=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l)
|
||
TOTAL=$(timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l)
|
||
UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
|
||
--filter health=unhealthy -q 2>/dev/null | wc -l)
|
||
|
||
# Stopped containers — bucket into clean vs unexpected, skip SCAN_IGNORE entirely
|
||
CLEAN_STOPPED=()
|
||
UNEXPECTED_STOPPED=()
|
||
while IFS= read -r name; do
|
||
[[ -z "$name" ]] && continue
|
||
SKIP=false
|
||
for ignore in "${WATCHDOG_SCAN_IGNORE[@]}"; do
|
||
[[ "$name" == "$ignore" ]] && SKIP=true && break
|
||
done
|
||
[[ "$SKIP" == true ]] && continue
|
||
exit_code=$(docker inspect --format '{{.State.ExitCode}}' "$name" 2>/dev/null)
|
||
if [[ "$exit_code" == "0" || "$exit_code" == "143" ]]; then
|
||
CLEAN_STOPPED+=("$name")
|
||
else
|
||
UNEXPECTED_STOPPED+=("$name")
|
||
fi
|
||
done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \
|
||
--format "{{.Names}}" 2>/dev/null)
|
||
|
||
echo " Running: $RUNNING / $TOTAL total"
|
||
[[ "$UNHEALTHY" -gt 0 ]] && echo " ⚠️ Unhealthy: $UNHEALTHY"
|
||
|
||
if [[ "${#UNEXPECTED_STOPPED[@]}" -gt 0 ]]; then
|
||
echo " ⚠️ Stopped (unexpected):"
|
||
for name in "${UNEXPECTED_STOPPED[@]}"; do
|
||
echo " → $name"
|
||
done
|
||
fi
|
||
|
||
if [[ "${#CLEAN_STOPPED[@]}" -gt 0 ]]; then
|
||
echo " ⏸️ Stopped (clean):"
|
||
for name in "${CLEAN_STOPPED[@]}"; do
|
||
echo " → $name"
|
||
done
|
||
fi
|
||
|
||
if [[ "${#UNEXPECTED_STOPPED[@]}" -eq 0 && "${#CLEAN_STOPPED[@]}" -eq 0 ]]; then
|
||
echo " ✅ All containers running"
|
||
fi
|
||
|
||
# Required containers — aliased by detect_hosts() → WATCHDOG_REQUIRED_CONTAINERS
|
||
REQUIRED_ISSUES=0
|
||
if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
|
||
echo ""
|
||
echo " 🔐 Required containers:"
|
||
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||
'{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
|
||
if [[ "$STATUS" == "true" ]]; then
|
||
echo " ✅ $container"
|
||
else
|
||
echo " ❌ $container — $STATUS"
|
||
(( REQUIRED_ISSUES++ ))
|
||
fi
|
||
done
|
||
fi
|
||
|
||
# Memory-monitored containers — aliased by detect_hosts() → WATCHDOG_CONTAINERS
|
||
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
|
||
echo ""
|
||
echo " 📊 Monitored containers (memory):"
|
||
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
|
||
LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
|
||
LIMIT_GB=$(awk "BEGIN {printf \"%.0f\", $LIMIT_MB / 1024}")
|
||
USAGE=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
|
||
--format "{{.MemUsage}}" "$container" 2>/dev/null | awk '{print $1}')
|
||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||
'{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
|
||
if [[ "$STATUS" == "true" ]]; then
|
||
echo " ✅ $container: ${USAGE:-?} (limit: ${LIMIT_GB}GB)"
|
||
else
|
||
echo " ❌ $container: not running (limit: ${LIMIT_GB}GB)"
|
||
fi
|
||
done
|
||
fi
|
||
else
|
||
echo " Docker not available"
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Failover ━━━
|
||
# ==============================================================================================
|
||
section "🔀 FALLBACK"
|
||
|
||
FALLBACK_PID=$(get_lock_pid "fallback")
|
||
FALLBACK_RUNNING=false
|
||
|
||
if is_script_running "fallback"; then
|
||
FALLBACK_RUNNING=true
|
||
FALLBACK_AGE=$(get_lock_age "fallback")
|
||
FALLBACK_UPTIME=$(format_uptime "$FALLBACK_AGE")
|
||
echo " ✅ Running │ PID: $FALLBACK_PID │ Uptime: $FALLBACK_UPTIME"
|
||
else
|
||
if [[ "${FALLBACK_ENABLED:-true}" == false ]]; then
|
||
echo " ⏸️ Disabled — FALLBACK_ENABLED=false in master.conf"
|
||
else
|
||
echo " ❌ NOT RUNNING — fallback.sh is not active"
|
||
echo " Start via: bash Orchestrators/array_started.sh"
|
||
fi
|
||
fi
|
||
|
||
echo ""
|
||
|
||
# Fallback state
|
||
FALLBACK_STATE="UNKNOWN"
|
||
FALLBACK_STATE_SECONDS=0
|
||
|
||
if [[ -f "$FALLBACK_STATE_FILE" ]]; then
|
||
FALLBACK_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||
FALLBACK_LAST_EPOCH=$(grep "^fallback_start=" "$FALLBACK_STATE_FILE" \
|
||
2>/dev/null | cut -d= -f2)
|
||
if [[ -n "$FALLBACK_LAST_EPOCH" && "$FALLBACK_LAST_EPOCH" -gt 0 ]]; then
|
||
FALLBACK_STATE_SECONDS=$(( $(date +%s) - FALLBACK_LAST_EPOCH ))
|
||
fi
|
||
fi
|
||
|
||
STATE_DURATION=$(format_uptime "${FALLBACK_STATE_SECONDS:-0}")
|
||
|
||
# Tier delays via REMOTE_ID — same logic as fallback.sh
|
||
REMOTE_TIER2_VAR="${REMOTE_ID}_TIER2_DELAY"
|
||
REMOTE_TIER3_VAR="${REMOTE_ID}_TIER3_DELAY"
|
||
REMOTE_TIER4_VAR="${REMOTE_ID}_TIER4_DELAY"
|
||
TIER2_DELAY="${!REMOTE_TIER2_VAR:-240}"
|
||
TIER3_DELAY="${!REMOTE_TIER3_VAR:-720}"
|
||
TIER4_DELAY="${!REMOTE_TIER4_VAR:-1440}"
|
||
|
||
case "$FALLBACK_STATE" in
|
||
NORMAL)
|
||
echo " ✅ State: NORMAL"
|
||
;;
|
||
FALLBACK)
|
||
echo " ⚠️ State: FALLBACK — $REMOTE_SERVER_NAME is down"
|
||
echo " ⏱️ Duration: $STATE_DURATION"
|
||
FALLBACK_MINS=$(( FALLBACK_STATE_SECONDS / 60 ))
|
||
echo ""
|
||
echo " 🔄 Tier status:"
|
||
echo " Tier 1 (immediate): ✅ active"
|
||
if (( FALLBACK_MINS >= TIER2_DELAY )); then
|
||
echo " Tier 2 (${TIER2_DELAY}min): ✅ active"
|
||
else
|
||
REMAINING=$(( TIER2_DELAY - FALLBACK_MINS ))
|
||
echo " Tier 2 (${TIER2_DELAY}min): ⏳ in ${REMAINING}min"
|
||
fi
|
||
if (( FALLBACK_MINS >= TIER3_DELAY )); then
|
||
echo " Tier 3 (${TIER3_DELAY}min): ✅ active"
|
||
else
|
||
REMAINING=$(( TIER3_DELAY - FALLBACK_MINS ))
|
||
echo " Tier 3 (${TIER3_DELAY}min): ⏳ in ${REMAINING}min"
|
||
fi
|
||
if (( FALLBACK_MINS >= TIER4_DELAY )); then
|
||
echo " Tier 4 (${TIER4_DELAY}min): ✅ active"
|
||
else
|
||
REMAINING=$(( TIER4_DELAY - FALLBACK_MINS ))
|
||
echo " Tier 4 (${TIER4_DELAY}min): ⏳ in ${REMAINING}min"
|
||
fi
|
||
;;
|
||
NO_INTERNET)
|
||
echo " ❌ State: NO_INTERNET — DDNS stopped"
|
||
echo " ⏱️ Down for: $STATE_DURATION"
|
||
;;
|
||
DARK)
|
||
echo " ❌ State: DARK — $REMOTE_SERVER_NAME down AND no internet"
|
||
echo " ⏱️ Duration: $STATE_DURATION"
|
||
;;
|
||
*)
|
||
echo " ❓ State: ${FALLBACK_STATE:-unknown}"
|
||
;;
|
||
esac
|
||
|
||
|
||
echo " 📡 Check interval: ${FALLBACK_CHECK_INTERVAL}s │ Handback strikes: ${FALLBACK_HANDBACK_STRIKES}"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Footer ━━━
|
||
# ==============================================================================================
|
||
echo ""
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
ISSUES=0
|
||
[[ "$SYS_RUNNING" == false ]] && (( ISSUES++ ))
|
||
[[ "$DOCKER_RUNNING" == false ]] && (( ISSUES++ ))
|
||
[[ "$FALLBACK_RUNNING" == false && "${FALLBACK_ENABLED:-true}" != false ]] && (( ISSUES++ ))
|
||
[[ -n "$ACTIVE_STRIKES" ]] && (( ISSUES++ ))
|
||
[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && (( ISSUES++ ))
|
||
[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && (( ISSUES++ ))
|
||
[[ "$FALLBACK_STATE" != "NORMAL" && "$FALLBACK_STATE" != "UNKNOWN" ]] && (( ISSUES++ ))
|
||
|
||
if [[ "$ISSUES" -eq 0 ]]; then
|
||
echo " ✅ $MY_ID — all continuous scripts healthy"
|
||
else
|
||
echo " ⚠️ $ISSUES issue(s) detected — review above"
|
||
fi
|
||
|
||
echo " 🕐 Checked: $(date '+%H:%M:%S')"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo "" |