#!/bin/bash # ============================================================================================== # ========================= Continuous Scripts Status ========================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Live status dashboard for all continuously running scripts in the ecosystem. # Run manually at any time — no schedule, no cron. # # For each script shows: running state, PID, uptime, approximate cycle count, # active strikes, skip list, recent restart history, and a live health snapshot. # # system_watchdog — rootfs, RAM, ZFS ARC, load, zombie count, CPU temp # docker_watchdog — running/stopped/unhealthy containers, required containers, # memory-monitored containers, recent restart history # failover — current state, tier status, remote Tailscale visibility # # If a script is mid-cycle, state files are read as-is — reflects last completed cycle. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Host-Aware Output # detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays. # Required containers and tier delays are shown for the correct host. # # Read-Only # Reads state files and docker inspect output only — makes no changes to any # running script, container, or state file. # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # continuous_scripts_status.sh # Show the full dashboard for all continuous scripts. # # continuous_scripts_status.sh --log # Verbose output with additional detail per script section. # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" # Dashboard script — output is the point SILENT_MODE=false parse_args "$@" DOCKER_TIMEOUT=15 # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi acquire_lock # detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays detect_hosts # ============================================================================================== # ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── # ============================================================================================== get_lock_pid() { local script_name="$1" local lockfile="$LOCK_DIR/${script_name}.lock" if [[ -f "$lockfile" ]]; then local content content=$(cat "$lockfile" 2>/dev/null) echo "${content%%:*}" fi } get_lock_name() { local script_name="$1" local lockfile="$LOCK_DIR/${script_name}.lock" if [[ -f "$lockfile" ]]; then local content content=$(cat "$lockfile" 2>/dev/null) echo "${content##*:}" fi } is_script_running() { local script_name="$1" local pid locked_name pid=$(get_lock_pid "$script_name") locked_name=$(get_lock_name "$script_name") [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$locked_name" == "$script_name" ]] } get_lock_age() { local script_name="$1" local lockfile="$LOCK_DIR/${script_name}.lock" if [[ -f "$lockfile" ]]; then local mtime now mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0) now=$(date +%s) echo $(( now - mtime )) else echo 0 fi } # Human readable uptime — days/hours/mins format_uptime() { local seconds=$1 local days=$(( seconds / 86400 )) local hours=$(( (seconds % 86400) / 3600 )) local mins=$(( (seconds % 3600) / 60 )) if (( days > 0 )); then echo "${days}d ${hours}h ${mins}m" elif (( hours > 0 )); then echo "${hours}h ${mins}m" else echo "${mins}m" fi } divider() { printf '%.0s─' {1..57}; echo; } section() { echo ""; echo " $1"; divider; } # ============================================================================================== # ━━━ Header ━━━ # ============================================================================================== clear echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " 🛡️ WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')" echo " $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # ============================================================================================== # ━━━ System Watchdog ━━━ # ============================================================================================== section "⚙️ SYSTEM WATCHDOG" SYS_PID=$(get_lock_pid "system_watchdog") SYS_RUNNING=false if is_script_running "system_watchdog"; then SYS_RUNNING=true SYS_AGE=$(get_lock_age "system_watchdog") SYS_UPTIME=$(format_uptime "$SYS_AGE") SYS_CYCLE=$(( SYS_AGE / SYSTEM_WATCHDOG_INTERVAL )) echo " ✅ Running │ PID: $SYS_PID │ Uptime: $SYS_UPTIME │ ~Cycle: $SYS_CYCLE" echo " ⏱️ Interval: ${SYSTEM_WATCHDOG_INTERVAL}s │ Heartbeat every: ${SYSTEM_WATCHDOG_HEARTBEAT_HOURS}hr" else echo " ❌ NOT RUNNING — system_watchdog.sh is not active" echo " Start via: bash Orchestrators/array_started.sh" fi echo "" # System strikes if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then ACTIVE_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") if [[ -n "$ACTIVE_STRIKES" ]]; then echo " ⚠️ Active strikes:" while IFS=: read -r key count; do [[ -z "$key" ]] && continue echo " → $key: $count/$SYS_WATCHDOG_STRIKE_LIMIT" done <<< "$ACTIVE_STRIKES" else echo " ✅ Strikes: none" fi else echo " ℹ️ Strike state file not found (watchdog may not have run yet)" fi # Reboot log if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then TOTAL_REBOOTS=$(grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0) TOTAL_REBOOTS="${TOTAL_REBOOTS//[^0-9]/}" TOTAL_REBOOTS="${TOTAL_REBOOTS:-0}" WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S') WEEK_REBOOTS=$(awk -v cutoff="$WEEK_CUTOFF" '$0 >= cutoff' \ "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l) echo " 🔄 Watchdog reboots: $WEEK_REBOOTS this week / $TOTAL_REBOOTS total" fi # Container skip list if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE") echo "" echo " ⛔ Skip list ($SKIP_COUNT — manual intervention needed):" while IFS= read -r container; do [[ -z "$container" ]] && continue echo " → $container" done < "$SYS_WATCHDOG_FAILED_FILE" else echo " ✅ Skip list: empty" fi # Live system health snapshot echo "" echo " 📊 Current system state:" ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') [[ "${ROOTFS_PCT:-0}" -ge "${SYS_WATCHDOG_ROOTFS_PCT:-95}" ]] && \ ROOTFS_ICON="⚠️ " || ROOTFS_ICON="✅" echo " ${ROOTFS_ICON} rootfs: ${ROOTFS_PCT}% (threshold: ${SYS_WATCHDOG_ROOTFS_PCT}%)" MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}") MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo) [[ $(printf "%.0f" "$MEM_FREE_GB") -lt "${SYS_WATCHDOG_MEM_GB:-4}" ]] && \ MEM_ICON="⚠️ " || MEM_ICON="✅" echo " ${MEM_ICON} RAM: ${MEM_FREE_GB}GB free / ${MEM_TOTAL_GB}GB total (threshold: ${SYS_WATCHDOG_MEM_GB}GB free)" if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX )) ARC_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}") [[ "$ARC_PCT" -ge "${SYS_WATCHDOG_ARC_PINNED_PCT:-98}" ]] && \ ARC_ICON="⚠️ " || ARC_ICON="✅" echo " ${ARC_ICON} ZFS ARC: ${ARC_GB}GB (${ARC_PCT}% of max, threshold: ${SYS_WATCHDOG_ARC_PINNED_PCT}%)" fi LOAD=$(awk '{print $1}' /proc/loadavg) CORES=$(nproc) LOAD_THRESH=$(( CORES * ${SYS_WATCHDOG_LOAD_MULTIPLIER:-3} )) LOAD_INT=$(printf "%.0f" "$LOAD") [[ "$LOAD_INT" -ge "$LOAD_THRESH" ]] && LOAD_ICON="⚠️ " || LOAD_ICON="✅" echo " ${LOAD_ICON} Load avg: $LOAD (threshold: ${LOAD_THRESH} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}x ${CORES} cores)" ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0) ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}" ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}" [[ "$ZOMBIE_COUNT" -ge "${SYS_WATCHDOG_ZOMBIE_LIMIT:-50}" ]] && \ ZOMBIE_ICON="⚠️ " || ZOMBIE_ICON="✅" echo " ${ZOMBIE_ICON} Zombies: $ZOMBIE_COUNT (threshold: ${SYS_WATCHDOG_ZOMBIE_LIMIT})" if command -v sensors >/dev/null 2>&1; then CPU_TEMP=$(sensors 2>/dev/null | \ grep -i "Package id 0\|Tctl\|CPU Temp" | \ awk '{print $NF}' | tr -d '+°C' | head -1) if [[ -n "$CPU_TEMP" ]]; then CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP") [[ "$CPU_TEMP_INT" -ge "${SYS_WATCHDOG_CPU_TEMP_MAX:-95}" ]] && \ TEMP_ICON="⚠️ " || TEMP_ICON="✅" echo " ${TEMP_ICON} CPU temp: ${CPU_TEMP_INT}°C (threshold: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C)" fi fi # ============================================================================================== # ━━━ Docker Watchdog ━━━ # ============================================================================================== section "🐳 DOCKER WATCHDOG" DOCKER_PID=$(get_lock_pid "docker_watchdog") DOCKER_RUNNING=false if is_script_running "docker_watchdog"; then DOCKER_RUNNING=true DOCKER_AGE=$(get_lock_age "docker_watchdog") DOCKER_UPTIME=$(format_uptime "$DOCKER_AGE") DOCKER_CYCLE=$(( DOCKER_AGE / DOCKER_WATCHDOG_INTERVAL )) echo " ✅ Running │ PID: $DOCKER_PID │ Uptime: $DOCKER_UPTIME │ ~Cycle: $DOCKER_CYCLE" echo " ⏱️ Interval: ${DOCKER_WATCHDOG_INTERVAL}s │ Heartbeat every: ${DOCKER_WATCHDOG_HEARTBEAT_HOURS}hr" else echo " ❌ NOT RUNNING — docker_watchdog.sh is not active" echo " Start via: bash Orchestrators/array_started.sh" fi echo "" # Container strikes if [[ -f "$WATCHDOG_STATE_FILE" ]]; then ACTIVE_CONTAINER_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") if [[ -n "$ACTIVE_CONTAINER_STRIKES" ]]; then echo " ⚠️ Active container strikes:" while IFS=: read -r key count; do [[ -z "$key" ]] && continue echo " → $key: $count" done <<< "$ACTIVE_CONTAINER_STRIKES" else echo " ✅ Container strikes: none" fi fi # Container restart history if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S') WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \ '$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l) if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then echo "" echo " 🔄 Container restarts this week: $WEEK_RESTARTS" awk -F'|' -v cutoff="$WEEK_CUTOFF" \ '$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \ sort | uniq -c | sort -rn | head -5 | \ while read -r count name; do echo " → $name: $count restart(s)" done else echo " ✅ Container restarts this week: none" fi fi # Container overview echo "" echo " 📦 Container overview:" if command -v docker >/dev/null 2>&1; then RUNNING=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l) TOTAL=$(timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l) UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \ --filter health=unhealthy -q 2>/dev/null | wc -l) # Stopped containers — bucket into clean vs unexpected, skip SCAN_IGNORE entirely CLEAN_STOPPED=() UNEXPECTED_STOPPED=() while IFS= read -r name; do [[ -z "$name" ]] && continue SKIP=false for ignore in "${WATCHDOG_SCAN_IGNORE[@]}"; do [[ "$name" == "$ignore" ]] && SKIP=true && break done [[ "$SKIP" == true ]] && continue exit_code=$(docker inspect --format '{{.State.ExitCode}}' "$name" 2>/dev/null) if [[ "$exit_code" == "0" || "$exit_code" == "143" ]]; then CLEAN_STOPPED+=("$name") else UNEXPECTED_STOPPED+=("$name") fi done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \ --format "{{.Names}}" 2>/dev/null) echo " Running: $RUNNING / $TOTAL total" [[ "$UNHEALTHY" -gt 0 ]] && echo " ⚠️ Unhealthy: $UNHEALTHY" if [[ "${#UNEXPECTED_STOPPED[@]}" -gt 0 ]]; then echo " ⚠️ Stopped (unexpected):" for name in "${UNEXPECTED_STOPPED[@]}"; do echo " → $name" done fi if [[ "${#CLEAN_STOPPED[@]}" -gt 0 ]]; then echo " ⏸️ Stopped (clean):" for name in "${CLEAN_STOPPED[@]}"; do echo " → $name" done fi if [[ "${#UNEXPECTED_STOPPED[@]}" -eq 0 && "${#CLEAN_STOPPED[@]}" -eq 0 ]]; then echo " ✅ All containers running" fi # Required containers — aliased by detect_hosts() → WATCHDOG_REQUIRED_CONTAINERS REQUIRED_ISSUES=0 if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then echo "" echo " 🔐 Required containers:" for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do [[ -z "$container" ]] && continue STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ '{{.State.Running}}' "$container" 2>/dev/null || echo "not found") if [[ "$STATUS" == "true" ]]; then echo " ✅ $container" else echo " ❌ $container — $STATUS" (( REQUIRED_ISSUES++ )) fi done fi # Memory-monitored containers — aliased by detect_hosts() → WATCHDOG_CONTAINERS if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then echo "" echo " 📊 Monitored containers (memory):" for container in "${!WATCHDOG_CONTAINERS[@]}"; do LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}" LIMIT_GB=$(awk "BEGIN {printf \"%.0f\", $LIMIT_MB / 1024}") USAGE=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \ --format "{{.MemUsage}}" "$container" 2>/dev/null | awk '{print $1}') STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ '{{.State.Running}}' "$container" 2>/dev/null || echo "not found") if [[ "$STATUS" == "true" ]]; then echo " ✅ $container: ${USAGE:-?} (limit: ${LIMIT_GB}GB)" else echo " ❌ $container: not running (limit: ${LIMIT_GB}GB)" fi done fi else echo " Docker not available" fi # ============================================================================================== # ━━━ Failover ━━━ # ============================================================================================== section "🔀 FALLBACK" FALLBACK_PID=$(get_lock_pid "fallback") FALLBACK_RUNNING=false if is_script_running "fallback"; then FALLBACK_RUNNING=true FALLBACK_AGE=$(get_lock_age "fallback") FALLBACK_UPTIME=$(format_uptime "$FALLBACK_AGE") echo " ✅ Running │ PID: $FALLBACK_PID │ Uptime: $FALLBACK_UPTIME" else if [[ "${FALLBACK_ENABLED:-true}" == false ]]; then echo " ⏸️ Disabled — FALLBACK_ENABLED=false in master.conf" else echo " ❌ NOT RUNNING — fallback.sh is not active" echo " Start via: bash Orchestrators/array_started.sh" fi fi echo "" # Fallback state FALLBACK_STATE="UNKNOWN" FALLBACK_STATE_SECONDS=0 if [[ -f "$FALLBACK_STATE_FILE" ]]; then FALLBACK_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) FALLBACK_LAST_EPOCH=$(grep "^fallback_start=" "$FALLBACK_STATE_FILE" \ 2>/dev/null | cut -d= -f2) if [[ -n "$FALLBACK_LAST_EPOCH" && "$FALLBACK_LAST_EPOCH" -gt 0 ]]; then FALLBACK_STATE_SECONDS=$(( $(date +%s) - FALLBACK_LAST_EPOCH )) fi fi STATE_DURATION=$(format_uptime "${FALLBACK_STATE_SECONDS:-0}") # Tier delays via REMOTE_ID — same logic as fallback.sh REMOTE_TIER2_VAR="${REMOTE_ID}_TIER2_DELAY" REMOTE_TIER3_VAR="${REMOTE_ID}_TIER3_DELAY" REMOTE_TIER4_VAR="${REMOTE_ID}_TIER4_DELAY" TIER2_DELAY="${!REMOTE_TIER2_VAR:-240}" TIER3_DELAY="${!REMOTE_TIER3_VAR:-720}" TIER4_DELAY="${!REMOTE_TIER4_VAR:-1440}" case "$FALLBACK_STATE" in NORMAL) echo " ✅ State: NORMAL" ;; FALLBACK) echo " ⚠️ State: FALLBACK — $REMOTE_SERVER_NAME is down" echo " ⏱️ Duration: $STATE_DURATION" FALLBACK_MINS=$(( FALLBACK_STATE_SECONDS / 60 )) echo "" echo " 🔄 Tier status:" echo " Tier 1 (immediate): ✅ active" if (( FALLBACK_MINS >= TIER2_DELAY )); then echo " Tier 2 (${TIER2_DELAY}min): ✅ active" else REMAINING=$(( TIER2_DELAY - FALLBACK_MINS )) echo " Tier 2 (${TIER2_DELAY}min): ⏳ in ${REMAINING}min" fi if (( FALLBACK_MINS >= TIER3_DELAY )); then echo " Tier 3 (${TIER3_DELAY}min): ✅ active" else REMAINING=$(( TIER3_DELAY - FALLBACK_MINS )) echo " Tier 3 (${TIER3_DELAY}min): ⏳ in ${REMAINING}min" fi if (( FALLBACK_MINS >= TIER4_DELAY )); then echo " Tier 4 (${TIER4_DELAY}min): ✅ active" else REMAINING=$(( TIER4_DELAY - FALLBACK_MINS )) echo " Tier 4 (${TIER4_DELAY}min): ⏳ in ${REMAINING}min" fi ;; NO_INTERNET) echo " ❌ State: NO_INTERNET — DDNS stopped" echo " ⏱️ Down for: $STATE_DURATION" ;; DARK) echo " ❌ State: DARK — $REMOTE_SERVER_NAME down AND no internet" echo " ⏱️ Duration: $STATE_DURATION" ;; *) echo " ❓ State: ${FALLBACK_STATE:-unknown}" ;; esac echo " 📡 Check interval: ${FALLBACK_CHECK_INTERVAL}s │ Handback strikes: ${FALLBACK_HANDBACK_STRIKES}" # ============================================================================================== # ━━━ Footer ━━━ # ============================================================================================== echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" ISSUES=0 [[ "$SYS_RUNNING" == false ]] && (( ISSUES++ )) [[ "$DOCKER_RUNNING" == false ]] && (( ISSUES++ )) [[ "$FALLBACK_RUNNING" == false && "${FALLBACK_ENABLED:-true}" != false ]] && (( ISSUES++ )) [[ -n "$ACTIVE_STRIKES" ]] && (( ISSUES++ )) [[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && (( ISSUES++ )) [[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && (( ISSUES++ )) [[ "$FALLBACK_STATE" != "NORMAL" && "$FALLBACK_STATE" != "UNKNOWN" ]] && (( ISSUES++ )) if [[ "$ISSUES" -eq 0 ]]; then echo " ✅ $MY_ID — all continuous scripts healthy" else echo " ⚠️ $ISSUES issue(s) detected — review above" fi echo " 🕐 Checked: $(date '+%H:%M:%S')" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo ""