Varaverk/Watchdogs/system_watchdog.sh

#!/bin/bash
# ==============================================================================================
# ================================= System Watchdog ============================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Thin orchestrator — runs SYSTEM_WATCHDOG_SCRIPTS from master.conf sequentially.
# Called by watchdog_orchestrator.sh each cycle. Covers system component health:
# storage pool growth, runaway logs, WebGUI availability, and network connectivity.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Driven by SYSTEM_WATCHDOG_SCRIPTS in master.conf — add, remove, or reorder there.
# Default: storage_watchdog → webgui_watchdog → network_watchdog
#
# All scripts run in the foreground. Each must complete before the next starts.
# A failed script is logged but does not prevent remaining scripts from running.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Configuration Owns the List
#   SYSTEM_WATCHDOG_SCRIPTS in master.conf is the only place scripts are added
#   or removed. This orchestrator never needs to be edited to change what runs.
#
# Non-Fatal Steps
#   A failed watchdog step is logged and noted in the summary, but the remaining
#   steps still execute. Partial coverage is better than a halted watchdog chain.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Enforcement
#   Every child script requires root. Failing here gives one clear error instead
#   of the same permission failure repeated once per child.
#
# Lock Acquisition
#   acquire_lock() prevents concurrent system watchdog runs. This is called every
#   cycle by watchdog_orchestrator.sh — a slow child must not cause two chains to
#   overlap and run the same watchdog twice.
#
# Host Detection
#   detect_hosts() sets MY_ID for notifications and logs.
#
# Empty List Guard
#   Warns and exits if SYSTEM_WATCHDOG_SCRIPTS is unconfigured. An empty list
#   would otherwise report "0/0 passed" every cycle — indistinguishable from
#   healthy, while no system monitoring is actually running.
#
# Missing Script Tolerance
#   run_orch_child() records a missing or failing child as a failed step and
#   continues. One broken watchdog never suppresses the rest of the chain.
#
# Non-Fatal Steps
#   A failed step is logged and surfaces in the summary and notification, but
#   remaining steps still execute. Partial coverage beats a halted chain.
#
# Dry Run Propagation
#   --dry-run and --log are passed through to every child script.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   SYSTEM_WATCHDOG_SCRIPTS — ordered list of system component watchdog scripts to run
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# system_watchdog.sh
#     Run all system component watchdogs.
#
# system_watchdog.sh --dry-run
#     Passes --dry-run to each sub-script — no changes made.
#
# system_watchdog.sh --status
#     Show configured scripts and exit.
#
# system_watchdog.sh --log
#     Passes --log to each sub-script for verbose output.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

source "$ECOSYSTEM_ROOT/load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi


acquire_lock

detect_hosts

# An empty list reports "0/0 passed" every cycle — reads as healthy while nothing is monitored.
if [[ ${#SYSTEM_WATCHDOG_SCRIPTS[@]} -eq 0 ]]; then
    warn "SYSTEM_WATCHDOG_SCRIPTS is empty — no system watchdogs will run"
    warn "Check SYSTEM_WATCHDOG_SCRIPTS in master.conf"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY SYSTEM WATCHDOG STATUS ━━━━━"
    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR  Scripts:  ${#SYSTEM_WATCHDOG_SCRIPTS[@]} configured"
    echo ""

    for entry in "${SYSTEM_WATCHDOG_SCRIPTS[@]}"; do
        [[ -z "$entry" ]] && continue
        read -r -a parts <<< "$entry"
        script_path="$ECOSYSTEM_ROOT/${parts[0]}"
        script_name=$(basename "${parts[0]}")
        if [[ ! -f "$script_path" ]]; then
            echo "  $ICON_ERROR $script_name — FILE NOT FOUND"
        else
            echo "  $ICON_GEAR $script_name"
        fi
    done
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Run Sequence ━━━
# ==============================================================================================
echo "━━━ $ICON_SHIELD System Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

START=$(date +%s)
JOB_PASS=()
JOB_FAIL=()

for entry in "${SYSTEM_WATCHDOG_SCRIPTS[@]}"; do
    [[ -z "$entry" ]] && continue
    run_orch_child "$entry"
done

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
log "System watchdog — ${#JOB_PASS[@]}/${#SYSTEM_WATCHDOG_SCRIPTS[@]} passed — $(format_duration $(( END - START )))"

if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
    notify "System watchdog failed on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
        "System Watchdog" "warning"
    exit 1
fi

exit 0