Gmer4Lfe
cf21efaeea
Watchdog renames and moves: system_watchdog.sh → stability_watchdog.sh (last line of defense — reboots) storage_watchdog.sh → Watchdogs/System/storage_watchdog.sh webgui_restart.sh → Watchdogs/System/webgui_watchdog.sh (renamed to match folder convention) New thin orchestrator: Watchdogs/system_watchdog.sh — runs SYSTEM_WATCHDOG_SCRIPTS from master.conf Sits between docker_watchdog and stability_watchdog in the orchestrator tier chain System/ subfolder is the growth seam for future system component watchdogs master.conf: WATCHDOG_ORCHESTRATOR_SCRIPTS updated — storage removed, system_watchdog added as tier SYSTEM_WATCHDOG_SCRIPTS array added — storage + webgui server_reboot.sh: Calls array_stopping.sh before VM shutdown for guaranteed safe array stop Removed raw rc.docker stop and exit trap — orchestrator owns container shutdown
138 lines
6.1 KiB
Bash
Executable File
138 lines
6.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= System Watchdog ============================================
|
|
# ==============================================================================================
|
|
# Thin orchestrator — runs SYSTEM_WATCHDOG_SCRIPTS from master.conf sequentially.
|
|
# Called by watchdog_orchestrator.sh each cycle. Covers system component health:
|
|
# storage pool growth, runaway logs, and WebGUI availability.
|
|
#
|
|
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
|
# Driven by SYSTEM_WATCHDOG_SCRIPTS in master.conf — add, remove, or reorder there.
|
|
# Default: storage_watchdog → webgui_watchdog
|
|
#
|
|
# ── SEQUENTIAL EXECUTION ─────────────────────────────────────────────────────────────────────
|
|
# All scripts run in the foreground. Each must complete before the next starts.
|
|
# A failed script is logged but does not prevent remaining scripts from running.
|
|
#
|
|
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
|
# Root check — child scripts require root
|
|
# acquire_lock — prevents concurrent system watchdog runs
|
|
# detect_hosts() — MY_ID in notifications and logs
|
|
# Non-fatal steps — a failed step is logged; remaining steps still run
|
|
#
|
|
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
|
# SYSTEM_WATCHDOG_SCRIPTS — ordered list of system component watchdog scripts to run
|
|
#
|
|
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
|
# system_watchdog.sh — run all system component watchdogs
|
|
# system_watchdog.sh --dry-run — preview without running anything
|
|
# system_watchdog.sh --status — show configured scripts and exit
|
|
# system_watchdog.sh --log — verbose output
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
|
|
source "$ECOSYSTEM_ROOT/load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
validate_unraid_cmd \
|
|
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
|
"" "" \
|
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
|
|
|
acquire_lock
|
|
|
|
detect_hosts
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no watchdog scripts will be executed"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY SYSTEM WATCHDOG STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_GEAR Scripts: ${#SYSTEM_WATCHDOG_SCRIPTS[@]} configured"
|
|
echo ""
|
|
|
|
for entry in "${SYSTEM_WATCHDOG_SCRIPTS[@]}"; do
|
|
[[ -z "$entry" ]] && continue
|
|
read -r -a parts <<< "$entry"
|
|
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
|
|
script_name=$(basename "${parts[0]}")
|
|
if [[ ! -f "$script_path" ]]; then
|
|
echo " $ICON_ERROR $script_name — FILE NOT FOUND"
|
|
else
|
|
echo " $ICON_GEAR $script_name"
|
|
fi
|
|
done
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Run Sequence ━━━
|
|
# ==============================================================================================
|
|
START=$(date +%s)
|
|
PASSED=()
|
|
FAILED=()
|
|
STEP=0
|
|
|
|
for entry in "${SYSTEM_WATCHDOG_SCRIPTS[@]}"; do
|
|
[[ -z "$entry" ]] && continue
|
|
(( STEP++ ))
|
|
|
|
read -r -a parts <<< "$entry"
|
|
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
|
|
script_name=$(basename "${parts[0]}")
|
|
|
|
if [[ ! -f "$script_path" ]]; then
|
|
error "$script_name — not found at $script_path"
|
|
FAILED+=("$script_name")
|
|
continue
|
|
fi
|
|
|
|
if [[ ! -x "$script_path" ]]; then
|
|
chmod +x "$script_path" || {
|
|
error "$script_name — chmod +x failed"
|
|
FAILED+=("$script_name")
|
|
continue
|
|
}
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would run: $script_name"
|
|
PASSED+=("$script_name")
|
|
continue
|
|
fi
|
|
|
|
if bash "$script_path"; then
|
|
log "$script_name — done ✅"
|
|
PASSED+=("$script_name")
|
|
else
|
|
warn "$script_name — exit non-zero (issues found or fixed) — continuing"
|
|
FAILED+=("$script_name")
|
|
fi
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
log "System watchdog — $STEP script(s) — $(format_duration $(( END - START )))"
|
|
|
|
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
|
exit 0
|