Structural reorganization: watchdog taxonomy + server_reboot integration

Watchdog renames and moves:
  system_watchdog.sh → stability_watchdog.sh (last line of defense — reboots)
  storage_watchdog.sh → Watchdogs/System/storage_watchdog.sh
  webgui_restart.sh → Watchdogs/System/webgui_watchdog.sh (renamed to match folder convention)

New thin orchestrator:
  Watchdogs/system_watchdog.sh — runs SYSTEM_WATCHDOG_SCRIPTS from master.conf
  Sits between docker_watchdog and stability_watchdog in the orchestrator tier chain
  System/ subfolder is the growth seam for future system component watchdogs

master.conf:
  WATCHDOG_ORCHESTRATOR_SCRIPTS updated — storage removed, system_watchdog added as tier
  SYSTEM_WATCHDOG_SCRIPTS array added — storage + webgui

server_reboot.sh:
  Calls array_stopping.sh before VM shutdown for guaranteed safe array stop
  Removed raw rc.docker stop and exit trap — orchestrator owns container shutdown
This commit is contained in:
Gmer4Lfe
2026-05-22 20:17:22 -04:00
parent 670f25fbbd
commit cf21efaeea
18 changed files with 962 additions and 824 deletions
+22 -29
View File
@@ -19,9 +19,9 @@
# 2. Wall message to all logged-in terminal users
# 3. unRAID dashboard notification
# 4. Wait REBOOT_SLEEP seconds — users time to save work
# 5. Graceful VM shutdown via virsh — ACPI signal, then wait REBOOT_VM_WAIT
# 6. Stop libvirt (VM Manager)
# 7. Stop Docker service
# 5. array_stopping.sh — user scripts, rsync, mover, containers (verified stop)
# 6. Graceful VM shutdown via virsh — ACPI signal, then wait REBOOT_VM_WAIT
# 7. Stop libvirt (VM Manager)
# 8. sync — filesystem buffers flushed to disk
# 9. /sbin/reboot
#
@@ -121,15 +121,6 @@ acquire_lock
detect_hosts
# ── Exit Trap — restart Docker service if reboot sequence aborts after stopping it ────────────
_REBOOT_DOCKER_STOPPED=false
_trap_restart_docker_service() {
[[ "$_REBOOT_DOCKER_STOPPED" == true ]] || return
warn "Exit trap: restarting Docker service after aborted reboot sequence"
/etc/rc.d/rc.docker start >/dev/null 2>&1 || true
}
trap _trap_restart_docker_service EXIT
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made, no reboot will occur"
# ==============================================================================================
@@ -239,6 +230,25 @@ if [[ "$REBOOT_SLEEP" -gt 0 ]]; then
fi
fi
# ==============================================================================================
# ━━━ Array Stop Orchestrator ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_STOP Array Stop Orchestrator ━━━"
ARRAY_STOP_SCRIPT="$SCRIPT_DIR/../Orchestrators/array_stopping.sh"
if [[ ! -f "$ARRAY_STOP_SCRIPT" ]]; then
warn "array_stopping.sh not found — skipping orchestrated stop"
elif [[ "$DRY_RUN" == true ]]; then
bash "$ARRAY_STOP_SCRIPT" --dry-run
else
if bash "$ARRAY_STOP_SCRIPT"; then
log "Array stop complete ✅"
else
warn "array_stopping.sh reported failures — proceeding with reboot"
fi
fi
# ==============================================================================================
# ━━━ Graceful VM Shutdown ━━━
# ==============================================================================================
@@ -280,22 +290,6 @@ else
fi
fi
# ==============================================================================================
# ━━━ Stop Docker ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Stop Docker ━━━"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would stop Docker service"
else
if /etc/rc.d/rc.docker stop >/dev/null 2>&1; then
_REBOOT_DOCKER_STOPPED=true
warn "Docker stopped ✅"
else
warn "Docker stop returned non-zero — may already be stopped"
fi
fi
# ==============================================================================================
# ━━━ Sync Disks ━━━
# ==============================================================================================
@@ -327,6 +321,5 @@ if [[ "$DRY_RUN" == true ]]; then
else
warn "$ICON_REBOOT Rebooting $MY_ID now..."
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
trap - EXIT # committed to reboot — Docker should stay down
/sbin/reboot
fi