Gmer4Lfe
3770bf80e7
Exit traps: resource_watchdog registers/clears trap for stopped containers; server_reboot registers trap to restart Docker service on abort; system_watchdog records containers before bulk stop and registers trap to restart if reboot aborts. echo/log audit: status conclusions, nothing-to-do, and clean-cycle confirmations → echo across clear_logs, docker_syslog_filter, mover_stop, php_fpm_max_children, rsync_stop, user_scripts_stop. resource_watchdog: disabled flag, restore-level messages, pressure status → echo. system_watchdog: clean-cycle summary → echo. Manual-Unraid_Essentials.md: added Output Tiers section.
332 lines
13 KiB
Bash
332 lines
13 KiB
Bash
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= Server Reboot ==============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Gracefully reboots the unRAID server with pre-flight checks, user warnings,
|
|
# clean service shutdown, and disk sync. Use instead of raw /sbin/reboot —
|
|
# gives users warning time and ensures services stop cleanly before the kernel
|
|
# drops.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Shutdown Sequence
|
|
# 1. Pre-flight warnings — rsync, mover, active Emby sessions (warn, not block)
|
|
# 2. Wall message to all logged-in terminal users
|
|
# 3. unRAID dashboard notification
|
|
# 4. Wait REBOOT_SLEEP seconds — users time to save work
|
|
# 5. Graceful VM shutdown via virsh — ACPI signal, then wait REBOOT_VM_WAIT
|
|
# 6. Stop libvirt (VM Manager)
|
|
# 7. Stop Docker service
|
|
# 8. sync — filesystem buffers flushed to disk
|
|
# 9. /sbin/reboot
|
|
#
|
|
# Pre-flight Warnings (informational — do not block)
|
|
# rsync running → partial files possible if mid-transfer
|
|
# mover running → files may be left mid-move on cache or array
|
|
# Emby sessions → active streams/transcodes interrupted
|
|
# Warnings do not block the reboot — you called this script, you know.
|
|
#
|
|
# VM Graceful Shutdown
|
|
# virsh shutdown sends the ACPI power button signal — same as pressing the
|
|
# physical power button. VM gets a chance to flush buffers and shut down.
|
|
# After REBOOT_VM_WAIT seconds, libvirt stops anyway — reboot takes priority.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root Required
|
|
# /sbin/reboot requires root.
|
|
#
|
|
# Single Instance Lock
|
|
# acquire_lock prevents concurrent reboot calls.
|
|
#
|
|
# Host Identity in All Messages
|
|
# detect_hosts() sets MY_ID — wall and notifications show which server is
|
|
# rebooting. Critical on a two-server setup.
|
|
#
|
|
# sync Before Reboot
|
|
# filesystem buffers flushed to disk before reboot command.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# REBOOT_SLEEP
|
|
# Seconds between warning and shutdown sequence start. (default: 30)
|
|
#
|
|
# REBOOT_VM_WAIT
|
|
# Seconds to wait for VMs to shut down gracefully. (default: 30)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# server_reboot.sh
|
|
# Run pre-flight, warn users, stop services cleanly, then reboot.
|
|
#
|
|
# server_reboot.sh --dry-run
|
|
# Walk through the entire shutdown sequence without stopping anything or rebooting.
|
|
#
|
|
# server_reboot.sh --status
|
|
# Show running processes that would be affected: rsync, mover, VMs, containers.
|
|
#
|
|
# server_reboot.sh --reason="maintenance"
|
|
# Include reason in wall message and notification. Defaults to "manual".
|
|
#
|
|
# server_reboot.sh --log
|
|
# Verbose output — show each step of the shutdown sequence.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
# ── Parse --reason flag before parse_args ─────────────────────────────────────────────────────
|
|
REBOOT_REASON="manual"
|
|
FILTERED_ARGS=()
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--reason=*) REBOOT_REASON="${arg#--reason=}" ;;
|
|
*) FILTERED_ARGS+=("$arg") ;;
|
|
esac
|
|
done
|
|
|
|
parse_args "${FILTERED_ARGS[@]}"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root — reboot requires root"
|
|
exit 1
|
|
fi
|
|
|
|
validate_unraid_cmd \
|
|
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
|
"" "" \
|
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
|
|
|
validate_int REBOOT_SLEEP "$REBOOT_SLEEP"
|
|
|
|
acquire_lock
|
|
|
|
detect_hosts
|
|
|
|
# ── Exit Trap — restart Docker service if reboot sequence aborts after stopping it ────────────
|
|
_REBOOT_DOCKER_STOPPED=false
|
|
_trap_restart_docker_service() {
|
|
[[ "$_REBOOT_DOCKER_STOPPED" == true ]] || return
|
|
warn "Exit trap: restarting Docker service after aborted reboot sequence"
|
|
/etc/rc.d/rc.docker start >/dev/null 2>&1 || true
|
|
}
|
|
trap _trap_restart_docker_service EXIT
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made, no reboot will occur"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY REBOOT STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s"
|
|
echo "$ICON_GEAR VM wait: ${REBOOT_VM_WAIT:-30}s"
|
|
echo "$ICON_GEAR Reason: $REBOOT_REASON"
|
|
echo ""
|
|
echo "━━━ Active Processes ━━━"
|
|
|
|
pgrep -x rsync >/dev/null 2>&1 && \
|
|
warn " rsync: RUNNING — partial files if rebooted now" || \
|
|
log " rsync: not running"
|
|
|
|
pgrep -f "emhttp.*Mover" >/dev/null 2>&1 && \
|
|
warn " mover: RUNNING — files may be left mid-move" || \
|
|
log " mover: not running"
|
|
|
|
if command -v virsh >/dev/null 2>&1; then
|
|
VM_COUNT=$(virsh list --name 2>/dev/null | grep -c "." || echo 0)
|
|
[[ "$VM_COUNT" -gt 0 ]] && \
|
|
warn " VMs: $VM_COUNT running — will be gracefully shut down" || \
|
|
log " VMs: none running"
|
|
fi
|
|
|
|
if command -v docker >/dev/null 2>&1; then
|
|
CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l || echo 0)
|
|
log " Docker: $CONTAINER_COUNT container(s) running"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Pre-flight Warnings ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
|
WARNINGS=()
|
|
|
|
# rsync check — partial files if killed mid-transfer
|
|
if pgrep -x rsync >/dev/null 2>&1; then
|
|
RSYNC_PIDS=$(pgrep -x rsync | tr '\n' ' ')
|
|
warn "rsync is running (PIDs: $RSYNC_PIDS) — partial files possible"
|
|
warn "Consider: rsync_stop.sh before rebooting"
|
|
WARNINGS+=("rsync running")
|
|
fi
|
|
|
|
# mover check — files may be left mid-move
|
|
if pgrep -f "emhttp.*Mover" >/dev/null 2>&1; then
|
|
warn "Mover is running — files may be left mid-move on cache or array"
|
|
warn "Consider: mover_stop.sh before rebooting"
|
|
WARNINGS+=("mover running")
|
|
fi
|
|
|
|
# Emby sessions check — active streams interrupted
|
|
if [[ -n "${EMBY_URL:-}" ]] && [[ -n "${EMBY_API_KEY:-}" ]]; then
|
|
ACTIVE_STREAMS=$(curl -sf --max-time 5 \
|
|
-H "X-Emby-Token: $EMBY_API_KEY" \
|
|
"${EMBY_URL}/Sessions" 2>/dev/null | \
|
|
grep -c "NowPlayingItem" 2>/dev/null || echo 0)
|
|
ACTIVE_STREAMS="${ACTIVE_STREAMS//[^0-9]/}"
|
|
if [[ "${ACTIVE_STREAMS:-0}" -gt 0 ]]; then
|
|
warn "$ACTIVE_STREAMS active Emby stream(s) — will be interrupted"
|
|
WARNINGS+=("${ACTIVE_STREAMS} Emby sessions")
|
|
fi
|
|
fi
|
|
|
|
if [[ ${#WARNINGS[@]} -eq 0 ]]; then
|
|
log "Pre-flight clean — no active processes to warn about"
|
|
else
|
|
warn "Proceeding with reboot despite warnings — ${WARNINGS[*]}"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Notify and Wait ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_REBOOT Reboot Sequence — $MY_ID ━━━"
|
|
echo " Reason: $REBOOT_REASON"
|
|
echo " Delay: ${REBOOT_SLEEP}s"
|
|
echo " Dry Run: $DRY_RUN"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
|
|
if [[ "$REBOOT_SLEEP" -gt 0 ]]; then
|
|
# Wall message — terminal users
|
|
wall "$ICON_WARN $MY_ID ($LOCAL_SERVER_NAME) rebooting in ${REBOOT_SLEEP}s — reason: $REBOOT_REASON. Save your work now."
|
|
|
|
# unRAID notification — dashboard
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
notify "$MY_ID ($LOCAL_SERVER_NAME) rebooting in ${REBOOT_SLEEP}s — reason: $REBOOT_REASON${WARNINGS:+ — warnings: ${WARNINGS[*]}}" \
|
|
"Server Reboot" "warning"
|
|
fi
|
|
|
|
warn "Waiting ${REBOOT_SLEEP}s before shutdown sequence..."
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
sleep "$REBOOT_SLEEP"
|
|
else
|
|
warn "DRY RUN — skipping sleep"
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Graceful VM Shutdown ━━━
|
|
# ==============================================================================================
|
|
if command -v virsh >/dev/null 2>&1; then
|
|
VM_LIST=$(virsh list --name 2>/dev/null | grep -v "^$" || true)
|
|
if [[ -n "$VM_LIST" ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Graceful VM Shutdown ━━━"
|
|
while IFS= read -r vm; do
|
|
[[ -z "$vm" ]] && continue
|
|
warn "Sending ACPI shutdown to VM: $vm"
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
virsh shutdown "$vm" >/dev/null 2>&1 || true
|
|
else
|
|
warn "DRY RUN — would virsh shutdown $vm"
|
|
fi
|
|
done <<< "$VM_LIST"
|
|
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
VM_WAIT="${REBOOT_VM_WAIT:-30}"
|
|
log "Waiting ${VM_WAIT}s for VMs to shut down..."
|
|
sleep "$VM_WAIT"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Stop VM Manager ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Stop VM Manager ━━━"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would stop VM Manager (libvirt)"
|
|
else
|
|
if /etc/rc.d/rc.libvirt stop >/dev/null 2>&1; then
|
|
warn "VM Manager stopped ✅"
|
|
else
|
|
warn "VM Manager stop returned non-zero — may already be stopped"
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Stop Docker ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_CONTAINERS Stop Docker ━━━"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would stop Docker service"
|
|
else
|
|
if /etc/rc.d/rc.docker stop >/dev/null 2>&1; then
|
|
_REBOOT_DOCKER_STOPPED=true
|
|
warn "Docker stopped ✅"
|
|
else
|
|
warn "Docker stop returned non-zero — may already be stopped"
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Sync Disks ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_DISK Sync Disks ━━━"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would sync filesystem buffers"
|
|
else
|
|
sync
|
|
log "Filesystem buffers flushed ✅"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Reboot ━━━
|
|
# ==============================================================================================
|
|
END=$(date +%s)
|
|
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY REBOOT SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_REBOOT Reason: $REBOOT_REASON"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
[[ ${#WARNINGS[@]} -gt 0 ]] && warn "Warnings: ${WARNINGS[*]}"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — sequence complete, no reboot executed"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
else
|
|
warn "$ICON_REBOOT Rebooting $MY_ID now..."
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
trap - EXIT # committed to reboot — Docker should stay down
|
|
/sbin/reboot
|
|
fi |