Gmer4Lfe
b00688bad1
Chunker splits on the header sections the audit standardised, then sub-splits named-paragraph safeguards — without that a specific question about one of rsync.sh's fourteen safeguards scored below unrelated chunks, because the other thirteen dominated the vector. Index is SQLite with raw float32 blobs and is incremental on mtime; a no-op re-index takes 66ms. The stale defaults were found by asking the system a question and checking its answer: it correctly reported what mover_stop.sh's header claimed, and the header was wrong.
360 lines
14 KiB
Bash
Executable File
360 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= Server Reboot ==============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Gracefully reboots the unRAID server with pre-flight checks, user warnings,
|
|
# clean service shutdown, and disk sync. Use instead of raw /sbin/reboot —
|
|
# gives users warning time and ensures services stop cleanly before the kernel
|
|
# drops.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Shutdown Sequence
|
|
# 1. Pre-flight warnings — rsync, mover, active Emby sessions (warn, not block)
|
|
# 2. Wall message to all logged-in terminal users
|
|
# 3. unRAID dashboard notification
|
|
# 4. Wait REBOOT_SLEEP seconds — users time to save work
|
|
# 5. array_stopping.sh — user scripts, rsync, mover, containers (verified stop)
|
|
# 6. Graceful VM shutdown via virsh — ACPI signal, then wait REBOOT_VM_WAIT
|
|
# 7. Stop libvirt (VM Manager)
|
|
# 8. sync — filesystem buffers flushed to disk
|
|
# 9. /sbin/reboot
|
|
#
|
|
# Pre-flight Warnings (informational — do not block)
|
|
# rsync running → partial files possible if mid-transfer
|
|
# mover running → files may be left mid-move on cache or array
|
|
# Emby sessions → active streams/transcodes interrupted
|
|
# Warnings do not block the reboot — you called this script, you know.
|
|
#
|
|
# VM Graceful Shutdown
|
|
# virsh shutdown sends the ACPI power button signal — same as pressing the
|
|
# physical power button. VM gets a chance to flush buffers and shut down.
|
|
# After REBOOT_VM_WAIT seconds, libvirt stops anyway — reboot takes priority.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Warn, Do Not Block
|
|
# Pre-flight reports what is in flight — rsync running, mover running — and proceeds
|
|
# anyway. This is an operator-invoked tool: the person running it has already decided
|
|
# to reboot, and refusing would just push them to /sbin/reboot with no warning, no
|
|
# wall message and no clean array stop. The warnings name the specific script to run
|
|
# first (rsync_stop.sh, mover_stop.sh) so the safer path is the obvious one.
|
|
#
|
|
# Announce Before Acting
|
|
# A wall message and a notification go out REBOOT_SLEEP seconds ahead, both naming
|
|
# which host is rebooting and why. On a two-server setup "the server is rebooting" is
|
|
# ambiguous and therefore useless.
|
|
#
|
|
# Clean Array Stop First
|
|
# The reboot routes through the normal array stop sequence rather than calling
|
|
# /sbin/reboot directly. Array stop failures are reported and the reboot continues —
|
|
# an already-committed reboot should not be abandoned halfway, leaving services down
|
|
# and the machine still up.
|
|
#
|
|
# Flush Before Cutting Power
|
|
# sync runs immediately before /sbin/reboot so buffered writes reach disk.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root Required
|
|
# /sbin/reboot requires root.
|
|
#
|
|
# Single Instance Lock
|
|
# acquire_lock prevents concurrent reboot calls.
|
|
#
|
|
# Host Identity in All Messages
|
|
# detect_hosts() sets MY_ID — wall and notifications show which server is
|
|
# rebooting. Critical on a two-server setup.
|
|
#
|
|
# sync Before Reboot
|
|
# filesystem buffers flushed to disk before reboot command.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# REBOOT_SLEEP
|
|
# Seconds between warning and shutdown sequence start. (shipped default: 300)
|
|
#
|
|
# REBOOT_VM_WAIT
|
|
# Seconds to wait for VMs to shut down gracefully. (default: 30)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# server_reboot.sh
|
|
# Run pre-flight, warn users, stop services cleanly, then reboot.
|
|
#
|
|
# server_reboot.sh --dry-run
|
|
# Walk through the entire shutdown sequence without stopping anything or rebooting.
|
|
#
|
|
# server_reboot.sh --status
|
|
# Show running processes that would be affected: rsync, mover, VMs, containers.
|
|
#
|
|
# server_reboot.sh --reason="maintenance"
|
|
# Include reason in wall message and notification. Defaults to "manual".
|
|
#
|
|
# server_reboot.sh --log
|
|
# Verbose output — show each step of the shutdown sequence.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
# ── Parse --reason flag before parse_args ─────────────────────────────────────────────────────
|
|
REBOOT_REASON="manual"
|
|
FILTERED_ARGS=()
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--reason=*) REBOOT_REASON="${arg#--reason=}" ;;
|
|
*) FILTERED_ARGS+=("$arg") ;;
|
|
esac
|
|
done
|
|
|
|
parse_args "${FILTERED_ARGS[@]}"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root — reboot requires root"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
validate_int REBOOT_SLEEP "$REBOOT_SLEEP"
|
|
|
|
acquire_lock
|
|
|
|
detect_hosts
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made, no reboot will occur"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY REBOOT STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_REBOOT Delay: ${REBOOT_SLEEP}s"
|
|
echo "$ICON_GEAR VM wait: ${REBOOT_VM_WAIT:-30}s"
|
|
echo "$ICON_GEAR Reason: $REBOOT_REASON"
|
|
echo ""
|
|
echo "━━━ Active Processes ━━━"
|
|
|
|
pgrep -x rsync >/dev/null 2>&1 && \
|
|
warn " rsync: RUNNING — partial files if rebooted now" || \
|
|
log " rsync: not running"
|
|
|
|
platform_is_mover_running && \
|
|
warn " mover: RUNNING — files may be left mid-move" || \
|
|
log " mover: not running"
|
|
|
|
if command -v virsh >/dev/null 2>&1; then
|
|
VM_COUNT=$(virsh list --name 2>/dev/null | grep -c "." || echo 0)
|
|
[[ "$VM_COUNT" -gt 0 ]] && \
|
|
warn " VMs: $VM_COUNT running — will be gracefully shut down" || \
|
|
log " VMs: none running"
|
|
fi
|
|
|
|
if command -v docker >/dev/null 2>&1; then
|
|
CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l || echo 0)
|
|
log " Docker: $CONTAINER_COUNT container(s) running"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Pre-flight Warnings ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
|
WARNINGS=()
|
|
|
|
# rsync check — partial files if killed mid-transfer
|
|
if pgrep -x rsync >/dev/null 2>&1; then
|
|
RSYNC_PIDS=$(pgrep -x rsync | tr '\n' ' ')
|
|
warn "rsync is running (PIDs: $RSYNC_PIDS) — partial files possible"
|
|
warn "Consider: rsync_stop.sh before rebooting"
|
|
WARNINGS+=("rsync running")
|
|
fi
|
|
|
|
# mover check — files may be left mid-move
|
|
if platform_is_mover_running; then
|
|
warn "Mover is running — files may be left mid-move on cache or array"
|
|
warn "Consider: mover_stop.sh before rebooting"
|
|
WARNINGS+=("mover running")
|
|
fi
|
|
|
|
# Emby sessions check — active streams interrupted
|
|
if [[ -n "${EMBY_URL:-}" ]] && [[ -n "${EMBY_API_KEY:-}" ]]; then
|
|
ACTIVE_STREAMS=$(curl -sf --max-time 5 \
|
|
-H "X-Emby-Token: $EMBY_API_KEY" \
|
|
"${EMBY_URL}/Sessions" 2>/dev/null | \
|
|
grep -c "NowPlayingItem" 2>/dev/null || echo 0)
|
|
ACTIVE_STREAMS="${ACTIVE_STREAMS//[^0-9]/}"
|
|
if [[ "${ACTIVE_STREAMS:-0}" -gt 0 ]]; then
|
|
warn "$ACTIVE_STREAMS active Emby stream(s) — will be interrupted"
|
|
WARNINGS+=("${ACTIVE_STREAMS} Emby sessions")
|
|
fi
|
|
fi
|
|
|
|
CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l || echo 0)
|
|
log "$ICON_CONTAINERS Docker: ${CONTAINER_COUNT} container(s) running"
|
|
|
|
if is_vm_manager_enabled && command -v virsh >/dev/null 2>&1; then
|
|
VM_COUNT=$(virsh list --name 2>/dev/null | grep -c "." || echo 0)
|
|
log "$ICON_GEAR VMs: ${VM_COUNT} running"
|
|
fi
|
|
|
|
if [[ ${#WARNINGS[@]} -eq 0 ]]; then
|
|
log "Pre-flight clean — no active processes to warn about"
|
|
else
|
|
warn "Proceeding with reboot despite warnings — ${WARNINGS[*]}"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Notify and Wait ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_REBOOT Reboot Sequence — $MY_ID ━━━"
|
|
echo " Reason: $REBOOT_REASON"
|
|
echo " Delay: ${REBOOT_SLEEP}s"
|
|
echo " Dry Run: $DRY_RUN"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
|
|
if [[ "$REBOOT_SLEEP" -gt 0 ]]; then
|
|
# Wall message — terminal users
|
|
wall "$ICON_WARN $MY_ID ($LOCAL_SERVER_NAME) rebooting in ${REBOOT_SLEEP}s — reason: $REBOOT_REASON. Save your work now."
|
|
|
|
# unRAID notification — dashboard
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
notify "$MY_ID ($LOCAL_SERVER_NAME) rebooting in ${REBOOT_SLEEP}s — reason: $REBOOT_REASON${WARNINGS:+ — warnings: ${WARNINGS[*]}}" \
|
|
"Server Reboot" "warning"
|
|
fi
|
|
|
|
warn "Waiting ${REBOOT_SLEEP}s before shutdown sequence..."
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
sleep "$REBOOT_SLEEP"
|
|
else
|
|
warn "DRY RUN — skipping sleep"
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Array Stop Orchestrator ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_STOP Array Stop Orchestrator ━━━"
|
|
ARRAY_STOP_SCRIPT="$SCRIPT_DIR/../Orchestrators/array_stopping.sh"
|
|
|
|
if [[ ! -f "$ARRAY_STOP_SCRIPT" ]]; then
|
|
warn "array_stopping.sh not found — skipping orchestrated stop"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
bash "$ARRAY_STOP_SCRIPT" --dry-run
|
|
else
|
|
if bash "$ARRAY_STOP_SCRIPT"; then
|
|
echo "Array stop complete ✅"
|
|
else
|
|
warn "array_stopping.sh reported failures — proceeding with reboot"
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Graceful VM Shutdown ━━━
|
|
# ==============================================================================================
|
|
if is_vm_manager_enabled && command -v virsh >/dev/null 2>&1; then
|
|
VM_LIST=$(virsh list --name 2>/dev/null | grep -v "^$" || true)
|
|
if [[ -n "$VM_LIST" ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Graceful VM Shutdown ━━━"
|
|
while IFS= read -r vm; do
|
|
[[ -z "$vm" ]] && continue
|
|
warn "Sending ACPI shutdown to VM: $vm"
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
virsh shutdown "$vm" >/dev/null 2>&1 || true
|
|
else
|
|
warn "DRY RUN — would virsh shutdown $vm"
|
|
fi
|
|
done <<< "$VM_LIST"
|
|
|
|
if [[ "$DRY_RUN" == false ]]; then
|
|
VM_WAIT="${REBOOT_VM_WAIT:-30}"
|
|
log "Waiting ${VM_WAIT}s for VMs to shut down..."
|
|
sleep "$VM_WAIT"
|
|
fi
|
|
else
|
|
log "VM Manager enabled but no VMs running — skipping shutdown"
|
|
fi
|
|
else
|
|
log "VM Manager not enabled — skipping VM shutdown"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Stop VM Manager ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Stop VM Manager ━━━"
|
|
if ! is_vm_manager_enabled; then
|
|
log "VM Manager not enabled — skipping"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would stop VM Manager (libvirt)"
|
|
else
|
|
if platform_stop_service libvirt; then
|
|
warn "VM Manager stopped ✅"
|
|
else
|
|
warn "VM Manager stop returned non-zero — may already be stopped"
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Sync Disks ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_DISK Sync Disks ━━━"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would sync filesystem buffers"
|
|
else
|
|
sync
|
|
echo "Filesystem buffers flushed ✅"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Reboot ━━━
|
|
# ==============================================================================================
|
|
END=$(date +%s)
|
|
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY REBOOT SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_REBOOT Reason: $REBOOT_REASON"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
[[ ${#WARNINGS[@]} -gt 0 ]] && warn "Warnings: ${WARNINGS[*]}"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — sequence complete, no reboot executed"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
else
|
|
warn "$ICON_REBOOT Rebooting $MY_ID now..."
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
/sbin/reboot
|
|
fi |