#!/bin/bash # ============================================================================================== # ================================= Array Stop Orchestrator ==================================== # ============================================================================================== # Planned shutdown orchestrator — stops all active processes cleanly before array maintenance. # Runs ARRAY_STOP_SCRIPTS from master.conf sequentially, each confirmed complete before next. # # ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── # 1. user_scripts_stop.sh — kill background user scripts (prevents new operations) # 2. rsync_stop.sh --rsync-only — kill rsync; skip container recovery (handled in step 4) # 3. mover_stop.sh — stop mover after rsync (both write to same paths) # 4. docker_container_stop.sh — stop all containers one-by-one with verification # # ── WHY THIS ORDER ──────────────────────────────────────────────────────────────────────────── # User scripts stopped first — they can spawn new rsync/docker operations mid-shutdown. # Rsync before mover — both write to the same paths; running together risks corruption. # Containers last — apps should stay available as long as possible during shutdown prep. # # ── SEQUENTIAL vs BACKGROUND ───────────────────────────────────────────────────────────────── # Unlike array_start.sh, all scripts run in the foreground. Each must complete (pass or fail) # before the next starts — a failed stop is noted but does not prevent remaining steps. # # ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # Root check — all stop scripts require root # acquire_lock — prevents concurrent array stop runs # detect_hosts() — MY_ID in notifications and logs # validate_unraid_cmd — notify validated before use # Non-fatal steps — a failed step is logged but remaining steps still run # notify on failures — alert if any stop script fails # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # ARRAY_STOP_SCRIPTS — ordered list of stop scripts to run # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # array_stop.sh — run full stop sequence # array_stop.sh --dry-run — preview without stopping anything # array_stop.sh --status — show configured scripts and exit # array_stop.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "$ECOSYSTEM_ROOT/load_config.sh" parse_args "$@" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi validate_unraid_cmd \ "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ "" "" \ "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" acquire_lock detect_hosts [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no stop scripts will be executed" # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY ARRAY STOP STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_GEAR Scripts: ${#ARRAY_STOP_SCRIPTS[@]} configured" echo "" for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do [[ -z "$entry" ]] && continue read -r -a parts <<< "$entry" script_path="$ECOSYSTEM_ROOT/${parts[0]}" script_name=$(basename "${parts[0]}") extra_args=("${parts[@]:1}") if [[ ! -f "$script_path" ]]; then echo " $ICON_ERROR $script_name — FILE NOT FOUND" else echo " $ICON_GEAR $script_name${extra_args:+ ${extra_args[*]}}" fi done echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ━━━ Stop Sequence ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_STOP Array Stop — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_GEAR Running ${#ARRAY_STOP_SCRIPTS[@]} stop script(s) sequentially..." echo "" START=$(date +%s) PASSED=() FAILED=() STEP=0 for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do [[ -z "$entry" ]] && continue (( STEP++ )) read -r -a parts <<< "$entry" script_path="$ECOSYSTEM_ROOT/${parts[0]}" script_name=$(basename "${parts[0]}") extra_args=("${parts[@]:1}") echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━" if [[ ! -f "$script_path" ]]; then error "$script_name — not found at $script_path" FAILED+=("$script_name") echo "" continue fi if [[ ! -x "$script_path" ]]; then warn "$script_name — not executable, fixing..." chmod +x "$script_path" || { error "$script_name — chmod +x failed" FAILED+=("$script_name") echo "" continue } fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would run: $script_name ${extra_args[*]}" PASSED+=("$script_name") echo "" continue fi if bash "$script_path" "${extra_args[@]}"; then log "$script_name — done ✅" PASSED+=("$script_name") else warn "$script_name — failed (exit $?) — continuing to next step" FAILED+=("$script_name") fi echo "" done END=$(date +%s) # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "━━━━━ $ICON_SUMMARY ARRAY STOP SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" [[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}" [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ ${#FAILED[@]} -eq 0 ]]; then log "$ICON_DONE Status: all $STEP step(s) complete ✅" notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \ "Array Stop" "normal" else warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}" notify "Array stop on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \ "Array Stop" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" [[ ${#FAILED[@]} -gt 0 ]] && exit 1 exit 0