Gmer4Lfe
d5cf3db2ec
A gated-off section left nothing failed, so the weekly could run for hours and report "all complete" beside "0 shares synced"; skipped is now derived from what was expected rather than self-reported, and the verdict degrades to PARTIAL instead of flattering.
196 lines
7.8 KiB
Bash
Executable File
196 lines
7.8 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= Array Stop Orchestrator ====================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Planned shutdown orchestrator — stops all active processes cleanly before
|
|
# array maintenance. Runs ARRAY_STOP_SCRIPTS from master.conf sequentially,
|
|
# each confirmed complete before the next starts.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# 1. user_scripts_stop.sh — kill background user scripts (prevents new operations)
|
|
# 2. rsync_stop.sh --rsync-only — kill rsync; skip container recovery (handled in step 4)
|
|
# 3. mover_stop.sh — stop mover after rsync (both write to same paths)
|
|
# 4. docker_container_stop.sh — stop all containers one-by-one with verification
|
|
#
|
|
# Unlike array_started.sh, all scripts run in the foreground. Each must complete
|
|
# (pass or fail) before the next starts — a failed stop is noted but does not
|
|
# prevent remaining steps from running.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Order Is Load-Bearing
|
|
# User scripts are stopped first — they can spawn new rsync or docker operations
|
|
# mid-shutdown. Rsync stops before mover — both write to the same paths and
|
|
# running together risks corruption. Containers stop last — apps should stay
|
|
# available as long as possible during shutdown prep.
|
|
#
|
|
# Non-Fatal Steps
|
|
# A failed stop step is logged and notified but does not abort the sequence.
|
|
# Remaining scripts still run — a partial stop is better than a halted one.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root Enforcement
|
|
# Every stop script launched here requires root.
|
|
#
|
|
# Lock Acquisition
|
|
# acquire_lock prevents concurrent array stop runs. Two overlapping shutdown
|
|
# sequences would fight over the same containers.
|
|
#
|
|
# Host Detection
|
|
# detect_hosts() sets MY_ID for notifications and logs.
|
|
#
|
|
# Empty Job List Guard
|
|
# Exits with an error and a notification if ARRAY_STOP_SCRIPTS is empty. An empty
|
|
# list means the array stops without saving the conf cache or gracefully stopping
|
|
# containers — the failure would only be discovered at the next boot.
|
|
#
|
|
# Non-Fatal Steps
|
|
# A failing stop script is recorded and the remaining ones still run. Abandoning the
|
|
# shutdown sequence partway would leave more state unsaved than continuing does.
|
|
#
|
|
# Failure Notification
|
|
# Any failing stop script raises a notification. Shutdown is unattended and its
|
|
# failures are invisible until they cause a problem on the way back up.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# ARRAY_STOP_SCRIPTS — ordered list of stop scripts to run
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# array_stopping.sh
|
|
# Run full stop sequence.
|
|
#
|
|
# array_stopping.sh --dry-run
|
|
# Preview without stopping anything.
|
|
#
|
|
# array_stopping.sh --status
|
|
# Show configured scripts and exit.
|
|
#
|
|
# array_stopping.sh --log
|
|
# Verbose output.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
|
|
source "$ECOSYSTEM_ROOT/load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
detect_hosts
|
|
|
|
# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
|
|
# from a healthy run. Fail loudly instead of silently doing no work.
|
|
if [[ ${#ARRAY_STOP_SCRIPTS[@]} -eq 0 ]]; then
|
|
error "ARRAY_STOP_SCRIPTS is empty — no array stop scripts will run"
|
|
error "Check ARRAY_STOP_SCRIPTS in master.conf"
|
|
notify "array stop scripts skipped on $(hostname) ($MY_ID) — ARRAY_STOP_SCRIPTS is empty" \
|
|
"$(basename "$0" .sh)" "warning"
|
|
exit 1
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no stop scripts will be executed"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY ARRAY STOP STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_GEAR Scripts: ${#ARRAY_STOP_SCRIPTS[@]} configured"
|
|
echo ""
|
|
|
|
for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
|
|
[[ -z "$entry" ]] && continue
|
|
read -r -a parts <<< "$entry"
|
|
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
|
|
script_name=$(basename "${parts[0]}")
|
|
extra_args=("${parts[@]:1}")
|
|
if [[ ! -f "$script_path" ]]; then
|
|
echo " $ICON_ERROR $script_name — FILE NOT FOUND"
|
|
else
|
|
echo " $ICON_GEAR $script_name${extra_args:+ ${extra_args[*]}}"
|
|
fi
|
|
done
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Stop Sequence ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_STOP Array Stop — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_GEAR Running ${#ARRAY_STOP_SCRIPTS[@]} stop script(s) sequentially..."
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
JOB_PASS=()
|
|
JOB_FAIL=()
|
|
STEP=0
|
|
|
|
for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
|
|
[[ -z "$entry" ]] && continue
|
|
(( STEP++ ))
|
|
|
|
read -r -a parts <<< "$entry"
|
|
script_name=$(basename "${parts[0]}")
|
|
extra_args=("${parts[@]:1}")
|
|
|
|
echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
|
|
run_orch_child "$entry"
|
|
echo ""
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo "━━━━━ $ICON_SUMMARY ARRAY STOP SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
[[ ${#JOB_PASS[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${JOB_PASS[*]}"
|
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
|
echo ""
|
|
|
|
JOB_COUNT="$STEP"
|
|
orchestrator_summary "ARRAY STOP" "$START" "Array Stop"
|
|
exit $?
|