Varaverk/Orchestrators/array_stopping.sh

#!/bin/bash
# ==============================================================================================
# ================================= Array Stop Orchestrator ====================================
# ==============================================================================================
# Planned shutdown orchestrator — stops all active processes cleanly before array maintenance.
# Runs ARRAY_STOP_SCRIPTS from master.conf sequentially, each confirmed complete before next.
#
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
#   1. user_scripts_stop.sh        — kill background user scripts (prevents new operations)
#   2. rsync_stop.sh --rsync-only  — kill rsync; skip container recovery (handled in step 4)
#   3. mover_stop.sh               — stop mover after rsync (both write to same paths)
#   4. docker_container_stop.sh    — stop all containers one-by-one with verification
#
# ── WHY THIS ORDER ────────────────────────────────────────────────────────────────────────────
# User scripts stopped first — they can spawn new rsync/docker operations mid-shutdown.
# Rsync before mover — both write to the same paths; running together risks corruption.
# Containers last — apps should stay available as long as possible during shutdown prep.
#
# ── SEQUENTIAL vs BACKGROUND ─────────────────────────────────────────────────────────────────
# Unlike array_started.sh, all scripts run in the foreground. Each must complete (pass or fail)
# before the next starts — a failed stop is noted but does not prevent remaining steps.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
#   Root check           — all stop scripts require root
#   acquire_lock         — prevents concurrent array stop runs
#   detect_hosts()       — MY_ID in notifications and logs
#   platform_require_cmd  — notify validated before use
#   Non-fatal steps      — a failed step is logged but remaining steps still run
#   notify on failures   — alert if any stop script fails
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   ARRAY_STOP_SCRIPTS — ordered list of stop scripts to run
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   array_stopping.sh              — run full stop sequence
#   array_stopping.sh --dry-run    — preview without stopping anything
#   array_stopping.sh --status     — show configured scripts and exit
#   array_stopping.sh --log        — verbose output
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

source "$ECOSYSTEM_ROOT/load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi


acquire_lock

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no stop scripts will be executed"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY ARRAY STOP STATUS ━━━━━"
    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR  Scripts:  ${#ARRAY_STOP_SCRIPTS[@]} configured"
    echo ""

    for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
        [[ -z "$entry" ]] && continue
        read -r -a parts <<< "$entry"
        script_path="$ECOSYSTEM_ROOT/${parts[0]}"
        script_name=$(basename "${parts[0]}")
        extra_args=("${parts[@]:1}")
        if [[ ! -f "$script_path" ]]; then
            echo "  $ICON_ERROR $script_name — FILE NOT FOUND"
        else
            echo "  $ICON_GEAR $script_name${extra_args:+ ${extra_args[*]}}"
        fi
    done
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Stop Sequence ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_STOP Array Stop — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_GEAR Running ${#ARRAY_STOP_SCRIPTS[@]} stop script(s) sequentially..."
echo ""

START=$(date +%s)
PASSED=()
FAILED=()
STEP=0

for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
    [[ -z "$entry" ]] && continue
    (( STEP++ ))

    read -r -a parts <<< "$entry"
    script_path="$ECOSYSTEM_ROOT/${parts[0]}"
    script_name=$(basename "${parts[0]}")
    extra_args=("${parts[@]:1}")

    echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"

    if [[ ! -f "$script_path" ]]; then
        error "$script_name — not found at $script_path"
        FAILED+=("$script_name")
        echo ""
        continue
    fi

    if [[ ! -x "$script_path" ]]; then
        warn "$script_name — not executable, fixing..."
        chmod +x "$script_path" || {
            error "$script_name — chmod +x failed"
            FAILED+=("$script_name")
            echo ""
            continue
        }
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would run: $script_name ${extra_args[*]}"
        PASSED+=("$script_name")
        echo ""
        continue
    fi

    if bash "$script_path" "${extra_args[@]}"; then
        echo "$script_name — done ✅"
        PASSED+=("$script_name")
    else
        warn "$script_name — failed (exit $?) — continuing to next step"
        FAILED+=("$script_name")
    fi

    echo ""
done

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY ARRAY STOP SUMMARY ━━━━━"
echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME    Duration: $(format_duration $(( END - START )))"
[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE    Passed:   ${PASSED[*]}"
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR   Failed:   ${FAILED[*]}"
echo ""

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ ${#FAILED[@]} -eq 0 ]]; then
    echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
    notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
        "Array Stop" "normal"
else
    warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}"
    notify "Array stop on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \
        "Array Stop" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ ${#FAILED[@]} -gt 0 ]] && exit 1
exit 0