Gmer4Lfe
2a062e5140
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
181 lines
7.3 KiB
Bash
Executable File
181 lines
7.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= Array Stop Orchestrator ====================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Planned shutdown orchestrator — stops all active processes cleanly before
|
|
# array maintenance. Runs ARRAY_STOP_SCRIPTS from master.conf sequentially,
|
|
# each confirmed complete before the next starts.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# 1. user_scripts_stop.sh — kill background user scripts (prevents new operations)
|
|
# 2. rsync_stop.sh --rsync-only — kill rsync; skip container recovery (handled in step 4)
|
|
# 3. mover_stop.sh — stop mover after rsync (both write to same paths)
|
|
# 4. docker_container_stop.sh — stop all containers one-by-one with verification
|
|
#
|
|
# Unlike array_started.sh, all scripts run in the foreground. Each must complete
|
|
# (pass or fail) before the next starts — a failed stop is noted but does not
|
|
# prevent remaining steps from running.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Order Is Load-Bearing
|
|
# User scripts are stopped first — they can spawn new rsync or docker operations
|
|
# mid-shutdown. Rsync stops before mover — both write to the same paths and
|
|
# running together risks corruption. Containers stop last — apps should stay
|
|
# available as long as possible during shutdown prep.
|
|
#
|
|
# Non-Fatal Steps
|
|
# A failed stop step is logged and notified but does not abort the sequence.
|
|
# Remaining scripts still run — a partial stop is better than a halted one.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root check — all stop scripts require root
|
|
# acquire_lock — prevents concurrent array stop runs
|
|
# detect_hosts() — MY_ID in notifications and logs
|
|
# platform_require_cmd — notify validated before use
|
|
# notify on failures — alert if any stop script fails
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# ARRAY_STOP_SCRIPTS — ordered list of stop scripts to run
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# array_stopping.sh
|
|
# Run full stop sequence.
|
|
#
|
|
# array_stopping.sh --dry-run
|
|
# Preview without stopping anything.
|
|
#
|
|
# array_stopping.sh --status
|
|
# Show configured scripts and exit.
|
|
#
|
|
# array_stopping.sh --log
|
|
# Verbose output.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
|
|
source "$ECOSYSTEM_ROOT/load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
detect_hosts
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no stop scripts will be executed"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY ARRAY STOP STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_GEAR Scripts: ${#ARRAY_STOP_SCRIPTS[@]} configured"
|
|
echo ""
|
|
|
|
for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
|
|
[[ -z "$entry" ]] && continue
|
|
read -r -a parts <<< "$entry"
|
|
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
|
|
script_name=$(basename "${parts[0]}")
|
|
extra_args=("${parts[@]:1}")
|
|
if [[ ! -f "$script_path" ]]; then
|
|
echo " $ICON_ERROR $script_name — FILE NOT FOUND"
|
|
else
|
|
echo " $ICON_GEAR $script_name${extra_args:+ ${extra_args[*]}}"
|
|
fi
|
|
done
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Stop Sequence ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_STOP Array Stop — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_GEAR Running ${#ARRAY_STOP_SCRIPTS[@]} stop script(s) sequentially..."
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
JOB_PASS=()
|
|
JOB_FAIL=()
|
|
STEP=0
|
|
|
|
for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
|
|
[[ -z "$entry" ]] && continue
|
|
(( STEP++ ))
|
|
|
|
read -r -a parts <<< "$entry"
|
|
script_name=$(basename "${parts[0]}")
|
|
extra_args=("${parts[@]:1}")
|
|
|
|
echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
|
|
run_orch_child "$entry"
|
|
echo ""
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo "━━━━━ $ICON_SUMMARY ARRAY STOP SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
[[ ${#JOB_PASS[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${JOB_PASS[*]}"
|
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no changes made"
|
|
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
|
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
|
notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
|
"Array Stop" "normal"
|
|
else
|
|
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
|
notify "Array stop on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
|
"Array Stop" "warning"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
|
exit 0
|