Gmer4Lfe
740710e0d0
docker_container_stop, docker_update, docker_update_remaining — concurrent Docker operations on the same containers would conflict; now locked. rsync.sh — two rsync processes running against the same share simultaneously would produce incomplete or corrupted mirrors; now locked. partnership_onboard, ssh_setup — one-shot setup scripts that mutate SSH config and deploy containers; concurrent runs would produce undefined state; now locked. Old_Arch_Still_Works: arr_cleanup, continuous_scripts_status — legacy scripts still sourcing load_config.sh; added lock for consistency even in old-arch context. partnership_manager.sh intentionally left unchanged — it uses a conditional lock that excludes read-only "check" mode and "offboard" mode (which delegates to partnership_offboard.sh, which has its own lock).
260 lines
9.5 KiB
Bash
Executable File
260 lines
9.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Docker Container Stop ==========================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Gracefully stops all running Docker containers in a verified sequential order.
|
|
#
|
|
# Called by array_stopping.sh during planned shutdowns, maintenance windows,
|
|
# and controlled reboot operations.
|
|
#
|
|
# The script guarantees each container is fully stopped before moving to the
|
|
# next one — preventing dependency breakage, partial shutdown states, and
|
|
# abrupt service termination cascades.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Shutdown is processed one container at a time:
|
|
#
|
|
# 1. docker stop -t 30
|
|
# → sends SIGTERM with graceful shutdown window
|
|
#
|
|
# 2. Verify container state
|
|
# → confirm container fully stopped before continuing
|
|
#
|
|
# 3. Retry if still running
|
|
# → up to RETRY_COUNT attempts
|
|
#
|
|
# 4. Escalate to docker kill
|
|
# → SIGKILL only after graceful attempts exhausted
|
|
#
|
|
# 5. Final verification
|
|
# → failure notification if container survives SIGKILL
|
|
#
|
|
# The script NEVER advances to the next container until the current one
|
|
# is confirmed stopped or declared failed.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Sequential Shutdown
|
|
# Containers may depend on upstream services still being available during
|
|
# shutdown. Sequential processing reduces dependency severance during stop.
|
|
#
|
|
# Graceful First, Forced Last
|
|
# SIGTERM is always attempted before SIGKILL. The script never force-kills
|
|
# first unless Docker itself escalates internally after timeout expiration.
|
|
#
|
|
# Verification Over Assumption
|
|
# Docker command success alone is not trusted. Container state is verified
|
|
# after every stop attempt.
|
|
#
|
|
# Fail Loudly
|
|
# Containers that cannot be stopped generate notifications and non-zero exit
|
|
# status so orchestrators know shutdown integrity was compromised.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root Enforcement
|
|
# Docker operations require root privileges.
|
|
#
|
|
# Docker Presence Check
|
|
# Verifies docker binary exists before execution.
|
|
#
|
|
# Timeout Protection
|
|
# All docker commands wrapped in timeout protection to prevent daemon hangs
|
|
# from stalling shutdown indefinitely.
|
|
#
|
|
# Retry Escalation
|
|
# Graceful retries occur before SIGKILL escalation.
|
|
#
|
|
# Per-Container Validation
|
|
# Every container state verified before progressing to the next.
|
|
#
|
|
# Deterministic Ordering
|
|
# Running container list sorted before processing for stable execution order.
|
|
#
|
|
# Failure Notification
|
|
# Containers surviving SIGKILL trigger operator notification.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# RETRY_COUNT
|
|
# Graceful retry attempts before force-kill escalation
|
|
#
|
|
# SLEEP
|
|
# Delay in seconds between retry attempts
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# docker_container_stop.sh
|
|
# Stop all running containers in verified sequential order
|
|
#
|
|
# docker_container_stop.sh --dry-run
|
|
# Preview shutdown actions without stopping containers
|
|
#
|
|
# docker_container_stop.sh --status
|
|
# Show currently running containers and configuration state
|
|
#
|
|
# docker_container_stop.sh --log
|
|
# Verbose per-container execution logging
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
detect_hosts
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped"
|
|
|
|
DOCKER_TIMEOUT=30
|
|
DOCKER_STOP_TIMEOUT=30 # grace period for SIGTERM before docker sends SIGKILL internally
|
|
_RETRY_COUNT="${RETRY_COUNT:-3}"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
mapfile -t RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort)
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_CONTAINERS Running: ${#RUNNING[@]} container(s)"
|
|
echo "$ICON_RETRY Retries: $_RETRY_COUNT"
|
|
for c in "${RUNNING[@]}"; do echo " $ICON_RUNNING $c"; done
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Stop Containers ━━━
|
|
# ==============================================================================================
|
|
mapfile -t RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort)
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_CONTAINERS Docker Container Stop — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
|
|
if [[ ${#RUNNING[@]} -eq 0 ]]; then
|
|
log "No running containers — nothing to do"
|
|
exit 0
|
|
fi
|
|
|
|
echo "$ICON_CONTAINERS Containers: ${#RUNNING[@]} running"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
STOPPED=()
|
|
FAILED=()
|
|
|
|
for container in "${RUNNING[@]}"; do
|
|
[[ -z "$container" ]] && continue
|
|
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would stop $container"
|
|
STOPPED+=("$container")
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
attempt=1
|
|
success=false
|
|
|
|
while [[ "$attempt" -le "$_RETRY_COUNT" ]]; do
|
|
log "$ICON_RETRY Attempt $attempt of $_RETRY_COUNT — stopping $container..."
|
|
|
|
timeout "$DOCKER_TIMEOUT" docker stop -t "$DOCKER_STOP_TIMEOUT" "$container" \
|
|
>/dev/null 2>&1
|
|
|
|
STATE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
|
|
-f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
|
|
|
if [[ "$STATE" != "true" ]]; then
|
|
echo "$ICON_DONE $container stopped ✅"
|
|
STOPPED+=("$container")
|
|
success=true
|
|
break
|
|
fi
|
|
|
|
warn "Attempt $attempt failed — $container still running"
|
|
(( attempt++ ))
|
|
[[ "$attempt" -le "$_RETRY_COUNT" ]] && sleep "${SLEEP:-5}"
|
|
done
|
|
|
|
if [[ "$success" == false ]]; then
|
|
warn "$container — retries exhausted, force-killing with SIGKILL..."
|
|
timeout "$DOCKER_TIMEOUT" docker kill "$container" >/dev/null 2>&1 || true
|
|
|
|
STATE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
|
|
-f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
|
|
|
if [[ "$STATE" != "true" ]]; then
|
|
warn "$container force-killed ✅"
|
|
STOPPED+=("$container")
|
|
else
|
|
error "$container still running after SIGKILL — manual intervention needed"
|
|
notify "$container could not be stopped on $(hostname) ($MY_ID)" \
|
|
"Docker Container Stop" "warning"
|
|
FAILED+=("$container")
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo "━━━━━ $ICON_SUMMARY DOCKER CONTAINER STOP SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo "$ICON_CONTAINERS Scope: ${#RUNNING[@]} running → ${#STOPPED[@]} stopped"
|
|
[[ ${#STOPPED[@]} -gt 0 ]] && echo "$ICON_DONE Stopped: ${STOPPED[*]}"
|
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no containers stopped"
|
|
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
|
log "$ICON_DONE Status: done ✅ — ${#STOPPED[@]} container(s) stopped"
|
|
else
|
|
warn "Status: ${#FAILED[@]} container(s) could not be stopped"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
|
exit 0
|