Varaverk/Docker_Essentials/docker_container_stop.sh

#!/bin/bash
# ==============================================================================================
# ============================= Docker Container Stop ==========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Gracefully stops all running Docker containers in a verified sequential order.
#
# Called by array_stopping.sh during planned shutdowns, maintenance windows,
# and controlled reboot operations.
#
# The script guarantees each container is fully stopped before moving to the
# next one — preventing dependency breakage, partial shutdown states, and
# abrupt service termination cascades.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Shutdown is processed one container at a time:
#
#   1. docker stop -t 30
#        → sends SIGTERM with graceful shutdown window
#
#   2. Verify container state
#        → confirm container fully stopped before continuing
#
#   3. Retry if still running
#        → up to RETRY_COUNT attempts
#
#   4. Escalate to docker kill
#        → SIGKILL only after graceful attempts exhausted
#
#   5. Final verification
#        → failure notification if container survives SIGKILL
#
# The script NEVER advances to the next container until the current one
# is confirmed stopped or declared failed.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Sequential Shutdown
#   Containers may depend on upstream services still being available during
#   shutdown. Sequential processing reduces dependency severance during stop.
#
# Graceful First, Forced Last
#   SIGTERM is always attempted before SIGKILL. The script never force-kills
#   first unless Docker itself escalates internally after timeout expiration.
#
# Verification Over Assumption
#   Docker command success alone is not trusted. Container state is verified
#   after every stop attempt.
#
# Fail Loudly
#   Containers that cannot be stopped generate notifications and non-zero exit
#   status so orchestrators know shutdown integrity was compromised.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Enforcement
#   Docker operations require root privileges.
#
# Docker Presence Check
#   Verifies docker binary exists before execution.
#
# Lock Acquisition
#   acquire_lock prevents overlapping runs (e.g. array_stopping firing twice).
#
# Timeout Protection
#   All docker commands wrapped in timeout protection to prevent daemon hangs
#   from stalling shutdown indefinitely.
#
# Retry Escalation
#   Graceful retries occur before SIGKILL escalation.
#
# Per-Container Validation
#   Every container state verified before progressing to the next.
#
# Deterministic Ordering
#   Running container list sorted before processing for stable execution order.
#
# Failure Notification
#   Containers surviving SIGKILL trigger operator notification.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   RETRY_COUNT
#       Graceful retry attempts before force-kill escalation
#
#   SLEEP
#       Delay in seconds between retry attempts
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# docker_container_stop.sh
#     Stop all running containers in verified sequential order
#
# docker_container_stop.sh --dry-run
#     Preview shutdown actions without stopping containers
#
# docker_container_stop.sh --status
#     Show currently running containers and configuration state
#
# docker_container_stop.sh --log
#     Verbose per-container execution logging
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped"

DOCKER_TIMEOUT=30
DOCKER_STOP_TIMEOUT=30    # grace period for SIGTERM before docker sends SIGKILL internally
_RETRY_COUNT="${RETRY_COUNT:-3}"

log "$ICON_GEAR Config: retries=$_RETRY_COUNT sleep=${SLEEP:-5}s grace=${DOCKER_STOP_TIMEOUT}s cmd-timeout=${DOCKER_TIMEOUT}s"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    mapfile -t RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort)
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST       Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_CONTAINERS Running:   ${#RUNNING[@]} container(s)"
    echo "$ICON_RETRY      Retries:   $_RETRY_COUNT"
    for c in "${RUNNING[@]}"; do echo "  $ICON_RUNNING $c"; done
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Stop Containers ━━━
# ==============================================================================================
mapfile -t RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort)

echo ""
echo "━━━ $ICON_CONTAINERS Docker Container Stop — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

if [[ ${#RUNNING[@]} -eq 0 ]]; then
    echo "No running containers — nothing to do"
    exit 0
fi

echo "$ICON_CONTAINERS Containers: ${#RUNNING[@]} running"
log "$ICON_CONTAINERS Queue: ${RUNNING[*]}"
echo ""

START=$(date +%s)
STOPPED=()
FAILED=()

for container in "${RUNNING[@]}"; do
    [[ -z "$container" ]] && continue
    local c_start
    c_start=$(date +%s)
    local c_image
    c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
    log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would stop $container"
        STOPPED+=("$container")
        continue
    fi

    attempt=1
    success=false

    while [[ "$attempt" -le "$_RETRY_COUNT" ]]; do
        log "$ICON_RETRY Attempt $attempt of $_RETRY_COUNT — stopping $container..."

        timeout "$DOCKER_TIMEOUT" docker stop -t "$DOCKER_STOP_TIMEOUT" "$container" \
            >/dev/null 2>&1

        STATE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
            -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")

        if [[ "$STATE" != "true" ]]; then
            log "$ICON_DONE $container stopped in $(format_duration $(( $(date +%s) - c_start ))) ✅"
            STOPPED+=("$container")
            success=true
            break
        fi

        warn "Attempt $attempt failed — $container still running"
        (( attempt++ ))
        [[ "$attempt" -le "$_RETRY_COUNT" ]] && sleep "${SLEEP:-5}"
    done

    if [[ "$success" == false ]]; then
        warn "$container — retries exhausted, force-killing with SIGKILL..."
        timeout "$DOCKER_TIMEOUT" docker kill "$container" >/dev/null 2>&1 || true

        STATE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
            -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")

        if [[ "$STATE" != "true" ]]; then
            warn "$container force-killed ✅"
            STOPPED+=("$container")
        else
            error "$container still running after SIGKILL — manual intervention needed"
            notify "$container could not be stopped on $(hostname) ($MY_ID)" \
                "Docker Container Stop" "warning"
            FAILED+=("$container")
        fi
    fi
done

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY DOCKER CONTAINER STOP SUMMARY ━━━━━"
echo "$ICON_HOST       Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME       Duration:  $(format_duration $(( END - START )))"
echo "$ICON_CONTAINERS Scope:     ${#RUNNING[@]} running → ${#STOPPED[@]} stopped"
[[ ${#STOPPED[@]} -gt 0 ]] && log "$ICON_DONE  Stopped:   ${STOPPED[*]}"
[[ ${#FAILED[@]} -gt 0 ]]  && echo "$ICON_ERROR Failed:    ${FAILED[*]}"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no containers stopped"
elif [[ ${#FAILED[@]} -eq 0 ]]; then
    echo "$ICON_DONE Status: done ✅ — ${#STOPPED[@]} container(s) stopped"
else
    warn "Status: ${#FAILED[@]} container(s) could not be stopped"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ ${#FAILED[@]} -gt 0 ]] && exit 1
exit 0