Varaverk/Docker_Essentials/docker_update_remaining.sh

#!/bin/bash
# ==============================================================================================
# ============================= Docker Update — Remaining ======================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Weekly sweep that pulls the latest image for every running container not
# already covered by the daily or weekly managed update cycles. Restarts
# containers that received a new image, then prunes dangling images.
#
# Called by weekly_sync_maintenance.sh as the final step in the weekly window.
# Derives its target list automatically from docker ps minus the two managed
# lists — there is nothing to configure for this script.
#
# Together with docker_update.sh (normal + remainder modes), every deployed
# container receives at least one image pull per week without any per-container
# configuration required here.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Lock Acquisition
#   Prevents concurrent execution via acquire_lock(). Safe to call from
#   weekly maintenance scripts without risk of overlap.
#
# Host Detection
#   detect_hosts() identifies which server is running the script and aliases
#   HOST*_DAILY_RESTART_CONTAINERS and HOST*_WEEKLY_RESTART_CONTAINERS to
#   the correct host's values for exclusion.
#
# Root Enforcement
#   Docker operations require root privileges.
#
# WEEKLY_REMAINING_UPDATES Toggle
#   Exits cleanly when disabled via master.conf.
#
# Running-Only Filter
#   Stopped containers excluded — intentionally down, pulling adds no value.
#
# Image ID Comparison
#   Containers not restarted unless their image actually changed.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   WEEKLY_REMAINING_UPDATES
#       Enable or disable this script. (default: true)
#       To disable without the toggle: remove from WEEKLY_MAINTENANCE_SCRIPTS.
#
# host*.conf
#
#   HOST*_DAILY_RESTART_CONTAINERS
#       Excluded from this script — already updated daily. Aliased by
#       detect_hosts() → DAILY_RESTART_CONTAINERS
#
#   HOST*_WEEKLY_RESTART_CONTAINERS
#       Excluded from this script — already updated by weekly sync window.
#       Aliased by detect_hosts() → WEEKLY_RESTART_CONTAINERS
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# docker_update_remaining.sh
#     Pull all remaining running containers, restart those updated, prune images
#
# docker_update_remaining.sh --dry-run
#     Preview which containers would be pulled and restarted
#
# docker_update_remaining.sh --status
#     Show exclusion lists and current remaining container count
#
# docker_update_remaining.sh --log
#     Verbose per-container pull and restart output
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts

if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then
    echo "WEEKLY_REMAINING_UPDATES=false — skipping remaining container updates"
    exit 0
fi

# ── Build exclusion set from daily + weekly managed lists ─────────────────────────────────────
declare -A EXCLUDED
for c in "${DAILY_RESTART_CONTAINERS[@]}"  "${WEEKLY_RESTART_CONTAINERS[@]}"; do
    [[ -n "$c" ]] && EXCLUDED["$c"]=1
done

# ── Get all running containers ────────────────────────────────────────────────────────────────
mapfile -t ALL_RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort)

# ── Derive remainder: running minus excluded ──────────────────────────────────────────────────
REMAINING=()
for c in "${ALL_RUNNING[@]}"; do
    [[ -z "$c" ]] && continue
    [[ -n "${EXCLUDED[$c]:-}" ]] && continue
    REMAINING+=("$c")
done

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR       Enabled:     ${WEEKLY_REMAINING_UPDATES:-true}"
    echo "$ICON_CONTAINERS All running: ${#ALL_RUNNING[@]}"
    echo "$ICON_CONTAINERS Excluded:    ${!EXCLUDED[*]}"
    echo "$ICON_CONTAINERS Remaining:   ${REMAINING[*]:-none}"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

if [[ ${#REMAINING[@]} -eq 0 ]]; then
    echo "No remaining containers to update — all running containers are covered by daily/weekly lists"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled or containers restarted"

# ==============================================================================================
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

DOCKER_TIMEOUT=30
docker_cmd() {
    timeout "$DOCKER_TIMEOUT" "$@"
    local exit_code=$?
    if [[ "$exit_code" -eq 124 ]]; then
        error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*"
        return 1
    fi
    return "$exit_code"
}

retry_docker() {
    local attempt=1
    while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
        log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
        if docker_cmd "$@"; then
            log "Succeeded on attempt $attempt"
            return 0
        else
            warn "Attempt $attempt failed"
            (( attempt++ ))
            [[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP"
        fi
    done
    error "Command failed after $RETRY_COUNT attempts: $*"
    return 1
}

RESTART_VERIFY_WAIT=5
verify_running() {
    local container="$1"
    sleep "$RESTART_VERIFY_WAIT"
    local state
    state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
    [[ "$state" == "true" ]]
}

# ==============================================================================================
# ━━━ Pull Updates ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Docker Update (Remaining) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
log "$ICON_CONTAINERS Containers: ${REMAINING[*]}"
log "$ICON_CONTAINERS Excluded (managed elsewhere): ${!EXCLUDED[*]}"
echo ""

START=$(date +%s)
UPDATED=()
UP_TO_DATE=()
FAILED=()

for container in "${REMAINING[@]}"; do
    [[ -z "$container" ]] && continue
    log "━━━ $ICON_CONTAINERS $container ━━━"

    IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
    if [[ -z "$IMAGE" ]]; then
        warn "$container — could not determine image, skipping"
        FAILED+=("$container")
        continue
    fi

    log "$container — image: $IMAGE"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would pull: $IMAGE"
        UPDATED+=("$container")
        continue
    fi

    OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")

    log "$ICON_SYNC  Pulling $IMAGE..."
    if [[ "$ENABLE_LOGGING" == "true" ]]; then
        docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/  /'
        _pull_rc=${PIPESTATUS[0]}
    else
        docker pull "$IMAGE" >/dev/null 2>&1
        _pull_rc=$?
    fi
    NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")

    if [[ $_pull_rc -eq 0 ]]; then
        if [[ -n "$OLD_ID" ]] && [[ "$OLD_ID" != "$NEW_ID" ]]; then
            log "$ICON_DONE  $container — updated ✅"
            UPDATED+=("$container")
        else
            log "$container — already up to date"
            UP_TO_DATE+=("$container")
        fi
    else
        warn "$container — pull failed ($IMAGE)"
        FAILED+=("$container")
    fi

done

# ==============================================================================================
# ━━━ Restart Updated Containers ━━━
# ==============================================================================================
RESTARTED=()
RESTART_FAILED=()
SKIPPED_STOPPED=()

if [[ ${#UPDATED[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_CONTAINERS Restarting Updated Containers — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
    log "$ICON_CONTAINERS Containers with new image: ${UPDATED[*]}"

    for container in "${UPDATED[@]}"; do
        [[ -z "$container" ]] && continue
        log "━━━ $ICON_CONTAINERS $container ━━━"

        STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)

        if [[ "$STATUS" != "true" ]]; then
            log "$ICON_NOT_RUNNING $container is stopped — skipping restart (respecting stopped state)"
            SKIPPED_STOPPED+=("$container")
            continue
        fi

        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would restart $container"
            RESTARTED+=("$container")
            continue
        fi

        log "$ICON_RUNNING $container — recreating from template on new image..."
        if /usr/local/emhttp/plugins/dynamix.docker.manager/scripts/rebuild_container "$container" >/dev/null 2>&1; then
            if verify_running "$container"; then
                log "$ICON_DONE  $container recreated and running ✅"
                RESTARTED+=("$container")
            else
                error "$container recreated but not running — may be intentionally stopped"
                RESTARTED+=("$container")
            fi
        else
            error "Failed to rebuild $container from template"
            notify "$container failed to rebuild after update on $(hostname)" "Docker Update Remaining" "warning"
            RESTART_FAILED+=("$container")
        fi
    done
else
    log "No containers received a new image — nothing to restart"
fi

# ==============================================================================================
# ━━━ Prune Old Images ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Pruning Dangling Images — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would prune dangling images"
    PRUNED_SUMMARY="(dry run)"
else
    PRUNED_OUTPUT=$(docker image prune -f 2>&1)
    [[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/  /'
    PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
fi

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINING) SUMMARY ━━━━━"
echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME       Duration:    $(format_duration $(( END - START )))"
echo "$ICON_CONTAINERS Scope:       ${#ALL_RUNNING[@]} running — ${#EXCLUDED[@]} managed = ${#REMAINING[@]} checked"
if [[ ${#UPDATED[@]} -gt 0 ]]; then
    echo "$ICON_DONE       New image:   ${#UPDATED[@]}"
    log "  ${UPDATED[*]}"
fi
[[ ${#UP_TO_DATE[@]} -gt 0 ]]    && log "$ICON_RUNNING    Up to date:  ${#UP_TO_DATE[@]}"
[[ ${#FAILED[@]} -gt 0 ]]        && echo "$ICON_ERROR      Pull failed: ${FAILED[*]}"
if [[ ${#RESTARTED[@]} -gt 0 ]]; then
    echo "$ICON_DONE       Restarted:   ${#RESTARTED[@]}"
    log "  ${RESTARTED[*]}"
fi
[[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && log "$ICON_WARN     Not running: ${#SKIPPED_STOPPED[@]} (skipped restart)"
[[ ${#RESTART_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR     Restart fail:${RESTART_FAILED[*]}"
echo "$ICON_SYNC       Pruned:      ${PRUNED_SUMMARY:-none}"

ALL_FAILED=$(( ${#FAILED[@]} + ${#RESTART_FAILED[@]} ))
if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ "$ALL_FAILED" -eq 0 ]]; then
    echo "$ICON_DONE Status: done ✅ — ${#RESTARTED[@]} restarted, ${#UP_TO_DATE[@]} current"
else
    warn "Status: $ALL_FAILED error(s) — ${#FAILED[@]} pull failure(s), ${#RESTART_FAILED[@]} restart failure(s)"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$ALL_FAILED" -gt 0 ]] && exit 1
exit 0