Varaverk/Docker_Essentials/docker_update_remaining.sh

#!/bin/bash
# ==============================================================================================
# ============================= Docker Update — Remaining ======================================
# ==============================================================================================
# Pulls the latest image for every running container NOT already covered by the daily or
# weekly update/restart cycles. Restarts containers that received a new image, then prunes
# dangling images. Runs at the end of the weekly maintenance window.
#
# ── WHAT THIS COVERS ──────────────────────────────────────────────────────────────────────────
#   Daily update:   DAILY_RESTART_CONTAINERS  — auth stack, NPM, Dispatcharr, etc.
#   Weekly update:  WEEKLY_RESTART_CONTAINERS — NextCloud, AdGuard, Immich, etc.
#   This script:    everything else running on the system (media stack, utilities, etc.)
#
# Together the three scripts ensure every deployed container receives at least one image
# pull per week, with no container list to maintain here — it derives the remainder
# automatically from `docker ps` minus the two managed lists.
#
# ── WHAT THIS DOES ────────────────────────────────────────────────────────────────────────────
#   1. Pull latest image for each remaining running container
#   2. Restart containers whose image ID changed (new update landed)
#   3. Prune dangling images left behind by the updates
#   Containers already up to date are not restarted.
#
# ── EXCLUSION LOGIC ───────────────────────────────────────────────────────────────────────────
# Exclusion set = DAILY_RESTART_CONTAINERS + WEEKLY_RESTART_CONTAINERS (aliased by detect_hosts)
# Only running containers are targeted — stopped containers are intentionally excluded
# (stopped = likely paused intentionally; pulling while stopped adds no value).
#
# ── TOGGLE ────────────────────────────────────────────────────────────────────────────────────
#   WEEKLY_REMAINING_UPDATES=false in master.conf — skips all pulls, exits cleanly
#
# ── CONFIGURATION ─────────────────────────────────────────────────────────────────────────────
#   master.conf:         WEEKLY_REMAINING_UPDATES — enable/disable (default: true)
#   master_host*.conf:   HOST*_DAILY_RESTART_CONTAINERS  — excluded from this script
#                        HOST*_WEEKLY_RESTART_CONTAINERS — excluded from this script
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   docker_update_remaining.sh              — normal run
#   docker_update_remaining.sh --dry-run    — show which containers would be pulled/restarted
#   docker_update_remaining.sh --log        — verbose output
#   docker_update_remaining.sh --status     — show config and exit
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts

if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then
    log "WEEKLY_REMAINING_UPDATES=false — skipping remaining container updates"
    exit 0
fi

# ── Build exclusion set from daily + weekly managed lists ─────────────────────────────────────
declare -A EXCLUDED
for c in "${DAILY_RESTART_CONTAINERS[@]}"  "${WEEKLY_RESTART_CONTAINERS[@]}"; do
    [[ -n "$c" ]] && EXCLUDED["$c"]=1
done

# ── Get all running containers ────────────────────────────────────────────────────────────────
mapfile -t ALL_RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort)

# ── Derive remainder: running minus excluded ──────────────────────────────────────────────────
REMAINING=()
for c in "${ALL_RUNNING[@]}"; do
    [[ -z "$c" ]] && continue
    [[ -n "${EXCLUDED[$c]:-}" ]] && continue
    REMAINING+=("$c")
done

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR       Enabled:     ${WEEKLY_REMAINING_UPDATES:-true}"
    echo "$ICON_CONTAINERS All running: ${#ALL_RUNNING[@]}"
    echo "$ICON_CONTAINERS Excluded:    ${!EXCLUDED[*]}"
    echo "$ICON_CONTAINERS Remaining:   ${REMAINING[*]:-none}"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

if [[ ${#REMAINING[@]} -eq 0 ]]; then
    log "No remaining containers to update — all running containers are covered by daily/weekly lists"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled or containers restarted"

# ==============================================================================================
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

DOCKER_TIMEOUT=30
docker_cmd() {
    timeout "$DOCKER_TIMEOUT" "$@"
    local exit_code=$?
    if [[ "$exit_code" -eq 124 ]]; then
        error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*"
        return 1
    fi
    return "$exit_code"
}

retry_docker() {
    local attempt=1
    while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
        log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
        if docker_cmd "$@"; then
            log "Succeeded on attempt $attempt"
            return 0
        else
            warn "Attempt $attempt failed"
            (( attempt++ ))
            [[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP"
        fi
    done
    error "Command failed after $RETRY_COUNT attempts: $*"
    return 1
}

RESTART_VERIFY_WAIT=5
verify_running() {
    local container="$1"
    sleep "$RESTART_VERIFY_WAIT"
    local state
    state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
    [[ "$state" == "true" ]]
}

# ==============================================================================================
# ━━━ Pull Updates ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Docker Update (Remaining) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_CONTAINERS Containers: ${REMAINING[*]}"
echo "$ICON_CONTAINERS Excluded (managed elsewhere): ${!EXCLUDED[*]}"
echo ""

START=$(date +%s)
UPDATED=()
UP_TO_DATE=()
FAILED=()

for container in "${REMAINING[@]}"; do
    [[ -z "$container" ]] && continue
    echo "━━━ $ICON_CONTAINERS $container ━━━"

    IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
    if [[ -z "$IMAGE" ]]; then
        warn "$container — could not determine image, skipping"
        FAILED+=("$container")
        echo ""
        continue
    fi

    log "$container — image: $IMAGE"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would pull: $IMAGE"
        UPDATED+=("$container")
        echo ""
        continue
    fi

    OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")

    echo "$ICON_SYNC  Pulling $IMAGE..."
    if docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/  /'; then
        NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")

        if [[ -n "$OLD_ID" ]] && [[ "$OLD_ID" != "$NEW_ID" ]]; then
            echo "$ICON_DONE  $container — updated ✅"
            UPDATED+=("$container")
        else
            log "$container — already up to date"
            UP_TO_DATE+=("$container")
        fi
    else
        warn "$container — pull failed ($IMAGE)"
        FAILED+=("$container")
    fi

    echo ""
done

# ==============================================================================================
# ━━━ Restart Updated Containers ━━━
# ==============================================================================================
RESTARTED=()
RESTART_FAILED=()
SKIPPED_STOPPED=()

if [[ ${#UPDATED[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_CONTAINERS Restarting Updated Containers — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
    echo "$ICON_CONTAINERS Containers with new image: ${UPDATED[*]}"
    echo ""

    for container in "${UPDATED[@]}"; do
        [[ -z "$container" ]] && continue
        echo "━━━ $ICON_CONTAINERS $container ━━━"

        STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)

        if [[ "$STATUS" != "true" ]]; then
            echo "$ICON_NOT_RUNNING $container is stopped — skipping restart (respecting stopped state)"
            SKIPPED_STOPPED+=("$container")
            echo ""
            continue
        fi

        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would restart $container"
            RESTARTED+=("$container")
            echo ""
            continue
        fi

        echo "$ICON_RUNNING $container is running — restarting on new image..."
        if retry_docker docker restart "$container"; then
            if verify_running "$container"; then
                echo "$ICON_DONE  $container restarted and running ✅"
                RESTARTED+=("$container")
            else
                error "$container restarted but crashed immediately"
                notify "$container crashed after update-restart on $(hostname)" "Docker Update Remaining" "warning"
                RESTART_FAILED+=("$container")
            fi
        else
            error "Failed to restart $container after $RETRY_COUNT attempts"
            notify "$container failed to restart after update on $(hostname)" "Docker Update Remaining" "warning"
            RESTART_FAILED+=("$container")
        fi
        echo ""
    done
else
    log "No containers received a new image — nothing to restart"
fi

# ==============================================================================================
# ━━━ Prune Old Images ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Pruning Dangling Images — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would prune dangling images"
    PRUNED_SUMMARY="(dry run)"
else
    PRUNED_OUTPUT=$(docker image prune -f 2>&1)
    echo "$PRUNED_OUTPUT" | sed 's/^/  /'
    PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
fi

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINING) SUMMARY ━━━━━"
echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME       Duration:    $(format_duration $(( END - START )))"
echo "$ICON_CONTAINERS Scope:       ${#ALL_RUNNING[@]} running — ${#EXCLUDED[@]} managed = ${#REMAINING[@]} checked"
[[ ${#UPDATED[@]} -gt 0 ]]       && echo "$ICON_DONE       New image:   ${UPDATED[*]}"
[[ ${#UP_TO_DATE[@]} -gt 0 ]]    && echo "$ICON_RUNNING    Up to date:  ${UP_TO_DATE[*]}"
[[ ${#FAILED[@]} -gt 0 ]]        && echo "$ICON_ERROR      Pull failed: ${FAILED[*]}"
[[ ${#RESTARTED[@]} -gt 0 ]]     && echo "$ICON_DONE       Restarted:   ${RESTARTED[*]}"
[[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && echo "$ICON_WARN     Not running: ${SKIPPED_STOPPED[*]} (skipped restart)"
[[ ${#RESTART_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR     Restart fail:${RESTART_FAILED[*]}"
echo "$ICON_SYNC       Pruned:      ${PRUNED_SUMMARY:-none}"

ALL_FAILED=$(( ${#FAILED[@]} + ${#RESTART_FAILED[@]} ))
if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ "$ALL_FAILED" -eq 0 ]]; then
    log "$ICON_DONE Status: done ✅ — ${#RESTARTED[@]} restarted, ${#UP_TO_DATE[@]} current"
else
    warn "Status: $ALL_FAILED error(s) — ${#FAILED[@]} pull failure(s), ${#RESTART_FAILED[@]} restart failure(s)"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$ALL_FAILED" -gt 0 ]] && exit 1
exit 0