#!/bin/bash # ============================================================================================== # ============================= Docker Update — Remaining ====================================== # ============================================================================================== # Pulls the latest image for every running container NOT already covered by the daily or # weekly update/restart cycles. Restarts containers that received a new image, then prunes # dangling images. Runs at the end of the weekly maintenance window. # # ── WHAT THIS COVERS ────────────────────────────────────────────────────────────────────────── # Daily update: DAILY_RESTART_CONTAINERS — auth stack, NPM, Dispatcharr, etc. # Weekly update: WEEKLY_RESTART_CONTAINERS — NextCloud, AdGuard, Immich, etc. # This script: everything else running on the system (media stack, utilities, etc.) # # Together the three scripts ensure every deployed container receives at least one image # pull per week, with no container list to maintain here — it derives the remainder # automatically from `docker ps` minus the two managed lists. # # ── WHAT THIS DOES ──────────────────────────────────────────────────────────────────────────── # 1. Pull latest image for each remaining running container # 2. Restart containers whose image ID changed (new update landed) # 3. Prune dangling images left behind by the updates # Containers already up to date are not restarted. # # ── EXCLUSION LOGIC ─────────────────────────────────────────────────────────────────────────── # Exclusion set = DAILY_RESTART_CONTAINERS + WEEKLY_RESTART_CONTAINERS (aliased by detect_hosts) # Only running containers are targeted — stopped containers are intentionally excluded # (stopped = likely paused intentionally; pulling while stopped adds no value). # # ── TOGGLE ──────────────────────────────────────────────────────────────────────────────────── # WEEKLY_REMAINING_UPDATES=false in master.conf — skips all pulls, exits cleanly # # ── CONFIGURATION ───────────────────────────────────────────────────────────────────────────── # master.conf: WEEKLY_REMAINING_UPDATES — enable/disable (default: true) # master_host*.conf: HOST*_DAILY_RESTART_CONTAINERS — excluded from this script # HOST*_WEEKLY_RESTART_CONTAINERS — excluded from this script # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # docker_update_remaining.sh — normal run # docker_update_remaining.sh --dry-run — show which containers would be pulled/restarted # docker_update_remaining.sh --log — verbose output # docker_update_remaining.sh --status — show config and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi if ! command -v docker &>/dev/null; then error "Docker command not found" exit 1 fi detect_hosts if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then log "WEEKLY_REMAINING_UPDATES=false — skipping remaining container updates" exit 0 fi # ── Build exclusion set from daily + weekly managed lists ───────────────────────────────────── declare -A EXCLUDED for c in "${DAILY_RESTART_CONTAINERS[@]}" "${WEEKLY_RESTART_CONTAINERS[@]}"; do [[ -n "$c" ]] && EXCLUDED["$c"]=1 done # ── Get all running containers ──────────────────────────────────────────────────────────────── mapfile -t ALL_RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort) # ── Derive remainder: running minus excluded ────────────────────────────────────────────────── REMAINING=() for c in "${ALL_RUNNING[@]}"; do [[ -z "$c" ]] && continue [[ -n "${EXCLUDED[$c]:-}" ]] && continue REMAINING+=("$c") done # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_GEAR Enabled: ${WEEKLY_REMAINING_UPDATES:-true}" echo "$ICON_CONTAINERS All running: ${#ALL_RUNNING[@]}" echo "$ICON_CONTAINERS Excluded: ${!EXCLUDED[*]}" echo "$ICON_CONTAINERS Remaining: ${REMAINING[*]:-none}" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi if [[ ${#REMAINING[@]} -eq 0 ]]; then log "No remaining containers to update — all running containers are covered by daily/weekly lists" exit 0 fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled or containers restarted" # ============================================================================================== # ── FUNCTIONS ───────────────────────────────────────────────────────────────────────────────── # ============================================================================================== DOCKER_TIMEOUT=30 docker_cmd() { timeout "$DOCKER_TIMEOUT" "$@" local exit_code=$? if [[ "$exit_code" -eq 124 ]]; then error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*" return 1 fi return "$exit_code" } retry_docker() { local attempt=1 while [[ "$attempt" -le "$RETRY_COUNT" ]]; do log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*" if docker_cmd "$@"; then log "Succeeded on attempt $attempt" return 0 else warn "Attempt $attempt failed" (( attempt++ )) [[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP" fi done error "Command failed after $RETRY_COUNT attempts: $*" return 1 } RESTART_VERIFY_WAIT=5 verify_running() { local container="$1" sleep "$RESTART_VERIFY_WAIT" local state state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) [[ "$state" == "true" ]] } # ============================================================================================== # ━━━ Pull Updates ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_CONTAINERS Docker Update (Remaining) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_CONTAINERS Containers: ${REMAINING[*]}" echo "$ICON_CONTAINERS Excluded (managed elsewhere): ${!EXCLUDED[*]}" echo "" START=$(date +%s) UPDATED=() UP_TO_DATE=() FAILED=() for container in "${REMAINING[@]}"; do [[ -z "$container" ]] && continue echo "━━━ $ICON_CONTAINERS $container ━━━" IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null) if [[ -z "$IMAGE" ]]; then warn "$container — could not determine image, skipping" FAILED+=("$container") echo "" continue fi log "$container — image: $IMAGE" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would pull: $IMAGE" UPDATED+=("$container") echo "" continue fi OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "") echo "$ICON_SYNC Pulling $IMAGE..." if docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /'; then NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "") if [[ -n "$OLD_ID" ]] && [[ "$OLD_ID" != "$NEW_ID" ]]; then echo "$ICON_DONE $container — updated ✅" UPDATED+=("$container") else log "$container — already up to date" UP_TO_DATE+=("$container") fi else warn "$container — pull failed ($IMAGE)" FAILED+=("$container") fi echo "" done # ============================================================================================== # ━━━ Restart Updated Containers ━━━ # ============================================================================================== RESTARTED=() RESTART_FAILED=() SKIPPED_STOPPED=() if [[ ${#UPDATED[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_CONTAINERS Restarting Updated Containers — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_CONTAINERS Containers with new image: ${UPDATED[*]}" echo "" for container in "${UPDATED[@]}"; do [[ -z "$container" ]] && continue echo "━━━ $ICON_CONTAINERS $container ━━━" STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) if [[ "$STATUS" != "true" ]]; then echo "$ICON_NOT_RUNNING $container is stopped — skipping restart (respecting stopped state)" SKIPPED_STOPPED+=("$container") echo "" continue fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart $container" RESTARTED+=("$container") echo "" continue fi echo "$ICON_RUNNING $container is running — restarting on new image..." if retry_docker docker restart "$container"; then if verify_running "$container"; then echo "$ICON_DONE $container restarted and running ✅" RESTARTED+=("$container") else error "$container restarted but crashed immediately" notify "$container crashed after update-restart on $(hostname)" "Docker Update Remaining" "warning" RESTART_FAILED+=("$container") fi else error "Failed to restart $container after $RETRY_COUNT attempts" notify "$container failed to restart after update on $(hostname)" "Docker Update Remaining" "warning" RESTART_FAILED+=("$container") fi echo "" done else log "No containers received a new image — nothing to restart" fi # ============================================================================================== # ━━━ Prune Old Images ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_SYNC Pruning Dangling Images — $(date '+%Y-%m-%d %H:%M:%S') ━━━" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would prune dangling images" PRUNED_SUMMARY="(dry run)" else PRUNED_OUTPUT=$(docker image prune -f 2>&1) echo "$PRUNED_OUTPUT" | sed 's/^/ /' PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed") fi END=$(date +%s) # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINING) SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_CONTAINERS Scope: ${#ALL_RUNNING[@]} running — ${#EXCLUDED[@]} managed = ${#REMAINING[@]} checked" [[ ${#UPDATED[@]} -gt 0 ]] && echo "$ICON_DONE New image: ${UPDATED[*]}" [[ ${#UP_TO_DATE[@]} -gt 0 ]] && echo "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}" [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Pull failed: ${FAILED[*]}" [[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_DONE Restarted: ${RESTARTED[*]}" [[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && echo "$ICON_WARN Not running: ${SKIPPED_STOPPED[*]} (skipped restart)" [[ ${#RESTART_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Restart fail:${RESTART_FAILED[*]}" echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}" ALL_FAILED=$(( ${#FAILED[@]} + ${#RESTART_FAILED[@]} )) if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ "$ALL_FAILED" -eq 0 ]]; then log "$ICON_DONE Status: done ✅ — ${#RESTARTED[@]} restarted, ${#UP_TO_DATE[@]} current" else warn "Status: $ALL_FAILED error(s) — ${#FAILED[@]} pull failure(s), ${#RESTART_FAILED[@]} restart failure(s)" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" [[ "$ALL_FAILED" -gt 0 ]] && exit 1 exit 0