#!/bin/bash # ============================================================================================== # ================================= Docker Update ============================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Pulls the latest images for configured containers. Three modes: # # Normal mode — called by daily_sync_maintenance.sh before docker_daily_restart.sh. # Containers stay running during the pull — no extra downtime beyond what the # nightly restart already causes. # # Weekly mode — called by weekly_sync_maintenance.sh via WEEKLY_MAINTENANCE_SCRIPTS # before docker_weekly_restart.sh. Pulls WEEKLY_RESTART_CONTAINERS so the # subsequent restart lands on the fresh image. # # Remainder mode — called by monthly_maintenance.sh. Catches everything not # already owned by daily or weekly — derived from docker ps, nothing to configure. # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # Normal mode (daily): # Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh. # Pull → compare old vs new image ID → mark updated or already current. # docker_daily_restart.sh runs after — containers restart onto the fresh image. # # Weekly mode: # Targets WEEKLY_RESTART_CONTAINERS — same list used by docker_weekly_restart.sh. # Pull → compare → rebuild if changed. # docker_weekly_restart.sh runs after — containers restart onto the fresh image. # # Remainder mode (monthly): # Targets all currently running containers NOT in: # DAILY_RESTART_CONTAINERS — already updated daily # WEEKLY_RESTART_CONTAINERS — already updated weekly # emby + critical-data profiles — updated inline by the weekly sync window # FALLBACK_*_TIER* — owned by the remote server's update cycle # Pull → compare → prune dangling images. # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Single List # Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a # separate update list. Adding or removing a container from the restart list # automatically updates the image pull list — one change, both places. # # Version Ownership # Fallback containers are excluded from remainder mode. This server only runs # them during a fallback. The remote server owns their version — if remainder # updates them independently and a handback occurs, the remote's older image # may not handle data written by the newer version. # # State Respect # Stopped containers are never targeted. Pulling while stopped adds no value # and a stopped container was likely halted intentionally. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Lock Acquisition # Prevents concurrent execution via acquire_lock(). Safe to call from # maintenance scripts without risk of overlap. # # Host Detection # detect_hosts() identifies which server is running the script and aliases # HOST*_DAILY_RESTART_CONTAINERS to the correct host's values. # # Root Enforcement # Docker operations require root privileges. # # Docker Presence Check # Verifies the docker binary exists before execution. # # Docker Daemon Check # Verifies the daemon is responsive before container discovery. Remainder mode # derives its entire target list from docker ps — against a hung daemon that # returns empty and the run silently reports "no containers to update". # # Timeout Protection # Inspect, discovery and image-query commands are wrapped in a timeout so a # hung daemon cannot stall the maintenance window. docker pull is deliberately # NOT wrapped — a large image legitimately takes longer than any sane timeout, # and killing it mid-layer wastes the transfer. # # Empty List Guards # Each mode exits cleanly with a pointer to the relevant conf key when its # container list is unconfigured for this host. # # Rebuild Failure Fallback # A container that fails to rebuild is excluded from the rebuilt-list handoff # file, so the follow-up restart script still gives it a normal restart pass. # # DAILY_CONTAINER_UPDATES / WEEKLY_CONTAINER_UPDATES Toggles # Each mode exits cleanly when disabled. Restart scripts run regardless — # update and restart are independent operations. # # Fallback Exclusion # Remainder mode excludes containers owned by the remote server's update cycle # to prevent version divergence across the fallback boundary. # # Running-Only Filter # Stopped containers excluded from remainder mode — intentionally down. # # Image ID Comparison # Containers not restarted unless their image actually changed. Pulls that # result in "already up to date" produce no restart. # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # master.conf # # DAILY_CONTAINER_UPDATES # Enable or disable normal mode. docker_daily_restart.sh runs regardless. # (default: true) # # WEEKLY_CONTAINER_UPDATES # Enable or disable weekly mode. docker_weekly_restart.sh runs regardless. # (default: true) # # MONTHLY_REMAINING_UPDATES # Enable or disable remainder mode. (default: true) # # DOCKER_UPDATE_REBUILT_DAILY_FILE / DOCKER_UPDATE_REBUILT_WEEKLY_FILE # Written after each normal/weekly run with the containers actually rebuilt # this pass — read by docker_daily_restart.sh / docker_weekly_restart.sh so # they skip restarting a container a second time right after this script # already rebuilt it onto the new image. Not written in remainder mode # (no follow-up restart script exists for it). # # PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data] # Container names for emby and critical-data profiles — excluded from # remainder mode (already updated by the weekly sync window) # # host*.conf # # HOST*_DAILY_RESTART_CONTAINERS # Containers updated in normal mode. Aliased by detect_hosts() → # DAILY_RESTART_CONTAINERS # # HOST*_WEEKLY_RESTART_CONTAINERS # Containers updated in weekly mode. Aliased by detect_hosts() → # WEEKLY_RESTART_CONTAINERS # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # docker_update.sh # Normal mode — pull latest images for DAILY_RESTART_CONTAINERS # Called by daily_sync_maintenance.sh before docker_daily_restart.sh # # docker_update.sh --weekly # Weekly mode — pull latest images for WEEKLY_RESTART_CONTAINERS # Called by weekly_sync_maintenance.sh (WEEKLY_MAINTENANCE_SCRIPTS) before docker_weekly_restart.sh # # docker_update.sh --remainder # Remainder mode — pull all running containers not in managed lists, # restart those that received updates, prune dangling images # Called by monthly_maintenance.sh # # docker_update.sh --dry-run # Preview which containers would be pulled without making changes # # docker_update.sh --status # Show configuration and container list for current mode # # docker_update.sh --log # Verbose per-container pull and comparison output # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" # Pre-parse mode flags before parse_args (unknown args pass through to PARSED_ARGS) REMAINDER_MODE=false WEEKLY_MODE=false _filtered_args=() for _arg in "$@"; do case "$_arg" in --remainder) REMAINDER_MODE=true ;; --weekly) WEEKLY_MODE=true ;; *) _filtered_args+=("$_arg") ;; esac done unset _arg parse_args "${_filtered_args[@]}" unset _filtered_args # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi acquire_lock if ! command -v docker &>/dev/null; then error "Docker command not found" exit 1 fi detect_hosts # Remainder mode builds its whole target list from docker ps — a hung daemon returns # empty and the run would report "no containers to update" instead of failing. if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then error "Docker daemon not responding — skipping image updates" notify "Docker update skipped on $(hostname) — Docker daemon not responding" "Docker Update" "warning" exit 1 fi # ============================================================================================== # ━━━ Container Discovery ━━━ # ============================================================================================== if [[ "$REMAINDER_MODE" == true ]]; then if [[ "${MONTHLY_REMAINING_UPDATES:-true}" != "true" ]]; then echo "MONTHLY_REMAINING_UPDATES=false — skipping remainder container updates" exit 0 fi declare -A _exclude=() # Daily containers — updated by docker_update.sh normal mode for _c in "${DAILY_RESTART_CONTAINERS[@]}"; do [[ -n "$_c" ]] && _exclude["$_c"]=1 done # Weekly containers — updated by docker_update.sh --weekly for _c in "${WEEKLY_RESTART_CONTAINERS[@]}"; do [[ -n "$_c" ]] && _exclude["$_c"]=1 done # Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh _weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}" read -r -a _weekly_arr <<< "$_weekly_str" for _c in "${_weekly_arr[@]}"; do [[ -n "$_c" ]] && _exclude["$_c"]=1 done unset _weekly_str _weekly_arr # Fallback coverage containers — owned by the remote server's update cycle. # This server runs them during fallback but should never update them independently. # Updating them here risks version divergence: if remote's writeback after handback # encounters data written by a newer version, it may not handle it correctly. for _tier in 1 2 3 4; do _tier_var="FALLBACK_${REMOTE_ID}_TIER${_tier}" eval "_tier_arr=(\"\${${_tier_var}[@]:-}\")" 2>/dev/null for _c in "${_tier_arr[@]}"; do [[ -n "$_c" ]] && _exclude["$_c"]=1 done done unset _tier _tier_var _tier_arr _c mapfile -t _all_running < <(timeout "$DOCKER_TIMEOUT" docker ps --format '{{.Names}}' | sort) TARGET_CONTAINERS=() for _c in "${_all_running[@]}"; do [[ -z "${_exclude[$_c]+x}" ]] && TARGET_CONTAINERS+=("$_c") done unset _all_running _exclude _c elif [[ "$WEEKLY_MODE" == true ]]; then if [[ "${WEEKLY_CONTAINER_UPDATES:-true}" != "true" ]]; then echo "WEEKLY_CONTAINER_UPDATES=false — skipping weekly container updates" exit 0 fi if [[ ${#WEEKLY_RESTART_CONTAINERS[@]} -eq 0 ]]; then warn "WEEKLY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update" warn "Check HOST${MY_ID#HOST}_WEEKLY_RESTART_CONTAINERS in host*.conf" exit 0 fi TARGET_CONTAINERS=("${WEEKLY_RESTART_CONTAINERS[@]}") else if [[ "${DAILY_CONTAINER_UPDATES:-true}" != "true" ]]; then echo "DAILY_CONTAINER_UPDATES=false — skipping container updates" exit 0 fi if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update" warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf" exit 0 fi TARGET_CONTAINERS=("${DAILY_RESTART_CONTAINERS[@]}") fi # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" if [[ "$REMAINDER_MODE" == true ]]; then echo "$ICON_GEAR Mode: remainder (monthly)" echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly, sync-window, fallback)" for _c in "${TARGET_CONTAINERS[@]}"; do echo " $_c"; done elif [[ "$WEEKLY_MODE" == true ]]; then echo "$ICON_GEAR Mode: weekly" echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}" echo "$ICON_GEAR Enabled: ${WEEKLY_CONTAINER_UPDATES:-true}" else echo "$ICON_GEAR Mode: normal (daily)" echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}" echo "$ICON_GEAR Enabled: ${DAILY_CONTAINER_UPDATES:-true}" fi echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled" if [[ ${#TARGET_CONTAINERS[@]} -eq 0 ]]; then echo "No containers to update" exit 0 fi # ============================================================================================== # ━━━ Pull Updates ━━━ # ============================================================================================== echo "" if [[ "$REMAINDER_MODE" == true ]]; then echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily, weekly, or sync window)" log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}" elif [[ "$WEEKLY_MODE" == true ]]; then echo "━━━ $ICON_CONTAINERS Docker Update (weekly) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}" else echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━" log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}" fi echo "" START=$(date +%s) UPDATED=() UP_TO_DATE=() FAILED=() OLD_IMAGE_IDS=() # old image IDs to explicitly remove after rebuilds SKIPPED=() for container in "${TARGET_CONTAINERS[@]}"; do [[ -z "$container" ]] && continue log "━━━ $ICON_CONTAINERS $container ━━━" if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then warn "$container — not found, skipping" SKIPPED+=("$container") continue fi IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null) if [[ -z "$IMAGE" ]]; then warn "$container — could not determine image, skipping" SKIPPED+=("$container") continue fi log "$container — image: $IMAGE" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would pull: $IMAGE" UPDATED+=("$container") continue fi # Capture the image ID the container is currently running on, and the # image ID :latest points to before the pull. After pulling, we rebuild if # either a new digest landed OR the container is behind what :latest is now. CONTAINER_IMAGE_ID=$(timeout "$DOCKER_TIMEOUT" docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "") OLD_ID=$(timeout "$DOCKER_TIMEOUT" docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "") log "$ICON_SYNC Pulling $IMAGE..." if [[ "$ENABLE_LOGGING" == "true" ]]; then docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /' _pull_rc=${PIPESTATUS[0]} else docker pull "$IMAGE" >/dev/null 2>&1 _pull_rc=$? fi NEW_ID=$(timeout "$DOCKER_TIMEOUT" docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "") if [[ $_pull_rc -eq 0 ]]; then _pull_new=$( [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false) _container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false) if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then [[ "$_pull_new" == true ]] && log "$ICON_DONE $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})" [[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})" UPDATED+=("$container") OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID") else log "$container — up to date (${NEW_ID:7:12})" UP_TO_DATE+=("$container") fi else warn "$container — pull failed ($IMAGE)" FAILED+=("$container") fi done # ── Recreate containers that received a new image ──────────────────────────── # docker restart uses the image ID baked in at creation time — it never picks # up the new digest. rebuild_container reads the stored XML template, stops the # old container, recreates it (new image, same config), then prunes the old image. REBUILT=() REBUILD_FAILED=() if [[ ${#UPDATED[@]} -gt 0 ]]; then for container in "${UPDATED[@]}"; do [[ -z "$container" ]] && continue if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would rebuild $container from template" REBUILT+=("$container") continue fi log "$ICON_SYNC Rebuilding $container from template on new image..." if platform_rebuild_container "$container"; then echo "$ICON_DONE $container rebuilt ✅" REBUILT+=("$container") else _fallback_msg=$([[ "$WEEKLY_MODE" == true ]] && echo "docker_weekly_restart.sh" || ([[ "$REMAINDER_MODE" == true ]] && echo "no follow-up restart" || echo "docker_daily_restart.sh")) error "Failed to rebuild $container — $_fallback_msg" unset _fallback_msg notify "$container failed to rebuild after image update on $(hostname)" "Docker Update" "warning" REBUILD_FAILED+=("$container") fi done fi # Record successfully-rebuilt containers so the follow-up restart script (daily/weekly only — # remainder has no follow-up) can skip them instead of restarting an already-fresh container a # second time. Deliberately excludes REBUILD_FAILED — those still need the restart script's # normal pass as a fallback, exactly as the error message above promises. Written even when # REBUILT is empty, so a stale file from a previous run doesn't linger and get misread later. if [[ "$DRY_RUN" == false && "$REMAINDER_MODE" != true ]]; then _rebuilt_file="$DOCKER_UPDATE_REBUILT_DAILY_FILE" [[ "$WEEKLY_MODE" == true ]] && _rebuilt_file="$DOCKER_UPDATE_REBUILT_WEEKLY_FILE" if [[ -n "$_rebuilt_file" ]]; then printf '%s\n' "${REBUILT[@]}" > "$_rebuilt_file" 2>/dev/null fi unset _rebuilt_file fi # ── Remove old images ──────────────────────────────────────────────────────── # Explicitly rmi by the IDs captured before each pull. Tagged images are never # caught by dangling-only prune, so this is the only reliable cleanup path. # Fall through to dangling prune to catch any leftovers from other update paths. if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling" PRUNED_SUMMARY="(dry run)" else for _old_id in "${OLD_IMAGE_IDS[@]}"; do timeout "$DOCKER_TIMEOUT" docker rmi "$_old_id" >/dev/null 2>&1 || true done PRUNED_OUTPUT=$(timeout "$DOCKER_TIMEOUT" docker image prune -f 2>&1) [[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /' PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed") fi END=$(date +%s) # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== if [[ "$REMAINDER_MODE" == true ]]; then echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━" elif [[ "$WEEKLY_MODE" == true ]]; then echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (WEEKLY) SUMMARY ━━━━━" else echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE SUMMARY ━━━━━" fi echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" if [[ ${#UPDATED[@]} -gt 0 ]]; then echo "$ICON_DONE Updated: ${#UPDATED[@]}" log " ${UPDATED[*]}" fi [[ ${#REBUILT[@]} -gt 0 ]] && echo "$ICON_SYNC Rebuilt: ${#REBUILT[@]}" [[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Rebuild fail:${REBUILD_FAILED[*]}" [[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}" [[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_WARN Skipped: ${SKIPPED[*]}" [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no images pulled" elif [[ ${#FAILED[@]} -eq 0 ]]; then echo "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current" else warn "Status: ${#FAILED[@]} pull(s) failed — restart will proceed with existing images" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Pull failures are non-fatal — restart proceeds regardless exit 0