#!/bin/bash # ============================================================================================== # ============================= Docker Update — Remaining ====================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Weekly sweep that pulls the latest image for every running container not # already covered by the daily or weekly managed update cycles. Restarts # containers that received a new image, then prunes dangling images. # # Called by weekly_sync_maintenance.sh as the final step in the weekly window. # Derives its target list automatically from docker ps minus the two managed # lists — there is nothing to configure for this script. # # Together with docker_update.sh (normal + remainder modes), every deployed # container receives at least one image pull per week without any per-container # configuration required here. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Lock Acquisition # Prevents concurrent execution via acquire_lock(). Safe to call from # weekly maintenance scripts without risk of overlap. # # Host Detection # detect_hosts() identifies which server is running the script and aliases # HOST*_DAILY_RESTART_CONTAINERS and HOST*_WEEKLY_RESTART_CONTAINERS to # the correct host's values for exclusion. # # Root Enforcement # Docker operations require root privileges. # # WEEKLY_REMAINING_UPDATES Toggle # Exits cleanly when disabled via master.conf. # # Running-Only Filter # Stopped containers excluded — intentionally down, pulling adds no value. # # Image ID Comparison # Containers not restarted unless their image actually changed. # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # master.conf # # WEEKLY_REMAINING_UPDATES # Enable or disable this script. (default: true) # To disable without the toggle: remove from WEEKLY_MAINTENANCE_SCRIPTS. # # host*.conf # # HOST*_DAILY_RESTART_CONTAINERS # Excluded from this script — already updated daily. Aliased by # detect_hosts() → DAILY_RESTART_CONTAINERS # # HOST*_WEEKLY_RESTART_CONTAINERS # Excluded from this script — already updated by weekly sync window. # Aliased by detect_hosts() → WEEKLY_RESTART_CONTAINERS # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # docker_update_remaining.sh # Pull all remaining running containers, restart those updated, prune images # # docker_update_remaining.sh --dry-run # Preview which containers would be pulled and restarted # # docker_update_remaining.sh --status # Show exclusion lists and current remaining container count # # docker_update_remaining.sh --log # Verbose per-container pull and restart output # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi acquire_lock if ! command -v docker &>/dev/null; then error "Docker command not found" exit 1 fi detect_hosts if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then echo "WEEKLY_REMAINING_UPDATES=false — skipping remaining container updates" exit 0 fi # ── Build exclusion set from daily + weekly managed lists ───────────────────────────────────── declare -A EXCLUDED for c in "${DAILY_RESTART_CONTAINERS[@]}" "${WEEKLY_RESTART_CONTAINERS[@]}"; do [[ -n "$c" ]] && EXCLUDED["$c"]=1 done # ── Get all running containers ──────────────────────────────────────────────────────────────── mapfile -t ALL_RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort) # ── Derive remainder: running minus excluded ────────────────────────────────────────────────── REMAINING=() for c in "${ALL_RUNNING[@]}"; do [[ -z "$c" ]] && continue [[ -n "${EXCLUDED[$c]:-}" ]] && continue REMAINING+=("$c") done # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_GEAR Enabled: ${WEEKLY_REMAINING_UPDATES:-true}" echo "$ICON_CONTAINERS All running: ${#ALL_RUNNING[@]}" echo "$ICON_CONTAINERS Excluded: ${!EXCLUDED[*]}" echo "$ICON_CONTAINERS Remaining: ${REMAINING[*]:-none}" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi if [[ ${#REMAINING[@]} -eq 0 ]]; then echo "No remaining containers to update — all running containers are covered by daily/weekly lists" exit 0 fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled or containers restarted" # ============================================================================================== # ── FUNCTIONS ───────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # docker_cmd, retry_docker, verify_running — defined in common.sh # ============================================================================================== # ━━━ Pull Updates ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_CONTAINERS Docker Update (Remaining) — $(date '+%Y-%m-%d %H:%M:%S') ━━━" log "$ICON_CONTAINERS Containers: ${REMAINING[*]}" log "$ICON_CONTAINERS Excluded (managed elsewhere): ${!EXCLUDED[*]}" echo "" START=$(date +%s) UPDATED=() UP_TO_DATE=() FAILED=() for container in "${REMAINING[@]}"; do [[ -z "$container" ]] && continue log "━━━ $ICON_CONTAINERS $container ━━━" IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null) if [[ -z "$IMAGE" ]]; then warn "$container — could not determine image, skipping" FAILED+=("$container") continue fi log "$container — image: $IMAGE" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would pull: $IMAGE" UPDATED+=("$container") continue fi OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "") log "$ICON_SYNC Pulling $IMAGE..." if [[ "$ENABLE_LOGGING" == "true" ]]; then docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /' _pull_rc=${PIPESTATUS[0]} else docker pull "$IMAGE" >/dev/null 2>&1 _pull_rc=$? fi NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "") if [[ $_pull_rc -eq 0 ]]; then if [[ -n "$OLD_ID" ]] && [[ "$OLD_ID" != "$NEW_ID" ]]; then log "$ICON_DONE $container — updated ✅ (${OLD_ID:7:12} → ${NEW_ID:7:12})" UPDATED+=("$container") else log "$container — already up to date (${NEW_ID:7:12})" UP_TO_DATE+=("$container") fi else warn "$container — pull failed ($IMAGE)" FAILED+=("$container") fi done # ============================================================================================== # ━━━ Restart Updated Containers ━━━ # ============================================================================================== RESTARTED=() RESTART_FAILED=() SKIPPED_STOPPED=() if [[ ${#UPDATED[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_CONTAINERS Restarting Updated Containers — $(date '+%Y-%m-%d %H:%M:%S') ━━━" log "$ICON_CONTAINERS Containers with new image: ${UPDATED[*]}" for container in "${UPDATED[@]}"; do [[ -z "$container" ]] && continue log "━━━ $ICON_CONTAINERS $container ━━━" STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) if [[ "$STATUS" != "true" ]]; then log "$ICON_NOT_RUNNING $container is stopped — skipping restart (respecting stopped state)" SKIPPED_STOPPED+=("$container") continue fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart $container" RESTARTED+=("$container") continue fi log "$ICON_RUNNING $container — recreating from template on new image..." if /usr/local/emhttp/plugins/dynamix.docker.manager/scripts/rebuild_container "$container" >/dev/null 2>&1; then if verify_running "$container"; then log "$ICON_DONE $container recreated and running ✅" RESTARTED+=("$container") else error "$container recreated but not running — may be intentionally stopped" RESTARTED+=("$container") fi else error "Failed to rebuild $container from template" notify "$container failed to rebuild after update on $(hostname)" "Docker Update Remaining" "warning" RESTART_FAILED+=("$container") fi done else log "No containers received a new image — nothing to restart" fi # ============================================================================================== # ━━━ Prune Old Images ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_SYNC Pruning Dangling Images — $(date '+%Y-%m-%d %H:%M:%S') ━━━" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would prune dangling images" PRUNED_SUMMARY="(dry run)" else PRUNED_OUTPUT=$(docker image prune -f 2>&1) [[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /' PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed") fi END=$(date +%s) # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINING) SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_CONTAINERS Scope: ${#ALL_RUNNING[@]} running — ${#EXCLUDED[@]} managed = ${#REMAINING[@]} checked" if [[ ${#UPDATED[@]} -gt 0 ]]; then echo "$ICON_DONE New image: ${#UPDATED[@]}" log " ${UPDATED[*]}" fi [[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}" [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Pull failed: ${FAILED[*]}" if [[ ${#RESTARTED[@]} -gt 0 ]]; then echo "$ICON_DONE Restarted: ${#RESTARTED[@]}" log " ${RESTARTED[*]}" fi [[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && log "$ICON_WARN Not running: ${SKIPPED_STOPPED[*]} (skipped restart)" [[ ${#RESTART_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Restart fail:${RESTART_FAILED[*]}" echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}" ALL_FAILED=$(( ${#FAILED[@]} + ${#RESTART_FAILED[@]} )) if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ "$ALL_FAILED" -eq 0 ]]; then echo "$ICON_DONE Status: done ✅ — ${#RESTARTED[@]} restarted, ${#UP_TO_DATE[@]} current" else warn "Status: $ALL_FAILED error(s) — ${#FAILED[@]} pull failure(s), ${#RESTART_FAILED[@]} restart failure(s)" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" [[ "$ALL_FAILED" -gt 0 ]] && exit 1 exit 0