Fix container updates: rebuild on new image, consolidate update scripts
Weekly sync window was pulling images but using docker start after rsync, which ignores the new digest. Containers in the emby/critical-data profiles (Emby, Mariadb, Redis) never actually landed on updated images. docker_update_remaining.sh merged into docker_update.sh --remainder, which already had better exclusion logic. Added WEEKLY_REMAINING_UPDATES toggle and WEEKLY_RESTART_CONTAINERS exclusion to remainder mode. Onboard now runs docker_network_connect.sh on mirror before deploying stacks.
This commit is contained in:
Gmer4Lfe
@@ -29,6 +29,7 @@
|
||||
# Targets all currently running containers NOT in:
|
||||
# DAILY_RESTART_CONTAINERS — already updated daily
|
||||
# emby + critical-data profiles — updated inline by the weekly sync window
|
||||
# WEEKLY_RESTART_CONTAINERS — restarted by docker_weekly_restart.sh after sync
|
||||
# FALLBACK_*_TIER* — owned by the remote server's update cycle
|
||||
# Pull → compare → prune dangling images.
|
||||
#
|
||||
@@ -91,6 +92,9 @@
|
||||
# Enable or disable normal mode. docker_daily_restart.sh runs regardless.
|
||||
# (default: true)
|
||||
#
|
||||
# WEEKLY_REMAINING_UPDATES
|
||||
# Enable or disable remainder mode. (default: true)
|
||||
#
|
||||
# PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
|
||||
# Container names for emby and critical-data profiles — excluded from
|
||||
# remainder mode (already updated by the weekly sync window)
|
||||
@@ -163,6 +167,11 @@ detect_hosts
|
||||
# ━━━ Container Discovery ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then
|
||||
echo "WEEKLY_REMAINING_UPDATES=false — skipping remainder container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
declare -A _exclude=()
|
||||
|
||||
# Daily containers — updated by docker_update.sh normal mode
|
||||
@@ -178,6 +187,11 @@ if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
done
|
||||
unset _weekly_str _weekly_arr
|
||||
|
||||
# Weekly restart containers — restarted by docker_weekly_restart.sh after sync
|
||||
for _c in "${WEEKLY_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -n "$_c" ]] && _exclude["$_c"]=1
|
||||
done
|
||||
|
||||
# Fallback coverage containers — owned by the remote server's update cycle.
|
||||
# This server runs them during fallback but should never update them independently.
|
||||
# Updating them here risks version divergence: if remote's writeback after handback
|
||||
@@ -221,7 +235,7 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Mode: $([[ "$REMAINDER_MODE" == true ]] && echo "remainder" || echo "normal (daily)")"
|
||||
if [[ "$REMAINDER_MODE" == true ]]; then
|
||||
echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, and fallback)"
|
||||
echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, weekly restart, and fallback)"
|
||||
for _c in "${TARGET_CONTAINERS[@]}"; do echo " $_c"; done
|
||||
else
|
||||
echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
|
||||
|
||||
@@ -1,325 +0,0 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Docker Update — Remaining ======================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Weekly sweep that pulls the latest image for every running container not
|
||||
# already covered by the daily or weekly managed update cycles. Restarts
|
||||
# containers that received a new image, then prunes dangling images.
|
||||
#
|
||||
# Called by weekly_sync_maintenance.sh as the final step in the weekly window.
|
||||
# Derives its target list automatically from docker ps minus the two managed
|
||||
# lists — there is nothing to configure for this script.
|
||||
#
|
||||
# Together with docker_update.sh (normal + remainder modes), every deployed
|
||||
# container receives at least one image pull per week without any per-container
|
||||
# configuration required here.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Lock Acquisition
|
||||
# Prevents concurrent execution via acquire_lock(). Safe to call from
|
||||
# weekly maintenance scripts without risk of overlap.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# HOST*_DAILY_RESTART_CONTAINERS and HOST*_WEEKLY_RESTART_CONTAINERS to
|
||||
# the correct host's values for exclusion.
|
||||
#
|
||||
# Root Enforcement
|
||||
# Docker operations require root privileges.
|
||||
#
|
||||
# WEEKLY_REMAINING_UPDATES Toggle
|
||||
# Exits cleanly when disabled via master.conf.
|
||||
#
|
||||
# Running-Only Filter
|
||||
# Stopped containers excluded — intentionally down, pulling adds no value.
|
||||
#
|
||||
# Image ID Comparison
|
||||
# Containers not restarted unless their image actually changed.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# WEEKLY_REMAINING_UPDATES
|
||||
# Enable or disable this script. (default: true)
|
||||
# To disable without the toggle: remove from WEEKLY_MAINTENANCE_SCRIPTS.
|
||||
#
|
||||
# host*.conf
|
||||
#
|
||||
# HOST*_DAILY_RESTART_CONTAINERS
|
||||
# Excluded from this script — already updated daily. Aliased by
|
||||
# detect_hosts() → DAILY_RESTART_CONTAINERS
|
||||
#
|
||||
# HOST*_WEEKLY_RESTART_CONTAINERS
|
||||
# Excluded from this script — already updated by weekly sync window.
|
||||
# Aliased by detect_hosts() → WEEKLY_RESTART_CONTAINERS
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# docker_update_remaining.sh
|
||||
# Pull all remaining running containers, restart those updated, prune images
|
||||
#
|
||||
# docker_update_remaining.sh --dry-run
|
||||
# Preview which containers would be pulled and restarted
|
||||
#
|
||||
# docker_update_remaining.sh --status
|
||||
# Show exclusion lists and current remaining container count
|
||||
#
|
||||
# docker_update_remaining.sh --log
|
||||
# Verbose per-container pull and restart output
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then
|
||||
echo "WEEKLY_REMAINING_UPDATES=false — skipping remaining container updates"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Build exclusion set from daily + weekly managed lists ─────────────────────────────────────
|
||||
declare -A EXCLUDED
|
||||
for c in "${DAILY_RESTART_CONTAINERS[@]}" "${WEEKLY_RESTART_CONTAINERS[@]}"; do
|
||||
[[ -n "$c" ]] && EXCLUDED["$c"]=1
|
||||
done
|
||||
|
||||
# ── Get all running containers ────────────────────────────────────────────────────────────────
|
||||
mapfile -t ALL_RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort)
|
||||
|
||||
# ── Derive remainder: running minus excluded ──────────────────────────────────────────────────
|
||||
REMAINING=()
|
||||
for c in "${ALL_RUNNING[@]}"; do
|
||||
[[ -z "$c" ]] && continue
|
||||
[[ -n "${EXCLUDED[$c]:-}" ]] && continue
|
||||
REMAINING+=("$c")
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR Enabled: ${WEEKLY_REMAINING_UPDATES:-true}"
|
||||
echo "$ICON_CONTAINERS All running: ${#ALL_RUNNING[@]}"
|
||||
echo "$ICON_CONTAINERS Excluded: ${!EXCLUDED[*]}"
|
||||
echo "$ICON_CONTAINERS Remaining: ${REMAINING[*]:-none}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ${#REMAINING[@]} -eq 0 ]]; then
|
||||
echo "No remaining containers to update — all running containers are covered by daily/weekly lists"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled or containers restarted"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# docker_cmd, retry_docker, verify_running — defined in common.sh
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pull Updates ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Docker Update (Remaining) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
log "$ICON_CONTAINERS Containers: ${REMAINING[*]}"
|
||||
log "$ICON_CONTAINERS Excluded (managed elsewhere): ${!EXCLUDED[*]}"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
UPDATED=()
|
||||
UP_TO_DATE=()
|
||||
FAILED=()
|
||||
OLD_IMAGE_IDS=()
|
||||
|
||||
for container in "${REMAINING[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
|
||||
if [[ -z "$IMAGE" ]]; then
|
||||
warn "$container — could not determine image, skipping"
|
||||
FAILED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
log "$container — image: $IMAGE"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull: $IMAGE"
|
||||
UPDATED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
|
||||
OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
log "$ICON_SYNC Pulling $IMAGE..."
|
||||
if [[ "$ENABLE_LOGGING" == "true" ]]; then
|
||||
docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /'
|
||||
_pull_rc=${PIPESTATUS[0]}
|
||||
else
|
||||
docker pull "$IMAGE" >/dev/null 2>&1
|
||||
_pull_rc=$?
|
||||
fi
|
||||
NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
||||
|
||||
if [[ $_pull_rc -eq 0 ]]; then
|
||||
_pull_new=$( [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
_container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false)
|
||||
|
||||
if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then
|
||||
[[ "$_pull_new" == true ]] && log "$ICON_DONE $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})"
|
||||
[[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})"
|
||||
UPDATED+=("$container")
|
||||
OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID")
|
||||
else
|
||||
log "$container — up to date (${NEW_ID:7:12})"
|
||||
UP_TO_DATE+=("$container")
|
||||
fi
|
||||
else
|
||||
warn "$container — pull failed ($IMAGE)"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Restart Updated Containers ━━━
|
||||
# ==============================================================================================
|
||||
RESTARTED=()
|
||||
RESTART_FAILED=()
|
||||
SKIPPED_STOPPED=()
|
||||
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Restarting Updated Containers — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
log "$ICON_CONTAINERS Containers with new image: ${UPDATED[*]}"
|
||||
|
||||
for container in "${UPDATED[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
log "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
|
||||
if [[ "$STATUS" != "true" ]]; then
|
||||
log "$ICON_NOT_RUNNING $container is stopped — skipping restart (respecting stopped state)"
|
||||
SKIPPED_STOPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container"
|
||||
RESTARTED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
log "$ICON_RUNNING $container — recreating from template on new image..."
|
||||
if platform_rebuild_container "$container"; then
|
||||
if verify_running "$container"; then
|
||||
log "$ICON_DONE $container recreated and running ✅"
|
||||
RESTARTED+=("$container")
|
||||
else
|
||||
error "$container recreated but not running — may be intentionally stopped"
|
||||
RESTARTED+=("$container")
|
||||
fi
|
||||
else
|
||||
error "Failed to rebuild $container from template"
|
||||
notify "$container failed to rebuild after update on $(hostname)" "Docker Update Remaining" "warning"
|
||||
RESTART_FAILED+=("$container")
|
||||
fi
|
||||
done
|
||||
else
|
||||
log "No containers received a new image — nothing to restart"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Prune Old Images ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Pruning Dangling Images — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling"
|
||||
PRUNED_SUMMARY="(dry run)"
|
||||
else
|
||||
for _old_id in "${OLD_IMAGE_IDS[@]}"; do
|
||||
docker rmi "$_old_id" >/dev/null 2>&1 || true
|
||||
done
|
||||
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
|
||||
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /'
|
||||
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
|
||||
fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINING) SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo "$ICON_CONTAINERS Scope: ${#ALL_RUNNING[@]} running — ${#EXCLUDED[@]} managed = ${#REMAINING[@]} checked"
|
||||
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_DONE New image: ${#UPDATED[@]}"
|
||||
log " ${UPDATED[*]}"
|
||||
fi
|
||||
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && log "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Pull failed: ${FAILED[*]}"
|
||||
if [[ ${#RESTARTED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_DONE Restarted: ${#RESTARTED[@]}"
|
||||
log " ${RESTARTED[*]}"
|
||||
fi
|
||||
[[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && log "$ICON_WARN Not running: ${SKIPPED_STOPPED[*]} (skipped restart)"
|
||||
[[ ${#RESTART_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Restart fail:${RESTART_FAILED[*]}"
|
||||
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
||||
|
||||
ALL_FAILED=$(( ${#FAILED[@]} + ${#RESTART_FAILED[@]} ))
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$ALL_FAILED" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: done ✅ — ${#RESTARTED[@]} restarted, ${#UP_TO_DATE[@]} current"
|
||||
else
|
||||
warn "Status: $ALL_FAILED error(s) — ${#FAILED[@]} pull failure(s), ${#RESTART_FAILED[@]} restart failure(s)"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$ALL_FAILED" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user