Gmer4Lfe
740710e0d0
docker_container_stop, docker_update, docker_update_remaining — concurrent Docker operations on the same containers would conflict; now locked. rsync.sh — two rsync processes running against the same share simultaneously would produce incomplete or corrupted mirrors; now locked. partnership_onboard, ssh_setup — one-shot setup scripts that mutate SSH config and deploy containers; concurrent runs would produce undefined state; now locked. Old_Arch_Still_Works: arr_cleanup, continuous_scripts_status — legacy scripts still sourcing load_config.sh; added lock for consistency even in old-arch context. partnership_manager.sh intentionally left unchanged — it uses a conditional lock that excludes read-only "check" mode and "offboard" mode (which delegates to partnership_offboard.sh, which has its own lock).
337 lines
14 KiB
Bash
337 lines
14 KiB
Bash
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Docker Update — Remaining ======================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Weekly sweep that pulls the latest image for every running container not
|
|
# already covered by the daily or weekly managed update cycles. Restarts
|
|
# containers that received a new image, then prunes dangling images.
|
|
#
|
|
# Called by weekly_sync_maintenance.sh as the final step in the weekly window.
|
|
# Derives its target list automatically from docker ps minus the two managed
|
|
# lists — there is nothing to configure for this script.
|
|
#
|
|
# Together with docker_update.sh (normal + remainder modes), every deployed
|
|
# container receives at least one image pull per week without any per-container
|
|
# configuration required here.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root Enforcement
|
|
# Docker operations require root privileges.
|
|
#
|
|
# WEEKLY_REMAINING_UPDATES Toggle
|
|
# Exits cleanly when disabled via master.conf.
|
|
#
|
|
# Running-Only Filter
|
|
# Stopped containers excluded — intentionally down, pulling adds no value.
|
|
#
|
|
# Image ID Comparison
|
|
# Containers not restarted unless their image actually changed.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# WEEKLY_REMAINING_UPDATES
|
|
# Enable or disable this script. (default: true)
|
|
# To disable without the toggle: remove from WEEKLY_MAINTENANCE_SCRIPTS.
|
|
#
|
|
# master_host*.conf
|
|
#
|
|
# HOST*_DAILY_RESTART_CONTAINERS
|
|
# Excluded from this script — already updated daily. Aliased by
|
|
# detect_hosts() → DAILY_RESTART_CONTAINERS
|
|
#
|
|
# HOST*_WEEKLY_RESTART_CONTAINERS
|
|
# Excluded from this script — already updated by weekly sync window.
|
|
# Aliased by detect_hosts() → WEEKLY_RESTART_CONTAINERS
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# docker_update_remaining.sh
|
|
# Pull all remaining running containers, restart those updated, prune images
|
|
#
|
|
# docker_update_remaining.sh --dry-run
|
|
# Preview which containers would be pulled and restarted
|
|
#
|
|
# docker_update_remaining.sh --status
|
|
# Show exclusion lists and current remaining container count
|
|
#
|
|
# docker_update_remaining.sh --log
|
|
# Verbose per-container pull and restart output
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
detect_hosts
|
|
|
|
if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then
|
|
log "WEEKLY_REMAINING_UPDATES=false — skipping remaining container updates"
|
|
exit 0
|
|
fi
|
|
|
|
# ── Build exclusion set from daily + weekly managed lists ─────────────────────────────────────
|
|
declare -A EXCLUDED
|
|
for c in "${DAILY_RESTART_CONTAINERS[@]}" "${WEEKLY_RESTART_CONTAINERS[@]}"; do
|
|
[[ -n "$c" ]] && EXCLUDED["$c"]=1
|
|
done
|
|
|
|
# ── Get all running containers ────────────────────────────────────────────────────────────────
|
|
mapfile -t ALL_RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort)
|
|
|
|
# ── Derive remainder: running minus excluded ──────────────────────────────────────────────────
|
|
REMAINING=()
|
|
for c in "${ALL_RUNNING[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
[[ -n "${EXCLUDED[$c]:-}" ]] && continue
|
|
REMAINING+=("$c")
|
|
done
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_GEAR Enabled: ${WEEKLY_REMAINING_UPDATES:-true}"
|
|
echo "$ICON_CONTAINERS All running: ${#ALL_RUNNING[@]}"
|
|
echo "$ICON_CONTAINERS Excluded: ${!EXCLUDED[*]}"
|
|
echo "$ICON_CONTAINERS Remaining: ${REMAINING[*]:-none}"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ ${#REMAINING[@]} -eq 0 ]]; then
|
|
log "No remaining containers to update — all running containers are covered by daily/weekly lists"
|
|
exit 0
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled or containers restarted"
|
|
|
|
# ==============================================================================================
|
|
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
DOCKER_TIMEOUT=30
|
|
docker_cmd() {
|
|
timeout "$DOCKER_TIMEOUT" "$@"
|
|
local exit_code=$?
|
|
if [[ "$exit_code" -eq 124 ]]; then
|
|
error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*"
|
|
return 1
|
|
fi
|
|
return "$exit_code"
|
|
}
|
|
|
|
retry_docker() {
|
|
local attempt=1
|
|
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
|
|
log "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
|
if docker_cmd "$@"; then
|
|
log "Succeeded on attempt $attempt"
|
|
return 0
|
|
else
|
|
warn "Attempt $attempt failed"
|
|
(( attempt++ ))
|
|
[[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP"
|
|
fi
|
|
done
|
|
error "Command failed after $RETRY_COUNT attempts: $*"
|
|
return 1
|
|
}
|
|
|
|
RESTART_VERIFY_WAIT=5
|
|
verify_running() {
|
|
local container="$1"
|
|
sleep "$RESTART_VERIFY_WAIT"
|
|
local state
|
|
state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
|
[[ "$state" == "true" ]]
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Pull Updates ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_CONTAINERS Docker Update (Remaining) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_CONTAINERS Containers: ${REMAINING[*]}"
|
|
echo "$ICON_CONTAINERS Excluded (managed elsewhere): ${!EXCLUDED[*]}"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
UPDATED=()
|
|
UP_TO_DATE=()
|
|
FAILED=()
|
|
|
|
for container in "${REMAINING[@]}"; do
|
|
[[ -z "$container" ]] && continue
|
|
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
|
|
|
IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
|
|
if [[ -z "$IMAGE" ]]; then
|
|
warn "$container — could not determine image, skipping"
|
|
FAILED+=("$container")
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
log "$container — image: $IMAGE"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would pull: $IMAGE"
|
|
UPDATED+=("$container")
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
|
|
|
echo "$ICON_SYNC Pulling $IMAGE..."
|
|
if docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/ /'; then
|
|
NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
|
|
|
|
if [[ -n "$OLD_ID" ]] && [[ "$OLD_ID" != "$NEW_ID" ]]; then
|
|
echo "$ICON_DONE $container — updated ✅"
|
|
UPDATED+=("$container")
|
|
else
|
|
log "$container — already up to date"
|
|
UP_TO_DATE+=("$container")
|
|
fi
|
|
else
|
|
warn "$container — pull failed ($IMAGE)"
|
|
FAILED+=("$container")
|
|
fi
|
|
|
|
echo ""
|
|
done
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Restart Updated Containers ━━━
|
|
# ==============================================================================================
|
|
RESTARTED=()
|
|
RESTART_FAILED=()
|
|
SKIPPED_STOPPED=()
|
|
|
|
if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_CONTAINERS Restarting Updated Containers — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_CONTAINERS Containers with new image: ${UPDATED[*]}"
|
|
echo ""
|
|
|
|
for container in "${UPDATED[@]}"; do
|
|
[[ -z "$container" ]] && continue
|
|
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
|
|
|
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
|
|
|
if [[ "$STATUS" != "true" ]]; then
|
|
echo "$ICON_NOT_RUNNING $container is stopped — skipping restart (respecting stopped state)"
|
|
SKIPPED_STOPPED+=("$container")
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would restart $container"
|
|
RESTARTED+=("$container")
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
echo "$ICON_RUNNING $container is running — restarting on new image..."
|
|
if retry_docker docker restart "$container"; then
|
|
if verify_running "$container"; then
|
|
echo "$ICON_DONE $container restarted and running ✅"
|
|
RESTARTED+=("$container")
|
|
else
|
|
error "$container restarted but crashed immediately"
|
|
notify "$container crashed after update-restart on $(hostname)" "Docker Update Remaining" "warning"
|
|
RESTART_FAILED+=("$container")
|
|
fi
|
|
else
|
|
error "Failed to restart $container after $RETRY_COUNT attempts"
|
|
notify "$container failed to restart after update on $(hostname)" "Docker Update Remaining" "warning"
|
|
RESTART_FAILED+=("$container")
|
|
fi
|
|
echo ""
|
|
done
|
|
else
|
|
log "No containers received a new image — nothing to restart"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Prune Old Images ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Pruning Dangling Images — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would prune dangling images"
|
|
PRUNED_SUMMARY="(dry run)"
|
|
else
|
|
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
|
|
echo "$PRUNED_OUTPUT" | sed 's/^/ /'
|
|
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
|
|
fi
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINING) SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo "$ICON_CONTAINERS Scope: ${#ALL_RUNNING[@]} running — ${#EXCLUDED[@]} managed = ${#REMAINING[@]} checked"
|
|
[[ ${#UPDATED[@]} -gt 0 ]] && echo "$ICON_DONE New image: ${UPDATED[*]}"
|
|
[[ ${#UP_TO_DATE[@]} -gt 0 ]] && echo "$ICON_RUNNING Up to date: ${UP_TO_DATE[*]}"
|
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Pull failed: ${FAILED[*]}"
|
|
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_DONE Restarted: ${RESTARTED[*]}"
|
|
[[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && echo "$ICON_WARN Not running: ${SKIPPED_STOPPED[*]} (skipped restart)"
|
|
[[ ${#RESTART_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Restart fail:${RESTART_FAILED[*]}"
|
|
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
|
|
|
ALL_FAILED=$(( ${#FAILED[@]} + ${#RESTART_FAILED[@]} ))
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no changes made"
|
|
elif [[ "$ALL_FAILED" -eq 0 ]]; then
|
|
log "$ICON_DONE Status: done ✅ — ${#RESTARTED[@]} restarted, ${#UP_TO_DATE[@]} current"
|
|
else
|
|
warn "Status: $ALL_FAILED error(s) — ${#FAILED[@]} pull failure(s), ${#RESTART_FAILED[@]} restart failure(s)"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ "$ALL_FAILED" -gt 0 ]] && exit 1
|
|
exit 0
|