From 9425d16c190644afdcb4017a115f6a35a452f18b Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sun, 19 Jul 2026 16:09:55 -0400 Subject: [PATCH] Skip redundant restart for containers docker_update.sh already rebuilt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docker_update.sh rebuilds (stop+recreate) any container whose image changed, in every mode — but for daily/weekly that was always followed by the restart script's own unconditional pass, stopping and starting the same container twice back to back. docker_update.sh now records which containers it rebuilt this run to a file; docker_daily_restart.sh and docker_weekly_restart.sh read it and skip those specifically, still restarting everything else as before. A file older than DOCKER_UPDATE_REBUILT_STALE_HOURS (default 12) is discarded rather than trusted, so a missed or failed update run can't suppress a restart indefinitely. --- Deployment/master.conf.template | 12 +++++++ Docker_Essentials/docker_daily_restart.sh | 41 ++++++++++++++++++++-- Docker_Essentials/docker_update.sh | 21 +++++++++++ Docker_Essentials/docker_weekly_restart.sh | 39 +++++++++++++++++++- 4 files changed, 110 insertions(+), 3 deletions(-) diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 4f3acfe..3da552d 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -453,6 +453,18 @@ # Set false to skip. MONTHLY_REMAINING_UPDATES=true +# docker_update.sh rebuilds (stop+recreate onto new image) any container whose image changed, +# in every mode. For daily/weekly, that rebuild is immediately followed by the restart script's +# own unconditional pass — a container that just got rebuilt would be stopped and started again +# right after for no reason. docker_update.sh records which containers it rebuilt to these files; +# docker_daily_restart.sh / docker_weekly_restart.sh read them and skip those containers rather +# than restarting them a second time. A file older than DOCKER_UPDATE_REBUILT_STALE_HOURS is +# treated as untrustworthy (docker_update.sh likely didn't run, or didn't run recently) — deleted, +# and every container in that tier restarts normally, same as if the file never existed. + DOCKER_UPDATE_REBUILT_DAILY_FILE="$DATA_DIR/docker_update_rebuilt_daily.list" + DOCKER_UPDATE_REBUILT_WEEKLY_FILE="$DATA_DIR/docker_update_rebuilt_weekly.list" + DOCKER_UPDATE_REBUILT_STALE_HOURS=12 + # Shares synced during the weekly maintenance window — defined per host in host*.conf. # HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES # Containers stopped both sides before sync — full clean state guaranteed. diff --git a/Docker_Essentials/docker_daily_restart.sh b/Docker_Essentials/docker_daily_restart.sh index b02ca6f..6667e7a 100755 --- a/Docker_Essentials/docker_daily_restart.sh +++ b/Docker_Essentials/docker_daily_restart.sh @@ -87,6 +87,11 @@ # Gives the process time to initialise before verify_running samples the state. # (default: 3) # +# DOCKER_UPDATE_REBUILT_DAILY_FILE / DOCKER_UPDATE_REBUILT_STALE_HOURS +# List of containers docker_update.sh already rebuilt onto a new image this run — +# read here so they're not restarted a second time. Discarded as stale (and every +# container restarts normally) if older than DOCKER_UPDATE_REBUILT_STALE_HOURS. +# # ============================================================================================== # RUNTIME MODES # ============================================================================================== @@ -175,10 +180,34 @@ START=$(date +%s) FAILED=() RESTARTED=() SKIPPED=() +ALREADY_UPDATED=() # Build dependency-safe restart order build_restart_order DAILY_RESTART_CONTAINERS +# ── Load containers docker_update.sh already rebuilt this run ─────────────────────────────────── +# docker_update.sh's rebuild (stop+recreate onto a new image) already restarted anything whose +# image changed today — doing a plain restart on it again here is redundant. A file older than +# DOCKER_UPDATE_REBUILT_STALE_HOURS means docker_update.sh either didn't run today or this is way +# out of sync with it, so it's discarded rather than trusted, and every container restarts as +# normal — same as if the file had never existed. +declare -A ALREADY_REBUILT_MAP +if [[ -n "${DOCKER_UPDATE_REBUILT_DAILY_FILE:-}" && -f "$DOCKER_UPDATE_REBUILT_DAILY_FILE" ]]; then + _rebuilt_age=$(( $(date +%s) - $(stat -c %Y "$DOCKER_UPDATE_REBUILT_DAILY_FILE" 2>/dev/null || echo 0) )) + _rebuilt_stale_seconds=$(( ${DOCKER_UPDATE_REBUILT_STALE_HOURS:-12} * 3600 )) + if [[ "$_rebuilt_age" -gt "$_rebuilt_stale_seconds" ]]; then + warn "Rebuilt-container list is stale ($(( _rebuilt_age / 3600 ))h old) — discarding, restarting all" + rm -f "$DOCKER_UPDATE_REBUILT_DAILY_FILE" + else + while IFS= read -r _c; do + [[ -n "$_c" ]] && ALREADY_REBUILT_MAP["$_c"]=1 + done < "$DOCKER_UPDATE_REBUILT_DAILY_FILE" + [[ "${#ALREADY_REBUILT_MAP[@]}" -gt 0 ]] && \ + log "Already rebuilt today by docker_update.sh, skipping restart: ${!ALREADY_REBUILT_MAP[*]}" + fi + unset _rebuilt_age _rebuilt_stale_seconds +fi + LAST_RESTARTED="" for container in "${ORDERED_RESTART[@]}"; do @@ -196,6 +225,13 @@ for container in "${ORDERED_RESTART[@]}"; do case "$STATUS" in true) + if [[ -n "${ALREADY_REBUILT_MAP[$container]:-}" ]]; then + log "$ICON_RUNNING $container already rebuilt onto new image by docker_update.sh — skipping redundant restart" + ALREADY_UPDATED+=("$container") + LAST_RESTARTED="$container" # it did restart, just moments ago via the rebuild + continue + fi + log "$ICON_RUNNING $container is running — restarting..." # Wait if this container depends on the last one restarted @@ -258,8 +294,9 @@ fi echo "━━━━━ $ICON_SUMMARY DAILY RESTART SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -echo "$ICON_CONTAINERS Scope: ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped, ${#FAILED[@]} failed" +echo "$ICON_CONTAINERS Scope: ${#RESTARTED[@]} restarted, ${#ALREADY_UPDATED[@]} already updated, ${#SKIPPED[@]} skipped, ${#FAILED[@]} failed" [[ ${#RESTARTED[@]} -gt 0 ]] && log "$ICON_STARTED Restarted: ${RESTARTED[*]}" +[[ ${#ALREADY_UPDATED[@]} -gt 0 ]] && log "$ICON_DONE Already updated (skipped): ${ALREADY_UPDATED[*]}" [[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]}" [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}" @@ -268,7 +305,7 @@ if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ ${#FAILED[@]} -eq 0 ]]; then echo "$ICON_DONE Status: ALL DONE ✅" - notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped on $(hostname)" "Docker Daily Restart" "normal" + notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#ALREADY_UPDATED[@]} already updated, ${#SKIPPED[@]} skipped on $(hostname)" "Docker Daily Restart" "normal" else echo "$ICON_ERROR Status: ${#FAILED[@]} container(s) failed" notify "Daily restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Daily Restart" "warning" diff --git a/Docker_Essentials/docker_update.sh b/Docker_Essentials/docker_update.sh index b303953..e9b887b 100755 --- a/Docker_Essentials/docker_update.sh +++ b/Docker_Essentials/docker_update.sh @@ -106,6 +106,13 @@ # MONTHLY_REMAINING_UPDATES # Enable or disable remainder mode. (default: true) # +# DOCKER_UPDATE_REBUILT_DAILY_FILE / DOCKER_UPDATE_REBUILT_WEEKLY_FILE +# Written after each normal/weekly run with the containers actually rebuilt +# this pass — read by docker_daily_restart.sh / docker_weekly_restart.sh so +# they skip restarting a container a second time right after this script +# already rebuilt it onto the new image. Not written in remainder mode +# (no follow-up restart script exists for it). +# # PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data] # Container names for emby and critical-data profiles — excluded from # remainder mode (already updated by the weekly sync window) @@ -406,6 +413,20 @@ if [[ ${#UPDATED[@]} -gt 0 ]]; then done fi +# Record successfully-rebuilt containers so the follow-up restart script (daily/weekly only — +# remainder has no follow-up) can skip them instead of restarting an already-fresh container a +# second time. Deliberately excludes REBUILD_FAILED — those still need the restart script's +# normal pass as a fallback, exactly as the error message above promises. Written even when +# REBUILT is empty, so a stale file from a previous run doesn't linger and get misread later. +if [[ "$DRY_RUN" == false && "$REMAINDER_MODE" != true ]]; then + _rebuilt_file="$DOCKER_UPDATE_REBUILT_DAILY_FILE" + [[ "$WEEKLY_MODE" == true ]] && _rebuilt_file="$DOCKER_UPDATE_REBUILT_WEEKLY_FILE" + if [[ -n "$_rebuilt_file" ]]; then + printf '%s\n' "${REBUILT[@]}" > "$_rebuilt_file" 2>/dev/null + fi + unset _rebuilt_file +fi + # ── Remove old images ──────────────────────────────────────────────────────── # Explicitly rmi by the IDs captured before each pull. Tagged images are never # caught by dangling-only prune, so this is the only reliable cleanup path. diff --git a/Docker_Essentials/docker_weekly_restart.sh b/Docker_Essentials/docker_weekly_restart.sh index b5d5c1b..8bc1074 100755 --- a/Docker_Essentials/docker_weekly_restart.sh +++ b/Docker_Essentials/docker_weekly_restart.sh @@ -84,6 +84,12 @@ # CONTAINER_DELAY # Seconds to wait after restarting a dependency before starting its dependents # +# DOCKER_UPDATE_REBUILT_WEEKLY_FILE / DOCKER_UPDATE_REBUILT_STALE_HOURS +# List of containers docker_update.sh --weekly already rebuilt onto a new image +# this run — read here so they're not restarted a second time. Discarded as +# stale (and every container restarts normally) if older than +# DOCKER_UPDATE_REBUILT_STALE_HOURS. +# # ============================================================================================== # RUNTIME MODES # ============================================================================================== @@ -173,10 +179,33 @@ START=$(date +%s) FAILED=() RESTARTED=() SKIPPED=() +ALREADY_UPDATED=() # Build dependency-safe restart order build_restart_order WEEKLY_RESTART_CONTAINERS +# ── Load containers docker_update.sh --weekly already rebuilt this run ────────────────────────── +# Same reasoning as docker_daily_restart.sh: a container docker_update.sh already rebuilt onto a +# new image doesn't need a plain restart right after. A file older than +# DOCKER_UPDATE_REBUILT_STALE_HOURS is discarded as untrustworthy rather than trusted, and every +# container restarts as normal. +declare -A ALREADY_REBUILT_MAP +if [[ -n "${DOCKER_UPDATE_REBUILT_WEEKLY_FILE:-}" && -f "$DOCKER_UPDATE_REBUILT_WEEKLY_FILE" ]]; then + _rebuilt_age=$(( $(date +%s) - $(stat -c %Y "$DOCKER_UPDATE_REBUILT_WEEKLY_FILE" 2>/dev/null || echo 0) )) + _rebuilt_stale_seconds=$(( ${DOCKER_UPDATE_REBUILT_STALE_HOURS:-12} * 3600 )) + if [[ "$_rebuilt_age" -gt "$_rebuilt_stale_seconds" ]]; then + warn "Rebuilt-container list is stale ($(( _rebuilt_age / 3600 ))h old) — discarding, restarting all" + rm -f "$DOCKER_UPDATE_REBUILT_WEEKLY_FILE" + else + while IFS= read -r _c; do + [[ -n "$_c" ]] && ALREADY_REBUILT_MAP["$_c"]=1 + done < "$DOCKER_UPDATE_REBUILT_WEEKLY_FILE" + [[ "${#ALREADY_REBUILT_MAP[@]}" -gt 0 ]] && \ + log "Already rebuilt today by docker_update.sh, skipping restart: ${!ALREADY_REBUILT_MAP[*]}" + fi + unset _rebuilt_age _rebuilt_stale_seconds +fi + LAST_RESTARTED="" for container in "${ORDERED_RESTART[@]}"; do @@ -194,6 +223,13 @@ for container in "${ORDERED_RESTART[@]}"; do case "$STATUS" in true) + if [[ -n "${ALREADY_REBUILT_MAP[$container]:-}" ]]; then + log "$ICON_RUNNING $container already rebuilt onto new image by docker_update.sh — skipping redundant restart" + ALREADY_UPDATED+=("$container") + LAST_RESTARTED="$container" # it did restart, just moments ago via the rebuild + continue + fi + log "$ICON_RUNNING $container is running — restarting..." # Wait if this container depends on the last one restarted @@ -260,6 +296,7 @@ if [[ ${#RESTARTED[@]} -gt 0 ]]; then echo "$ICON_STARTED Restarted: ${#RESTARTED[@]}" log " Names: ${RESTARTED[*]}" fi +[[ ${#ALREADY_UPDATED[@]} -gt 0 ]] && log "$ICON_DONE Already updated (skipped): ${ALREADY_UPDATED[*]}" [[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]} (were stopped)" [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}" @@ -268,7 +305,7 @@ if [[ "$DRY_RUN" == true ]]; then echo "$ICON_WARN Status: DRY RUN — no changes made" elif [[ ${#FAILED[@]} -eq 0 ]]; then echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE" - notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped (stopped) on $(hostname)" "Docker Weekly Restart" "normal" + notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#ALREADY_UPDATED[@]} already updated, ${#SKIPPED[@]} skipped (stopped) on $(hostname)" "Docker Weekly Restart" "normal" else echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed" notify "Weekly restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Weekly Restart" "warning"