Skip redundant restart for containers docker_update.sh already rebuilt
docker_update.sh rebuilds (stop+recreate) any container whose image changed, in every mode — but for daily/weekly that was always followed by the restart script's own unconditional pass, stopping and starting the same container twice back to back. docker_update.sh now records which containers it rebuilt this run to a file; docker_daily_restart.sh and docker_weekly_restart.sh read it and skip those specifically, still restarting everything else as before. A file older than DOCKER_UPDATE_REBUILT_STALE_HOURS (default 12) is discarded rather than trusted, so a missed or failed update run can't suppress a restart indefinitely.
This commit is contained in:
Gmer4Lfe
@@ -453,6 +453,18 @@
|
||||
# Set false to skip.
|
||||
MONTHLY_REMAINING_UPDATES=true
|
||||
|
||||
# docker_update.sh rebuilds (stop+recreate onto new image) any container whose image changed,
|
||||
# in every mode. For daily/weekly, that rebuild is immediately followed by the restart script's
|
||||
# own unconditional pass — a container that just got rebuilt would be stopped and started again
|
||||
# right after for no reason. docker_update.sh records which containers it rebuilt to these files;
|
||||
# docker_daily_restart.sh / docker_weekly_restart.sh read them and skip those containers rather
|
||||
# than restarting them a second time. A file older than DOCKER_UPDATE_REBUILT_STALE_HOURS is
|
||||
# treated as untrustworthy (docker_update.sh likely didn't run, or didn't run recently) — deleted,
|
||||
# and every container in that tier restarts normally, same as if the file never existed.
|
||||
DOCKER_UPDATE_REBUILT_DAILY_FILE="$DATA_DIR/docker_update_rebuilt_daily.list"
|
||||
DOCKER_UPDATE_REBUILT_WEEKLY_FILE="$DATA_DIR/docker_update_rebuilt_weekly.list"
|
||||
DOCKER_UPDATE_REBUILT_STALE_HOURS=12
|
||||
|
||||
# Shares synced during the weekly maintenance window — defined per host in host*.conf.
|
||||
# HOST1_WEEKLY_SYNC_SHARES / HOST2_WEEKLY_SYNC_SHARES
|
||||
# Containers stopped both sides before sync — full clean state guaranteed.
|
||||
|
||||
@@ -87,6 +87,11 @@
|
||||
# Gives the process time to initialise before verify_running samples the state.
|
||||
# (default: 3)
|
||||
#
|
||||
# DOCKER_UPDATE_REBUILT_DAILY_FILE / DOCKER_UPDATE_REBUILT_STALE_HOURS
|
||||
# List of containers docker_update.sh already rebuilt onto a new image this run —
|
||||
# read here so they're not restarted a second time. Discarded as stale (and every
|
||||
# container restarts normally) if older than DOCKER_UPDATE_REBUILT_STALE_HOURS.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
@@ -175,10 +180,34 @@ START=$(date +%s)
|
||||
FAILED=()
|
||||
RESTARTED=()
|
||||
SKIPPED=()
|
||||
ALREADY_UPDATED=()
|
||||
|
||||
# Build dependency-safe restart order
|
||||
build_restart_order DAILY_RESTART_CONTAINERS
|
||||
|
||||
# ── Load containers docker_update.sh already rebuilt this run ───────────────────────────────────
|
||||
# docker_update.sh's rebuild (stop+recreate onto a new image) already restarted anything whose
|
||||
# image changed today — doing a plain restart on it again here is redundant. A file older than
|
||||
# DOCKER_UPDATE_REBUILT_STALE_HOURS means docker_update.sh either didn't run today or this is way
|
||||
# out of sync with it, so it's discarded rather than trusted, and every container restarts as
|
||||
# normal — same as if the file had never existed.
|
||||
declare -A ALREADY_REBUILT_MAP
|
||||
if [[ -n "${DOCKER_UPDATE_REBUILT_DAILY_FILE:-}" && -f "$DOCKER_UPDATE_REBUILT_DAILY_FILE" ]]; then
|
||||
_rebuilt_age=$(( $(date +%s) - $(stat -c %Y "$DOCKER_UPDATE_REBUILT_DAILY_FILE" 2>/dev/null || echo 0) ))
|
||||
_rebuilt_stale_seconds=$(( ${DOCKER_UPDATE_REBUILT_STALE_HOURS:-12} * 3600 ))
|
||||
if [[ "$_rebuilt_age" -gt "$_rebuilt_stale_seconds" ]]; then
|
||||
warn "Rebuilt-container list is stale ($(( _rebuilt_age / 3600 ))h old) — discarding, restarting all"
|
||||
rm -f "$DOCKER_UPDATE_REBUILT_DAILY_FILE"
|
||||
else
|
||||
while IFS= read -r _c; do
|
||||
[[ -n "$_c" ]] && ALREADY_REBUILT_MAP["$_c"]=1
|
||||
done < "$DOCKER_UPDATE_REBUILT_DAILY_FILE"
|
||||
[[ "${#ALREADY_REBUILT_MAP[@]}" -gt 0 ]] && \
|
||||
log "Already rebuilt today by docker_update.sh, skipping restart: ${!ALREADY_REBUILT_MAP[*]}"
|
||||
fi
|
||||
unset _rebuilt_age _rebuilt_stale_seconds
|
||||
fi
|
||||
|
||||
LAST_RESTARTED=""
|
||||
|
||||
for container in "${ORDERED_RESTART[@]}"; do
|
||||
@@ -196,6 +225,13 @@ for container in "${ORDERED_RESTART[@]}"; do
|
||||
|
||||
case "$STATUS" in
|
||||
true)
|
||||
if [[ -n "${ALREADY_REBUILT_MAP[$container]:-}" ]]; then
|
||||
log "$ICON_RUNNING $container already rebuilt onto new image by docker_update.sh — skipping redundant restart"
|
||||
ALREADY_UPDATED+=("$container")
|
||||
LAST_RESTARTED="$container" # it did restart, just moments ago via the rebuild
|
||||
continue
|
||||
fi
|
||||
|
||||
log "$ICON_RUNNING $container is running — restarting..."
|
||||
|
||||
# Wait if this container depends on the last one restarted
|
||||
@@ -258,8 +294,9 @@ fi
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY RESTART SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_CONTAINERS Scope: ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped, ${#FAILED[@]} failed"
|
||||
echo "$ICON_CONTAINERS Scope: ${#RESTARTED[@]} restarted, ${#ALREADY_UPDATED[@]} already updated, ${#SKIPPED[@]} skipped, ${#FAILED[@]} failed"
|
||||
[[ ${#RESTARTED[@]} -gt 0 ]] && log "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||
[[ ${#ALREADY_UPDATED[@]} -gt 0 ]] && log "$ICON_DONE Already updated (skipped): ${ALREADY_UPDATED[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
||||
@@ -268,7 +305,7 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: ALL DONE ✅"
|
||||
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped on $(hostname)" "Docker Daily Restart" "normal"
|
||||
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#ALREADY_UPDATED[@]} already updated, ${#SKIPPED[@]} skipped on $(hostname)" "Docker Daily Restart" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: ${#FAILED[@]} container(s) failed"
|
||||
notify "Daily restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Daily Restart" "warning"
|
||||
|
||||
@@ -106,6 +106,13 @@
|
||||
# MONTHLY_REMAINING_UPDATES
|
||||
# Enable or disable remainder mode. (default: true)
|
||||
#
|
||||
# DOCKER_UPDATE_REBUILT_DAILY_FILE / DOCKER_UPDATE_REBUILT_WEEKLY_FILE
|
||||
# Written after each normal/weekly run with the containers actually rebuilt
|
||||
# this pass — read by docker_daily_restart.sh / docker_weekly_restart.sh so
|
||||
# they skip restarting a container a second time right after this script
|
||||
# already rebuilt it onto the new image. Not written in remainder mode
|
||||
# (no follow-up restart script exists for it).
|
||||
#
|
||||
# PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
|
||||
# Container names for emby and critical-data profiles — excluded from
|
||||
# remainder mode (already updated by the weekly sync window)
|
||||
@@ -406,6 +413,20 @@ if [[ ${#UPDATED[@]} -gt 0 ]]; then
|
||||
done
|
||||
fi
|
||||
|
||||
# Record successfully-rebuilt containers so the follow-up restart script (daily/weekly only —
|
||||
# remainder has no follow-up) can skip them instead of restarting an already-fresh container a
|
||||
# second time. Deliberately excludes REBUILD_FAILED — those still need the restart script's
|
||||
# normal pass as a fallback, exactly as the error message above promises. Written even when
|
||||
# REBUILT is empty, so a stale file from a previous run doesn't linger and get misread later.
|
||||
if [[ "$DRY_RUN" == false && "$REMAINDER_MODE" != true ]]; then
|
||||
_rebuilt_file="$DOCKER_UPDATE_REBUILT_DAILY_FILE"
|
||||
[[ "$WEEKLY_MODE" == true ]] && _rebuilt_file="$DOCKER_UPDATE_REBUILT_WEEKLY_FILE"
|
||||
if [[ -n "$_rebuilt_file" ]]; then
|
||||
printf '%s\n' "${REBUILT[@]}" > "$_rebuilt_file" 2>/dev/null
|
||||
fi
|
||||
unset _rebuilt_file
|
||||
fi
|
||||
|
||||
# ── Remove old images ────────────────────────────────────────────────────────
|
||||
# Explicitly rmi by the IDs captured before each pull. Tagged images are never
|
||||
# caught by dangling-only prune, so this is the only reliable cleanup path.
|
||||
|
||||
@@ -84,6 +84,12 @@
|
||||
# CONTAINER_DELAY
|
||||
# Seconds to wait after restarting a dependency before starting its dependents
|
||||
#
|
||||
# DOCKER_UPDATE_REBUILT_WEEKLY_FILE / DOCKER_UPDATE_REBUILT_STALE_HOURS
|
||||
# List of containers docker_update.sh --weekly already rebuilt onto a new image
|
||||
# this run — read here so they're not restarted a second time. Discarded as
|
||||
# stale (and every container restarts normally) if older than
|
||||
# DOCKER_UPDATE_REBUILT_STALE_HOURS.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
@@ -173,10 +179,33 @@ START=$(date +%s)
|
||||
FAILED=()
|
||||
RESTARTED=()
|
||||
SKIPPED=()
|
||||
ALREADY_UPDATED=()
|
||||
|
||||
# Build dependency-safe restart order
|
||||
build_restart_order WEEKLY_RESTART_CONTAINERS
|
||||
|
||||
# ── Load containers docker_update.sh --weekly already rebuilt this run ──────────────────────────
|
||||
# Same reasoning as docker_daily_restart.sh: a container docker_update.sh already rebuilt onto a
|
||||
# new image doesn't need a plain restart right after. A file older than
|
||||
# DOCKER_UPDATE_REBUILT_STALE_HOURS is discarded as untrustworthy rather than trusted, and every
|
||||
# container restarts as normal.
|
||||
declare -A ALREADY_REBUILT_MAP
|
||||
if [[ -n "${DOCKER_UPDATE_REBUILT_WEEKLY_FILE:-}" && -f "$DOCKER_UPDATE_REBUILT_WEEKLY_FILE" ]]; then
|
||||
_rebuilt_age=$(( $(date +%s) - $(stat -c %Y "$DOCKER_UPDATE_REBUILT_WEEKLY_FILE" 2>/dev/null || echo 0) ))
|
||||
_rebuilt_stale_seconds=$(( ${DOCKER_UPDATE_REBUILT_STALE_HOURS:-12} * 3600 ))
|
||||
if [[ "$_rebuilt_age" -gt "$_rebuilt_stale_seconds" ]]; then
|
||||
warn "Rebuilt-container list is stale ($(( _rebuilt_age / 3600 ))h old) — discarding, restarting all"
|
||||
rm -f "$DOCKER_UPDATE_REBUILT_WEEKLY_FILE"
|
||||
else
|
||||
while IFS= read -r _c; do
|
||||
[[ -n "$_c" ]] && ALREADY_REBUILT_MAP["$_c"]=1
|
||||
done < "$DOCKER_UPDATE_REBUILT_WEEKLY_FILE"
|
||||
[[ "${#ALREADY_REBUILT_MAP[@]}" -gt 0 ]] && \
|
||||
log "Already rebuilt today by docker_update.sh, skipping restart: ${!ALREADY_REBUILT_MAP[*]}"
|
||||
fi
|
||||
unset _rebuilt_age _rebuilt_stale_seconds
|
||||
fi
|
||||
|
||||
LAST_RESTARTED=""
|
||||
|
||||
for container in "${ORDERED_RESTART[@]}"; do
|
||||
@@ -194,6 +223,13 @@ for container in "${ORDERED_RESTART[@]}"; do
|
||||
|
||||
case "$STATUS" in
|
||||
true)
|
||||
if [[ -n "${ALREADY_REBUILT_MAP[$container]:-}" ]]; then
|
||||
log "$ICON_RUNNING $container already rebuilt onto new image by docker_update.sh — skipping redundant restart"
|
||||
ALREADY_UPDATED+=("$container")
|
||||
LAST_RESTARTED="$container" # it did restart, just moments ago via the rebuild
|
||||
continue
|
||||
fi
|
||||
|
||||
log "$ICON_RUNNING $container is running — restarting..."
|
||||
|
||||
# Wait if this container depends on the last one restarted
|
||||
@@ -260,6 +296,7 @@ if [[ ${#RESTARTED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_STARTED Restarted: ${#RESTARTED[@]}"
|
||||
log " Names: ${RESTARTED[*]}"
|
||||
fi
|
||||
[[ ${#ALREADY_UPDATED[@]} -gt 0 ]] && log "$ICON_DONE Already updated (skipped): ${ALREADY_UPDATED[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]} (were stopped)"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
||||
@@ -268,7 +305,7 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE"
|
||||
notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped (stopped) on $(hostname)" "Docker Weekly Restart" "normal"
|
||||
notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#ALREADY_UPDATED[@]} already updated, ${#SKIPPED[@]} skipped (stopped) on $(hostname)" "Docker Weekly Restart" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed"
|
||||
notify "Weekly restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Weekly Restart" "warning"
|
||||
|
||||
Reference in New Issue
Block a user