Gmer4Lfe
9425d16c19
docker_update.sh rebuilds (stop+recreate) any container whose image changed, in every mode — but for daily/weekly that was always followed by the restart script's own unconditional pass, stopping and starting the same container twice back to back. docker_update.sh now records which containers it rebuilt this run to a file; docker_daily_restart.sh and docker_weekly_restart.sh read it and skip those specifically, still restarting everything else as before. A file older than DOCKER_UPDATE_REBUILT_STALE_HOURS (default 12) is discarded rather than trusted, so a missed or failed update run can't suppress a restart indefinitely.
316 lines
15 KiB
Bash
Executable File
316 lines
15 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= Docker Daily Restart =======================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Restarts configured containers every night at 1am as proactive maintenance.
|
|
#
|
|
# Called by daily_sync_maintenance.sh via DAILY_MAINTENANCE_SCRIPTS. Runs inside
|
|
# the daily maintenance window — any service downtime is absorbed by a window
|
|
# that is already happening. Also drives docker_update.sh in normal mode: the
|
|
# same DAILY_RESTART_CONTAINERS list is used for both restarts and image pulls,
|
|
# so there is no second list to maintain.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Proactive Maintenance
|
|
# Daily restarts target containers known to degrade over time without
|
|
# crossing a clear failure threshold — connection table growth, scheduler
|
|
# state accumulation, session cache bloat. The watchdog cannot detect this
|
|
# class of degradation. Scheduled restarts clear it before it becomes visible.
|
|
#
|
|
# State Respect
|
|
# Running containers are restarted. Stopped containers are left stopped — they
|
|
# were intentionally halted and this script has no authority to override that
|
|
# decision. This rule is consistent across the entire ecosystem.
|
|
#
|
|
# Dependency-Safe Ordering
|
|
# Restarts follow the same dependency ordering used by docker_watchdog.sh.
|
|
# Services that other containers depend on restart first. A dependent is never
|
|
# restarted while its dependency is still coming up.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Dependency Ordering
|
|
# Containers restart in dependency-safe order using HOST*_WATCHDOG_DEPENDENCIES.
|
|
# CONTAINER_DELAY seconds between dependency restart and dependent restart gives
|
|
# the dependency time to fully initialise before dependents try to connect.
|
|
#
|
|
# Restart Verification
|
|
# After each restart, container state is checked after a settle period. A
|
|
# container that starts and immediately crashes is marked failed and a
|
|
# notification is sent — the script does not silently pass a restart that
|
|
# did not stick.
|
|
#
|
|
# Timeout Protection
|
|
# All docker commands wrapped in a 30 second timeout. A hung Docker daemon
|
|
# cannot cause this script to hang indefinitely. Timed-out commands retry
|
|
# per RETRY_COUNT before marking as failed.
|
|
#
|
|
# Lock Acquisition
|
|
# acquire_lock() prevents concurrent execution if a previous run is still active.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_DAILY_RESTART_CONTAINERS
|
|
# Containers restarted nightly. Also used by docker_update.sh normal mode
|
|
# for image pulls — add a container once, it gets both. Aliased by
|
|
# detect_hosts() → DAILY_RESTART_CONTAINERS
|
|
#
|
|
# HOST*_WATCHDOG_DEPENDENCIES
|
|
# Dependency ordering shared with docker_watchdog.sh. Aliased by
|
|
# detect_hosts() → WATCHDOG_DEPENDENCIES
|
|
#
|
|
# master.conf
|
|
#
|
|
# RETRY_COUNT
|
|
# Retry attempts before giving up on a container
|
|
#
|
|
# SLEEP
|
|
# Seconds between retry attempts
|
|
#
|
|
# CONTAINER_DELAY
|
|
# Seconds to wait after restarting a dependency before starting its dependents
|
|
#
|
|
# RESTART_VERIFY_WAIT
|
|
# Seconds to wait after docker restart before checking the container is running.
|
|
# Gives the process time to initialise before verify_running samples the state.
|
|
# (default: 3)
|
|
#
|
|
# DOCKER_UPDATE_REBUILT_DAILY_FILE / DOCKER_UPDATE_REBUILT_STALE_HOURS
|
|
# List of containers docker_update.sh already rebuilt onto a new image this run —
|
|
# read here so they're not restarted a second time. Discarded as stale (and every
|
|
# container restarts normally) if older than DOCKER_UPDATE_REBUILT_STALE_HOURS.
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# docker_daily_restart.sh
|
|
# Restart all containers in DAILY_RESTART_CONTAINERS
|
|
#
|
|
# docker_daily_restart.sh --dry-run
|
|
# Preview which containers would be restarted and which would be skipped
|
|
#
|
|
# docker_daily_restart.sh --status
|
|
# Show configured restart list, container states, and dependency ordering
|
|
#
|
|
# docker_daily_restart.sh --log
|
|
# Verbose per-container execution output
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found — check PATH or Docker installation"
|
|
notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "warning"
|
|
exit 1
|
|
fi
|
|
|
|
# detect_hosts() sets MY_ID and aliases HOST*_DAILY_RESTART_CONTAINERS → DAILY_RESTART_CONTAINERS
|
|
detect_hosts
|
|
|
|
if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
|
warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart"
|
|
warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}"
|
|
echo "$ICON_RETRY Retries: $RETRY_COUNT"
|
|
echo "$ICON_TIME Sleep: ${SLEEP}s between retries"
|
|
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
|
|
|
# ==============================================================================================
|
|
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
|
|
# ==============================================================================================
|
|
|
|
# docker_cmd, verify_running, retry_docker — defined in common.sh
|
|
|
|
# build_restart_order() / check_dependency_delay() — provided by common.sh
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Daily Restart ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_CONTAINERS Daily Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) — ${#DAILY_RESTART_CONTAINERS[@]} container(s)"
|
|
log "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}"
|
|
log "$ICON_RETRY Retries: $RETRY_COUNT"
|
|
log "$ICON_GEAR Config: sleep=${SLEEP}s delay=${CONTAINER_DELAY}s verify-wait=${RESTART_VERIFY_WAIT}s cmd-timeout=${DOCKER_TIMEOUT}s"
|
|
|
|
START=$(date +%s)
|
|
FAILED=()
|
|
RESTARTED=()
|
|
SKIPPED=()
|
|
ALREADY_UPDATED=()
|
|
|
|
# Build dependency-safe restart order
|
|
build_restart_order DAILY_RESTART_CONTAINERS
|
|
|
|
# ── Load containers docker_update.sh already rebuilt this run ───────────────────────────────────
|
|
# docker_update.sh's rebuild (stop+recreate onto a new image) already restarted anything whose
|
|
# image changed today — doing a plain restart on it again here is redundant. A file older than
|
|
# DOCKER_UPDATE_REBUILT_STALE_HOURS means docker_update.sh either didn't run today or this is way
|
|
# out of sync with it, so it's discarded rather than trusted, and every container restarts as
|
|
# normal — same as if the file had never existed.
|
|
declare -A ALREADY_REBUILT_MAP
|
|
if [[ -n "${DOCKER_UPDATE_REBUILT_DAILY_FILE:-}" && -f "$DOCKER_UPDATE_REBUILT_DAILY_FILE" ]]; then
|
|
_rebuilt_age=$(( $(date +%s) - $(stat -c %Y "$DOCKER_UPDATE_REBUILT_DAILY_FILE" 2>/dev/null || echo 0) ))
|
|
_rebuilt_stale_seconds=$(( ${DOCKER_UPDATE_REBUILT_STALE_HOURS:-12} * 3600 ))
|
|
if [[ "$_rebuilt_age" -gt "$_rebuilt_stale_seconds" ]]; then
|
|
warn "Rebuilt-container list is stale ($(( _rebuilt_age / 3600 ))h old) — discarding, restarting all"
|
|
rm -f "$DOCKER_UPDATE_REBUILT_DAILY_FILE"
|
|
else
|
|
while IFS= read -r _c; do
|
|
[[ -n "$_c" ]] && ALREADY_REBUILT_MAP["$_c"]=1
|
|
done < "$DOCKER_UPDATE_REBUILT_DAILY_FILE"
|
|
[[ "${#ALREADY_REBUILT_MAP[@]}" -gt 0 ]] && \
|
|
log "Already rebuilt today by docker_update.sh, skipping restart: ${!ALREADY_REBUILT_MAP[*]}"
|
|
fi
|
|
unset _rebuilt_age _rebuilt_stale_seconds
|
|
fi
|
|
|
|
LAST_RESTARTED=""
|
|
|
|
for container in "${ORDERED_RESTART[@]}"; do
|
|
[[ -z "$container" ]] && continue
|
|
c_start=$(date +%s)
|
|
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
|
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
|
|
|
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
|
warn "$container does not exist — skipping"
|
|
continue
|
|
fi
|
|
|
|
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
|
|
|
case "$STATUS" in
|
|
true)
|
|
if [[ -n "${ALREADY_REBUILT_MAP[$container]:-}" ]]; then
|
|
log "$ICON_RUNNING $container already rebuilt onto new image by docker_update.sh — skipping redundant restart"
|
|
ALREADY_UPDATED+=("$container")
|
|
LAST_RESTARTED="$container" # it did restart, just moments ago via the rebuild
|
|
continue
|
|
fi
|
|
|
|
log "$ICON_RUNNING $container is running — restarting..."
|
|
|
|
# Wait if this container depends on the last one restarted
|
|
check_dependency_delay "$container" "$LAST_RESTARTED"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would restart $container"
|
|
RESTARTED+=("$container")
|
|
else
|
|
if retry_docker docker restart "$container"; then
|
|
[[ "${RESTART_VERIFY_WAIT:-3}" -gt 0 ]] && sleep "${RESTART_VERIFY_WAIT:-3}"
|
|
if verify_running "$container"; then
|
|
echo "$ICON_STARTED $container restarted and running in $(format_duration $(( $(date +%s) - c_start ))) ✅"
|
|
RESTARTED+=("$container")
|
|
LAST_RESTARTED="$container"
|
|
else
|
|
error "$container restarted but crashed immediately"
|
|
notify "$container crashed after restart on $(hostname)" "Docker Daily Restart" "warning"
|
|
FAILED+=("$container")
|
|
fi
|
|
else
|
|
error "Failed to restart $container after $RETRY_COUNT attempts"
|
|
notify "$container failed to restart on $(hostname)" "Docker Daily Restart" "warning"
|
|
FAILED+=("$container")
|
|
fi
|
|
fi
|
|
;;
|
|
false)
|
|
log "$ICON_NOT_RUNNING $container is stopped — skipping"
|
|
SKIPPED+=("$container")
|
|
;;
|
|
*)
|
|
error "Unknown status for $container: $STATUS"
|
|
FAILED+=("$container")
|
|
;;
|
|
esac
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Prune Old Images ━━━
|
|
# ==============================================================================================
|
|
# Restarts above swap containers onto new images — old images are now dangling. Prune immediately.
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Pruning Dangling Images — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would prune dangling images"
|
|
PRUNED_SUMMARY="(dry run)"
|
|
else
|
|
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
|
|
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /'
|
|
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo "━━━━━ $ICON_SUMMARY DAILY RESTART SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
|
echo "$ICON_CONTAINERS Scope: ${#RESTARTED[@]} restarted, ${#ALREADY_UPDATED[@]} already updated, ${#SKIPPED[@]} skipped, ${#FAILED[@]} failed"
|
|
[[ ${#RESTARTED[@]} -gt 0 ]] && log "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
|
[[ ${#ALREADY_UPDATED[@]} -gt 0 ]] && log "$ICON_DONE Already updated (skipped): ${ALREADY_UPDATED[*]}"
|
|
[[ ${#SKIPPED[@]} -gt 0 ]] && log "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]}"
|
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
|
echo "$ICON_SYNC Pruned: ${PRUNED_SUMMARY:-none}"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no changes made"
|
|
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
|
echo "$ICON_DONE Status: ALL DONE ✅"
|
|
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#ALREADY_UPDATED[@]} already updated, ${#SKIPPED[@]} skipped on $(hostname)" "Docker Daily Restart" "normal"
|
|
else
|
|
echo "$ICON_ERROR Status: ${#FAILED[@]} container(s) failed"
|
|
notify "Daily restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Daily Restart" "warning"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
|
exit 0 |