Bring script headers onto the template and close safeguard gaps
Headers claimed protections the code never had, and several destructive paths had no guard against a collapsed config value.
This commit is contained in:
@@ -14,6 +14,33 @@
|
||||
# so there is no second list to maintain.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Containers are processed one at a time in dependency-safe order:
|
||||
#
|
||||
# 1. Build restart order
|
||||
# → build_restart_order() sorts DAILY_RESTART_CONTAINERS by WATCHDOG_DEPENDENCIES
|
||||
#
|
||||
# 2. Skip anything docker_update.sh already rebuilt this run
|
||||
# → a rebuild onto a new image already restarted it moments ago
|
||||
#
|
||||
# 3. Inspect container state
|
||||
# missing → skip, not an error
|
||||
# stopped → skip, stopped state is respected
|
||||
# running → restart
|
||||
#
|
||||
# 4. Restart with retry
|
||||
# → retry_docker wraps each attempt in a timeout, up to RETRY_COUNT
|
||||
#
|
||||
# 5. Verify it stayed running
|
||||
# → verify_running() settles for RESTART_VERIFY_WAIT then checks State.Running
|
||||
# → a container that crashes immediately is marked failed and notified
|
||||
#
|
||||
# 6. Prune dangling images
|
||||
# → restarts swap onto new images, leaving the old ones dangling
|
||||
#
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
@@ -37,6 +64,27 @@
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root Enforcement
|
||||
# Docker operations require root privileges.
|
||||
#
|
||||
# Docker Presence Check
|
||||
# Verifies the docker binary exists before execution. Notifies on absence —
|
||||
# a missing binary during the maintenance window is worth knowing about.
|
||||
#
|
||||
# Docker Daemon Check
|
||||
# Verifies the daemon is responsive before any restart work. Every container
|
||||
# would otherwise fail its inspect and be logged as an unknown-status failure,
|
||||
# burying one daemon fault under a list of bogus per-container errors.
|
||||
#
|
||||
# Host Detection
|
||||
# detect_hosts() identifies which server is running the script and aliases
|
||||
# HOST*_DAILY_RESTART_CONTAINERS and HOST*_WATCHDOG_DEPENDENCIES to the
|
||||
# correct host's values.
|
||||
#
|
||||
# Empty List Guard
|
||||
# Exits cleanly with a pointer to the relevant conf key if
|
||||
# DAILY_RESTART_CONTAINERS is unconfigured for this host.
|
||||
#
|
||||
# Dependency Ordering
|
||||
# Containers restart in dependency-safe order using HOST*_WATCHDOG_DEPENDENCIES.
|
||||
# CONTAINER_DELAY seconds between dependency restart and dependent restart gives
|
||||
@@ -53,6 +101,11 @@
|
||||
# cannot cause this script to hang indefinitely. Timed-out commands retry
|
||||
# per RETRY_COUNT before marking as failed.
|
||||
#
|
||||
# Stale Rebuild-List Guard
|
||||
# The rebuilt-container list written by docker_update.sh is discarded if older
|
||||
# than DOCKER_UPDATE_REBUILT_STALE_HOURS. A stale file would otherwise suppress
|
||||
# real restarts based on an update run that never happened today.
|
||||
#
|
||||
# Lock Acquisition
|
||||
# acquire_lock() prevents concurrent execution if a previous run is still active.
|
||||
#
|
||||
@@ -134,6 +187,14 @@ fi
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_DAILY_RESTART_CONTAINERS → DAILY_RESTART_CONTAINERS
|
||||
detect_hosts
|
||||
|
||||
# Without this, a hung daemon fails every container's inspect individually and the summary
|
||||
# reports a list of unknown-status failures instead of the one fault that caused them.
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
|
||||
error "Docker daemon not responding — skipping daily restart"
|
||||
notify "Daily restart skipped on $(hostname) — Docker daemon not responding" "Docker Daily Restart" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart"
|
||||
warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
|
||||
@@ -213,7 +274,7 @@ LAST_RESTARTED=""
|
||||
for container in "${ORDERED_RESTART[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
c_start=$(date +%s)
|
||||
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||
c_image=$(timeout "$DOCKER_TIMEOUT" docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
|
||||
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
|
||||
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
||||
@@ -242,7 +303,6 @@ for container in "${ORDERED_RESTART[@]}"; do
|
||||
RESTARTED+=("$container")
|
||||
else
|
||||
if retry_docker docker restart "$container"; then
|
||||
[[ "${RESTART_VERIFY_WAIT:-3}" -gt 0 ]] && sleep "${RESTART_VERIFY_WAIT:-3}"
|
||||
if verify_running "$container"; then
|
||||
echo "$ICON_STARTED $container restarted and running in $(format_duration $(( $(date +%s) - c_start ))) ✅"
|
||||
RESTARTED+=("$container")
|
||||
@@ -283,7 +343,7 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would prune dangling images"
|
||||
PRUNED_SUMMARY="(dry run)"
|
||||
else
|
||||
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
|
||||
PRUNED_OUTPUT=$(timeout "$DOCKER_TIMEOUT" docker image prune -f 2>&1)
|
||||
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /'
|
||||
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user