Bring script headers onto the template and close safeguard gaps

Headers claimed protections the code never had, and several destructive paths had no
guard against a collapsed config value.
This commit is contained in:
Gmer4Lfe
2026-08-01 20:37:59 -04:00
parent cdce877601
commit e8b114094a
78 changed files with 3301 additions and 277 deletions
+63 -3
View File
@@ -14,6 +14,33 @@
# so there is no second list to maintain.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Containers are processed one at a time in dependency-safe order:
#
# 1. Build restart order
# → build_restart_order() sorts DAILY_RESTART_CONTAINERS by WATCHDOG_DEPENDENCIES
#
# 2. Skip anything docker_update.sh already rebuilt this run
# → a rebuild onto a new image already restarted it moments ago
#
# 3. Inspect container state
# missing → skip, not an error
# stopped → skip, stopped state is respected
# running → restart
#
# 4. Restart with retry
# → retry_docker wraps each attempt in a timeout, up to RETRY_COUNT
#
# 5. Verify it stayed running
# → verify_running() settles for RESTART_VERIFY_WAIT then checks State.Running
# → a container that crashes immediately is marked failed and notified
#
# 6. Prune dangling images
# → restarts swap onto new images, leaving the old ones dangling
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
@@ -37,6 +64,27 @@
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Enforcement
# Docker operations require root privileges.
#
# Docker Presence Check
# Verifies the docker binary exists before execution. Notifies on absence —
# a missing binary during the maintenance window is worth knowing about.
#
# Docker Daemon Check
# Verifies the daemon is responsive before any restart work. Every container
# would otherwise fail its inspect and be logged as an unknown-status failure,
# burying one daemon fault under a list of bogus per-container errors.
#
# Host Detection
# detect_hosts() identifies which server is running the script and aliases
# HOST*_DAILY_RESTART_CONTAINERS and HOST*_WATCHDOG_DEPENDENCIES to the
# correct host's values.
#
# Empty List Guard
# Exits cleanly with a pointer to the relevant conf key if
# DAILY_RESTART_CONTAINERS is unconfigured for this host.
#
# Dependency Ordering
# Containers restart in dependency-safe order using HOST*_WATCHDOG_DEPENDENCIES.
# CONTAINER_DELAY seconds between dependency restart and dependent restart gives
@@ -53,6 +101,11 @@
# cannot cause this script to hang indefinitely. Timed-out commands retry
# per RETRY_COUNT before marking as failed.
#
# Stale Rebuild-List Guard
# The rebuilt-container list written by docker_update.sh is discarded if older
# than DOCKER_UPDATE_REBUILT_STALE_HOURS. A stale file would otherwise suppress
# real restarts based on an update run that never happened today.
#
# Lock Acquisition
# acquire_lock() prevents concurrent execution if a previous run is still active.
#
@@ -134,6 +187,14 @@ fi
# detect_hosts() sets MY_ID and aliases HOST*_DAILY_RESTART_CONTAINERS → DAILY_RESTART_CONTAINERS
detect_hosts
# Without this, a hung daemon fails every container's inspect individually and the summary
# reports a list of unknown-status failures instead of the one fault that caused them.
if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
error "Docker daemon not responding — skipping daily restart"
notify "Daily restart skipped on $(hostname) — Docker daemon not responding" "Docker Daily Restart" "warning"
exit 1
fi
if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart"
warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
@@ -213,7 +274,7 @@ LAST_RESTARTED=""
for container in "${ORDERED_RESTART[@]}"; do
[[ -z "$container" ]] && continue
c_start=$(date +%s)
c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
c_image=$(timeout "$DOCKER_TIMEOUT" docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
@@ -242,7 +303,6 @@ for container in "${ORDERED_RESTART[@]}"; do
RESTARTED+=("$container")
else
if retry_docker docker restart "$container"; then
[[ "${RESTART_VERIFY_WAIT:-3}" -gt 0 ]] && sleep "${RESTART_VERIFY_WAIT:-3}"
if verify_running "$container"; then
echo "$ICON_STARTED $container restarted and running in $(format_duration $(( $(date +%s) - c_start )))"
RESTARTED+=("$container")
@@ -283,7 +343,7 @@ if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would prune dangling images"
PRUNED_SUMMARY="(dry run)"
else
PRUNED_OUTPUT=$(docker image prune -f 2>&1)
PRUNED_OUTPUT=$(timeout "$DOCKER_TIMEOUT" docker image prune -f 2>&1)
[[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/ /'
PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
fi