Bring script headers onto the template and close safeguard gaps

Headers claimed protections the code never had, and several destructive paths had no
guard against a collapsed config value.

Docker_Essentials/docker_weekly_restart.sh

@@ -16,6 +16,33 @@
 # stopped → leave, missing → skip. Container state is always respected.
 #
 # ==============================================================================================
+# OPERATIONAL MODEL
+# ==============================================================================================
+#
+# Identical to docker_daily_restart.sh, against WEEKLY_RESTART_CONTAINERS:
+#
+#   1. Build restart order
+#        → build_restart_order() sorts WEEKLY_RESTART_CONTAINERS by WATCHDOG_DEPENDENCIES
+#
+#   2. Skip anything docker_update.sh --weekly already rebuilt this run
+#        → a rebuild onto a new image already restarted it moments ago
+#
+#   3. Inspect container state
+#        missing  → skip, not an error
+#        stopped  → skip, stopped state is respected
+#        running  → restart
+#
+#   4. Restart with retry
+#        → retry_docker wraps each attempt in a timeout, up to RETRY_COUNT
+#
+#   5. Verify it stayed running
+#        → verify_running() settles for RESTART_VERIFY_WAIT then checks State.Running
+#        → a container that crashes immediately is marked failed and notified
+#
+#   6. Prune dangling images
+#        → restarts swap onto new images, leaving the old ones dangling
+#
+# ==============================================================================================
 # DESIGN PRINCIPLES
 # ==============================================================================================
 #
@@ -39,6 +66,26 @@
 # OPERATIONAL SAFEGUARDS
 # ==============================================================================================
 #
+# Root Enforcement
+#   Docker operations require root privileges.
+#
+# Docker Presence Check
+#   Verifies the docker binary exists before execution. Notifies on absence.
+#
+# Docker Daemon Check
+#   Verifies the daemon is responsive before any restart work. Every container
+#   would otherwise fail its inspect and be logged as an unknown-status failure,
+#   burying one daemon fault under a list of bogus per-container errors.
+#
+# Host Detection
+#   detect_hosts() identifies which server is running the script and aliases
+#   HOST*_WEEKLY_RESTART_CONTAINERS and HOST*_WATCHDOG_DEPENDENCIES to the
+#   correct host's values.
+#
+# Empty List Guard
+#   Exits cleanly with a pointer to the relevant conf key if
+#   WEEKLY_RESTART_CONTAINERS is unconfigured for this host.
+#
 # Dependency Ordering
 #   Containers restart in dependency-safe order using HOST*_WATCHDOG_DEPENDENCIES.
 #   CONTAINER_DELAY seconds between dependency restart and dependent restart.
@@ -51,10 +98,10 @@
 #   All docker commands wrapped in a 30 second timeout. A hung Docker daemon
 #   cannot cause this script to hang indefinitely.
 #
-# Host Detection
-#   detect_hosts() identifies which server is running the script and aliases
-#   HOST*_WEEKLY_RESTART_CONTAINERS and HOST*_WATCHDOG_DEPENDENCIES to the
-#   correct host's values.
+# Stale Rebuild-List Guard
+#   The rebuilt-container list written by docker_update.sh --weekly is discarded
+#   if older than DOCKER_UPDATE_REBUILT_STALE_HOURS. A stale file would otherwise
+#   suppress real restarts based on an update run that never happened this week.
 #
 # Lock Acquisition
 #   acquire_lock() prevents concurrent execution.
@@ -84,6 +131,11 @@
 #   CONTAINER_DELAY
 #       Seconds to wait after restarting a dependency before starting its dependents
 #
+#   RESTART_VERIFY_WAIT
+#       Seconds verify_running() waits after docker restart before checking the
+#       container is running. Gives the process time to initialise before the
+#       state is sampled. (default: 3)
+#
 #   DOCKER_UPDATE_REBUILT_WEEKLY_FILE / DOCKER_UPDATE_REBUILT_STALE_HOURS
 #       List of containers docker_update.sh --weekly already rebuilt onto a new image
 #       this run — read here so they're not restarted a second time. Discarded as
@@ -132,6 +184,14 @@ fi
 # detect_hosts() sets MY_ID and aliases HOST*_WEEKLY_RESTART_CONTAINERS → WEEKLY_RESTART_CONTAINERS
 detect_hosts
 
+# Without this, a hung daemon fails every container's inspect individually and the summary
+# reports a list of unknown-status failures instead of the one fault that caused them.
+if ! timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
+    error "Docker daemon not responding — skipping weekly restart"
+    notify "Weekly restart skipped on $(hostname) — Docker daemon not responding" "Docker Weekly Restart" "warning"
+    exit 1
+fi
+
 if [[ ${#WEEKLY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
     warn "WEEKLY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart"
     warn "Check HOST*_WEEKLY_RESTART_CONTAINERS in host*.conf"
@@ -211,7 +271,7 @@ LAST_RESTARTED=""
 for container in "${ORDERED_RESTART[@]}"; do
     [[ -z "$container" ]] && continue
     c_start=$(date +%s)
-    c_image=$(docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
+    c_image=$(timeout "$DOCKER_TIMEOUT" docker inspect --format '{{.Config.Image}}' "$container" 2>/dev/null || echo "unknown")
     log "━━━ $ICON_CONTAINERS $container ($c_image) ━━━"
 
     if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
@@ -281,7 +341,7 @@ if [[ "$DRY_RUN" == true ]]; then
     warn "DRY RUN — would prune dangling images"
     PRUNED_SUMMARY="(dry run)"
 else
-    PRUNED_OUTPUT=$(docker image prune -f 2>&1)
+    PRUNED_OUTPUT=$(timeout "$DOCKER_TIMEOUT" docker image prune -f 2>&1)
     [[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/  /'
     PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
 fi