fix: remove container check from stability_watchdog, add USB boot warning

stability_watchdog: container health is owned by docker_watchdog — rebooting
when docker_watchdog skip-lists a container creates a reboot loop. Removed
SYS_WATCHDOG_CHECK_CONTAINERS check entirely. docker_watchdog handles
strikes, restart attempts, skip-listing, and notifications.

varaverk.plg: warn on USB/removable boot device at install time. Plugin is
designed for internal NVMe/SSD boot. Warning is non-blocking but clear.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Gmer4Lfe
2026-05-31 14:30:48 -04:00
co-authored by Claude Sonnet 4.6
parent b9b1d20499
commit 9c68429e25
2 changed files with 21 additions and 18 deletions
+15
View File
@@ -86,6 +86,21 @@ log() { echo "[$(date '+%H:%M:%S')] $*" | tee -a "$LOG"; }
SCRIPTS_DIR="$CFG_DIR" SCRIPTS_DIR="$CFG_DIR"
CONF_DIR="$SCRIPTS_DIR/Configurations" CONF_DIR="$SCRIPTS_DIR/Configurations"
# ── Boot device check ─────────────────────────────────────────────────────────
# Warn if /boot is on a USB/removable device. Varaverk is designed for internal
# NVMe/SSD boot — git repo + state files + data writes on USB will wear it out
# fast and may run out of space. Install proceeds but user is warned.
_boot_dev=$(df /boot --output=source 2>/dev/null | tail -1)
_boot_base=$(lsblk -no pkname "$_boot_dev" 2>/dev/null || basename "${_boot_dev%[0-9p]*}")
_removable=$(cat "/sys/block/${_boot_base}/removable" 2>/dev/null || echo "0")
if [[ "$_removable" == "1" ]]; then
log "WARNING: /boot is on a removable/USB device ($_boot_dev)"
log "WARNING: Varaverk is designed for internal NVMe/SSD boot."
log "WARNING: Running from USB risks drive wear and space exhaustion."
log "WARNING: Strongly recommend migrating boot to an internal NVMe/SSD drive."
fi
unset _boot_dev _boot_base _removable
# Seed varaverk.cfg with defaults (SCRIPTS_DIR + Gitea settings) if not present. # Seed varaverk.cfg with defaults (SCRIPTS_DIR + Gitea settings) if not present.
# Requires internal NVMe/SSD boot — scripts live on flash, available before array mounts. # Requires internal NVMe/SSD boot — scripts live on flash, available before array mounts.
if [[ ! -f "$CFG_FILE" ]]; then if [[ ! -f "$CFG_FILE" ]]; then
+6 -18
View File
@@ -50,7 +50,8 @@
# Checks Run Every Cycle # Checks Run Every Cycle
# rootfs usage, /var/log, /tmp, free RAM, ZFS ARC, CPU temp, load avg, # rootfs usage, /var/log, /tmp, free RAM, ZFS ARC, CPU temp, load avg,
# zombie processes, Docker daemon, OOM rate, /boot read-only, kernel oops, # zombie processes, Docker daemon, OOM rate, /boot read-only, kernel oops,
# file descriptor exhaustion, array disk errors, NIC state, required containers. # file descriptor exhaustion, array disk errors, NIC state.
# (Container health is owned by docker_watchdog — not checked here.)
# #
# ============================================================================================== # ==============================================================================================
# OPERATIONAL SAFEGUARDS # OPERATIONAL SAFEGUARDS
@@ -84,7 +85,6 @@
# #
# SYS_WATCHDOG_STATE_FILE — strike counters and cycle state # SYS_WATCHDOG_STATE_FILE — strike counters and cycle state
# SYS_WATCHDOG_REBOOT_LOG — reboot history for rate limiting # SYS_WATCHDOG_REBOOT_LOG — reboot history for rate limiting
# SYS_WATCHDOG_FAILED_FILE — containers confirmed down for skip list integration
# SYS_WATCHDOG_OOM_FILE — OOM kill counter from previous cycle # SYS_WATCHDOG_OOM_FILE — OOM kill counter from previous cycle
# #
# ============================================================================================== # ==============================================================================================
@@ -766,22 +766,10 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
fi fi
fi fi
# ── Required containers from docker_watchdog skip list ──────────────────────────────────── # ── Required containers — removed from stability_watchdog ────────────────────────────────
if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then # Container health is owned entirely by docker_watchdog: strike system, restart attempts,
FAILED_CONTAINERS=() # skip-listing, and critical notifications. Rebooting here when docker_watchdog already
while IFS= read -r container; do # gave up creates a reboot loop — the container is still broken after reboot.
[[ -z "$container" ]] && continue
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
'{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
[[ "$STATUS" != "true" ]] && FAILED_CONTAINERS+=("$container")
done < "$SYS_WATCHDOG_FAILED_FILE"
TRIGGERED=false
[[ ${#FAILED_CONTAINERS[@]} -gt 0 ]] && TRIGGERED=true
run_strike_check "failed_containers" "$TRIGGERED" \
"required containers stopped: ${FAILED_CONTAINERS[*]:-}" && \
TRIGGERS+=("containers=${FAILED_CONTAINERS[*]:-}")
fi
# ========================================================================================== # ==========================================================================================
# ━━━ Evaluate Standard Triggers ━━━ # ━━━ Evaluate Standard Triggers ━━━