fix: remove container check from stability_watchdog, add USB boot warning
stability_watchdog: container health is owned by docker_watchdog — rebooting when docker_watchdog skip-lists a container creates a reboot loop. Removed SYS_WATCHDOG_CHECK_CONTAINERS check entirely. docker_watchdog handles strikes, restart attempts, skip-listing, and notifications. varaverk.plg: warn on USB/removable boot device at install time. Plugin is designed for internal NVMe/SSD boot. Warning is non-blocking but clear. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
b9b1d20499
commit
9c68429e25
@@ -86,6 +86,21 @@ log() { echo "[$(date '+%H:%M:%S')] $*" | tee -a "$LOG"; }
|
||||
SCRIPTS_DIR="$CFG_DIR"
|
||||
CONF_DIR="$SCRIPTS_DIR/Configurations"
|
||||
|
||||
# ── Boot device check ─────────────────────────────────────────────────────────
|
||||
# Warn if /boot is on a USB/removable device. Varaverk is designed for internal
|
||||
# NVMe/SSD boot — git repo + state files + data writes on USB will wear it out
|
||||
# fast and may run out of space. Install proceeds but user is warned.
|
||||
_boot_dev=$(df /boot --output=source 2>/dev/null | tail -1)
|
||||
_boot_base=$(lsblk -no pkname "$_boot_dev" 2>/dev/null || basename "${_boot_dev%[0-9p]*}")
|
||||
_removable=$(cat "/sys/block/${_boot_base}/removable" 2>/dev/null || echo "0")
|
||||
if [[ "$_removable" == "1" ]]; then
|
||||
log "WARNING: /boot is on a removable/USB device ($_boot_dev)"
|
||||
log "WARNING: Varaverk is designed for internal NVMe/SSD boot."
|
||||
log "WARNING: Running from USB risks drive wear and space exhaustion."
|
||||
log "WARNING: Strongly recommend migrating boot to an internal NVMe/SSD drive."
|
||||
fi
|
||||
unset _boot_dev _boot_base _removable
|
||||
|
||||
# Seed varaverk.cfg with defaults (SCRIPTS_DIR + Gitea settings) if not present.
|
||||
# Requires internal NVMe/SSD boot — scripts live on flash, available before array mounts.
|
||||
if [[ ! -f "$CFG_FILE" ]]; then
|
||||
|
||||
@@ -50,7 +50,8 @@
|
||||
# Checks Run Every Cycle
|
||||
# rootfs usage, /var/log, /tmp, free RAM, ZFS ARC, CPU temp, load avg,
|
||||
# zombie processes, Docker daemon, OOM rate, /boot read-only, kernel oops,
|
||||
# file descriptor exhaustion, array disk errors, NIC state, required containers.
|
||||
# file descriptor exhaustion, array disk errors, NIC state.
|
||||
# (Container health is owned by docker_watchdog — not checked here.)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
@@ -84,7 +85,6 @@
|
||||
#
|
||||
# SYS_WATCHDOG_STATE_FILE — strike counters and cycle state
|
||||
# SYS_WATCHDOG_REBOOT_LOG — reboot history for rate limiting
|
||||
# SYS_WATCHDOG_FAILED_FILE — containers confirmed down for skip list integration
|
||||
# SYS_WATCHDOG_OOM_FILE — OOM kill counter from previous cycle
|
||||
#
|
||||
# ==============================================================================================
|
||||
@@ -766,22 +766,10 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Required containers from docker_watchdog skip list ────────────────────────────────────
|
||||
if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
FAILED_CONTAINERS=()
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
[[ "$STATUS" != "true" ]] && FAILED_CONTAINERS+=("$container")
|
||||
done < "$SYS_WATCHDOG_FAILED_FILE"
|
||||
|
||||
TRIGGERED=false
|
||||
[[ ${#FAILED_CONTAINERS[@]} -gt 0 ]] && TRIGGERED=true
|
||||
run_strike_check "failed_containers" "$TRIGGERED" \
|
||||
"required containers stopped: ${FAILED_CONTAINERS[*]:-}" && \
|
||||
TRIGGERS+=("containers=${FAILED_CONTAINERS[*]:-}")
|
||||
fi
|
||||
# ── Required containers — removed from stability_watchdog ────────────────────────────────
|
||||
# Container health is owned entirely by docker_watchdog: strike system, restart attempts,
|
||||
# skip-listing, and critical notifications. Rebooting here when docker_watchdog already
|
||||
# gave up creates a reboot loop — the container is still broken after reboot.
|
||||
|
||||
# ==========================================================================================
|
||||
# ━━━ Evaluate Standard Triggers ━━━
|
||||
|
||||
Reference in New Issue
Block a user