fix: remove container check from stability_watchdog, add USB boot warning
stability_watchdog: container health is owned by docker_watchdog — rebooting when docker_watchdog skip-lists a container creates a reboot loop. Removed SYS_WATCHDOG_CHECK_CONTAINERS check entirely. docker_watchdog handles strikes, restart attempts, skip-listing, and notifications. varaverk.plg: warn on USB/removable boot device at install time. Plugin is designed for internal NVMe/SSD boot. Warning is non-blocking but clear. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
b9b1d20499
commit
9c68429e25
@@ -86,6 +86,21 @@ log() { echo "[$(date '+%H:%M:%S')] $*" | tee -a "$LOG"; }
|
|||||||
SCRIPTS_DIR="$CFG_DIR"
|
SCRIPTS_DIR="$CFG_DIR"
|
||||||
CONF_DIR="$SCRIPTS_DIR/Configurations"
|
CONF_DIR="$SCRIPTS_DIR/Configurations"
|
||||||
|
|
||||||
|
# ── Boot device check ─────────────────────────────────────────────────────────
|
||||||
|
# Warn if /boot is on a USB/removable device. Varaverk is designed for internal
|
||||||
|
# NVMe/SSD boot — git repo + state files + data writes on USB will wear it out
|
||||||
|
# fast and may run out of space. Install proceeds but user is warned.
|
||||||
|
_boot_dev=$(df /boot --output=source 2>/dev/null | tail -1)
|
||||||
|
_boot_base=$(lsblk -no pkname "$_boot_dev" 2>/dev/null || basename "${_boot_dev%[0-9p]*}")
|
||||||
|
_removable=$(cat "/sys/block/${_boot_base}/removable" 2>/dev/null || echo "0")
|
||||||
|
if [[ "$_removable" == "1" ]]; then
|
||||||
|
log "WARNING: /boot is on a removable/USB device ($_boot_dev)"
|
||||||
|
log "WARNING: Varaverk is designed for internal NVMe/SSD boot."
|
||||||
|
log "WARNING: Running from USB risks drive wear and space exhaustion."
|
||||||
|
log "WARNING: Strongly recommend migrating boot to an internal NVMe/SSD drive."
|
||||||
|
fi
|
||||||
|
unset _boot_dev _boot_base _removable
|
||||||
|
|
||||||
# Seed varaverk.cfg with defaults (SCRIPTS_DIR + Gitea settings) if not present.
|
# Seed varaverk.cfg with defaults (SCRIPTS_DIR + Gitea settings) if not present.
|
||||||
# Requires internal NVMe/SSD boot — scripts live on flash, available before array mounts.
|
# Requires internal NVMe/SSD boot — scripts live on flash, available before array mounts.
|
||||||
if [[ ! -f "$CFG_FILE" ]]; then
|
if [[ ! -f "$CFG_FILE" ]]; then
|
||||||
|
|||||||
@@ -50,7 +50,8 @@
|
|||||||
# Checks Run Every Cycle
|
# Checks Run Every Cycle
|
||||||
# rootfs usage, /var/log, /tmp, free RAM, ZFS ARC, CPU temp, load avg,
|
# rootfs usage, /var/log, /tmp, free RAM, ZFS ARC, CPU temp, load avg,
|
||||||
# zombie processes, Docker daemon, OOM rate, /boot read-only, kernel oops,
|
# zombie processes, Docker daemon, OOM rate, /boot read-only, kernel oops,
|
||||||
# file descriptor exhaustion, array disk errors, NIC state, required containers.
|
# file descriptor exhaustion, array disk errors, NIC state.
|
||||||
|
# (Container health is owned by docker_watchdog — not checked here.)
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# OPERATIONAL SAFEGUARDS
|
# OPERATIONAL SAFEGUARDS
|
||||||
@@ -84,7 +85,6 @@
|
|||||||
#
|
#
|
||||||
# SYS_WATCHDOG_STATE_FILE — strike counters and cycle state
|
# SYS_WATCHDOG_STATE_FILE — strike counters and cycle state
|
||||||
# SYS_WATCHDOG_REBOOT_LOG — reboot history for rate limiting
|
# SYS_WATCHDOG_REBOOT_LOG — reboot history for rate limiting
|
||||||
# SYS_WATCHDOG_FAILED_FILE — containers confirmed down for skip list integration
|
|
||||||
# SYS_WATCHDOG_OOM_FILE — OOM kill counter from previous cycle
|
# SYS_WATCHDOG_OOM_FILE — OOM kill counter from previous cycle
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -766,22 +766,10 @@ echo "━━━ $ICON_REBOOT Stability Watchdog — $(date '+%Y-%m-%d %H:%M:%S')
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ── Required containers from docker_watchdog skip list ────────────────────────────────────
|
# ── Required containers — removed from stability_watchdog ────────────────────────────────
|
||||||
if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
# Container health is owned entirely by docker_watchdog: strike system, restart attempts,
|
||||||
FAILED_CONTAINERS=()
|
# skip-listing, and critical notifications. Rebooting here when docker_watchdog already
|
||||||
while IFS= read -r container; do
|
# gave up creates a reboot loop — the container is still broken after reboot.
|
||||||
[[ -z "$container" ]] && continue
|
|
||||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
|
||||||
'{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
|
||||||
[[ "$STATUS" != "true" ]] && FAILED_CONTAINERS+=("$container")
|
|
||||||
done < "$SYS_WATCHDOG_FAILED_FILE"
|
|
||||||
|
|
||||||
TRIGGERED=false
|
|
||||||
[[ ${#FAILED_CONTAINERS[@]} -gt 0 ]] && TRIGGERED=true
|
|
||||||
run_strike_check "failed_containers" "$TRIGGERED" \
|
|
||||||
"required containers stopped: ${FAILED_CONTAINERS[*]:-}" && \
|
|
||||||
TRIGGERS+=("containers=${FAILED_CONTAINERS[*]:-}")
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ==========================================================================================
|
# ==========================================================================================
|
||||||
# ━━━ Evaluate Standard Triggers ━━━
|
# ━━━ Evaluate Standard Triggers ━━━
|
||||||
|
|||||||
Reference in New Issue
Block a user