rename: SYS_WATCHDOG_FAILED_FILE → DOCKER_WATCHDOG_FAILED_FILE
Skip list belongs to docker_watchdog, not stability_watchdog (legacy naming from when docker+system were the only two watchdogs). Rename variable and physical file system_watchdog_failed.db → docker_watchdog_failed.db across all files. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
9d6772fbd6
commit
070f9caf7f
@@ -1347,7 +1347,7 @@
|
||||
|
||||
# ━━━ State Files ━━━
|
||||
SYS_WATCHDOG_STATE_FILE="$STATE_DIR/system_watchdog_state.db"
|
||||
SYS_WATCHDOG_FAILED_FILE="$STATE_DIR/system_watchdog_failed.db"
|
||||
DOCKER_WATCHDOG_FAILED_FILE="$STATE_DIR/docker_watchdog_failed.db"
|
||||
SYS_WATCHDOG_REBOOT_LOG="$STATE_DIR/system_watchdog_reboots.db"
|
||||
SYS_WATCHDOG_OOM_FILE="$STATE_DIR/system_watchdog_oom.db"
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@
|
||||
#
|
||||
# Data sources (reads only):
|
||||
# FALLBACK_STATE_FILE — current fallback state
|
||||
# SYS_WATCHDOG_FAILED_FILE — container skip list (manual intervention needed)
|
||||
# DOCKER_WATCHDOG_FAILED_FILE — container skip list (manual intervention needed)
|
||||
# WATCHDOG_STATE_FILE — active container watchdog strikes
|
||||
# SYS_WATCHDOG_STATE_FILE — active system watchdog strikes
|
||||
# BANDWIDTH_LOG — yesterday's transfer totals
|
||||
@@ -213,9 +213,9 @@ else
|
||||
fi
|
||||
|
||||
# ── Container Skip List ───────────────────────────────────────────────────────────────────────
|
||||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||||
SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ')
|
||||
if [[ -f "$DOCKER_WATCHDOG_FAILED_FILE" ]] && [[ -s "$DOCKER_WATCHDOG_FAILED_FILE" ]]; then
|
||||
SKIP_COUNT=$(wc -l < "$DOCKER_WATCHDOG_FAILED_FILE")
|
||||
SKIP_LIST=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" | tr '\n' ' ')
|
||||
DIGEST_LINES+=("$ICON_NOT_RUNNING Skip list: $SKIP_COUNT containers — $SKIP_LIST")
|
||||
ISSUES+=("Containers on skip list (manual intervention needed): $SKIP_LIST")
|
||||
SHOULD_SEND=true
|
||||
|
||||
@@ -189,14 +189,14 @@ if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
|
||||
fi
|
||||
|
||||
# Container skip list
|
||||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||||
if [[ -f "$DOCKER_WATCHDOG_FAILED_FILE" ]] && [[ -s "$DOCKER_WATCHDOG_FAILED_FILE" ]]; then
|
||||
SKIP_COUNT=$(wc -l < "$DOCKER_WATCHDOG_FAILED_FILE")
|
||||
echo ""
|
||||
echo " ⛔ Skip list ($SKIP_COUNT — manual intervention needed):"
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
echo " → $container"
|
||||
done < "$SYS_WATCHDOG_FAILED_FILE"
|
||||
done < "$DOCKER_WATCHDOG_FAILED_FILE"
|
||||
else
|
||||
echo " ✅ Skip list: empty"
|
||||
fi
|
||||
|
||||
@@ -49,7 +49,7 @@
|
||||
# STATE FILES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# SYS_WATCHDOG_FAILED_FILE — persistent container skip list (on /boot/config)
|
||||
# DOCKER_WATCHDOG_FAILED_FILE — persistent container skip list (on /boot/config)
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history used for loop detection
|
||||
#
|
||||
# ==============================================================================================
|
||||
@@ -128,7 +128,7 @@ detect_hosts
|
||||
[[ "$FORCE" == true ]] && warn "FORCE mode — confirmation prompt skipped"
|
||||
|
||||
# Ensure state files exist
|
||||
touch "$SYS_WATCHDOG_FAILED_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
|
||||
touch "$DOCKER_WATCHDOG_FAILED_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status — always shown regardless of action ━━━
|
||||
@@ -136,7 +136,7 @@ touch "$SYS_WATCHDOG_FAILED_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Skip List Status — $MY_ID ━━━"
|
||||
|
||||
SKIP_COUNT=$(grep -c "." "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || echo 0)
|
||||
SKIP_COUNT=$(grep -c "." "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null || echo 0)
|
||||
SKIP_COUNT="${SKIP_COUNT//[^0-9]/}"; SKIP_COUNT="${SKIP_COUNT:-0}"
|
||||
RESTART_COUNT=$(wc -l < "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
|
||||
RESTART_COUNT="${RESTART_COUNT//[^0-9]/}"; RESTART_COUNT="${RESTART_COUNT:-0}"
|
||||
@@ -171,7 +171,7 @@ else
|
||||
echo " $ICON_WARN $container — not found on this server"
|
||||
;;
|
||||
esac
|
||||
done < "$SYS_WATCHDOG_FAILED_FILE"
|
||||
done < "$DOCKER_WATCHDOG_FAILED_FILE"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
@@ -212,7 +212,7 @@ if [[ "$ACTION" == "clear-all" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
> "$SYS_WATCHDOG_FAILED_FILE"
|
||||
> "$DOCKER_WATCHDOG_FAILED_FILE"
|
||||
> "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
warn "Skip list cleared ✅"
|
||||
warn "Restart history cleared ✅"
|
||||
@@ -221,7 +221,7 @@ if [[ "$ACTION" == "clear-all" ]]; then
|
||||
notify "Watchdog skip list cleared on $(hostname) ($MY_ID) — all containers will be monitored normally" \
|
||||
"Watchdog Manager" "warning"
|
||||
else
|
||||
warn "DRY RUN — would clear: $SYS_WATCHDOG_FAILED_FILE"
|
||||
warn "DRY RUN — would clear: $DOCKER_WATCHDOG_FAILED_FILE"
|
||||
warn "DRY RUN — would clear: $WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
fi
|
||||
fi
|
||||
@@ -240,11 +240,11 @@ if [[ "$ACTION" == "clear" ]]; then
|
||||
fi
|
||||
|
||||
# Remove from skip list
|
||||
if ! grep -q "^${TARGET_CONTAINER}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null; then
|
||||
if ! grep -q "^${TARGET_CONTAINER}$" "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null; then
|
||||
warn "$TARGET_CONTAINER is not on the skip list"
|
||||
else
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sed -i "/^${TARGET_CONTAINER}$/d" "$SYS_WATCHDOG_FAILED_FILE"
|
||||
sed -i "/^${TARGET_CONTAINER}$/d" "$DOCKER_WATCHDOG_FAILED_FILE"
|
||||
warn "$TARGET_CONTAINER removed from skip list ✅"
|
||||
else
|
||||
warn "DRY RUN — would remove $TARGET_CONTAINER from skip list"
|
||||
|
||||
@@ -134,7 +134,7 @@
|
||||
# ==============================================================================================
|
||||
#
|
||||
# WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots)
|
||||
# SYS_WATCHDOG_FAILED_FILE — container skip list (STATE_DIR — survives reboots)
|
||||
# DOCKER_WATCHDOG_FAILED_FILE — container skip list (STATE_DIR — survives reboots)
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR)
|
||||
# RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag
|
||||
#
|
||||
@@ -255,7 +255,7 @@ validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" "
|
||||
|
||||
# Ensure state files exist
|
||||
touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \
|
||||
"$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
"$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
|
||||
# Timeout for all docker commands — prevents hung daemon from stalling the watchdog
|
||||
DOCKER_TIMEOUT=15
|
||||
@@ -306,14 +306,14 @@ set_strikes() {
|
||||
|
||||
# Check if container is on the persistent skip list
|
||||
is_skipped() {
|
||||
grep -q "^${1}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
grep -q "^${1}$" "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
}
|
||||
|
||||
# Add container to persistent skip list — manual intervention required to recover
|
||||
add_to_skip_list() {
|
||||
local container="$1" reason="$2"
|
||||
if ! is_skipped "$container"; then
|
||||
echo "$container" >> "$SYS_WATCHDOG_FAILED_FILE"
|
||||
echo "$container" >> "$DOCKER_WATCHDOG_FAILED_FILE"
|
||||
error "$container added to skip list — $reason"
|
||||
queue_notify "$container added to skip list on $(hostname) — $reason — manual intervention needed" "critical"
|
||||
fi
|
||||
@@ -321,7 +321,7 @@ add_to_skip_list() {
|
||||
|
||||
# Remove container from skip list — called when container is seen running again
|
||||
remove_from_skip_list() {
|
||||
sed -i "/^${1}$/d" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
sed -i "/^${1}$/d" "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
warn "$1 recovered — removed from skip list ✅"
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user