diff --git a/Deployment/conf_templates/master.conf b/Deployment/conf_templates/master.conf index 949ac22..874aef1 100644 --- a/Deployment/conf_templates/master.conf +++ b/Deployment/conf_templates/master.conf @@ -1347,7 +1347,7 @@ # ━━━ State Files ━━━ SYS_WATCHDOG_STATE_FILE="$STATE_DIR/system_watchdog_state.db" - SYS_WATCHDOG_FAILED_FILE="$STATE_DIR/system_watchdog_failed.db" + DOCKER_WATCHDOG_FAILED_FILE="$STATE_DIR/docker_watchdog_failed.db" SYS_WATCHDOG_REBOOT_LOG="$STATE_DIR/system_watchdog_reboots.db" SYS_WATCHDOG_OOM_FILE="$STATE_DIR/system_watchdog_oom.db" diff --git a/Monitors/weekly_health_digest.sh b/Monitors/weekly_health_digest.sh index 0c7f4d7..70dc32c 100644 --- a/Monitors/weekly_health_digest.sh +++ b/Monitors/weekly_health_digest.sh @@ -31,7 +31,7 @@ # # Data sources (reads only): # FALLBACK_STATE_FILE — current fallback state -# SYS_WATCHDOG_FAILED_FILE — container skip list (manual intervention needed) +# DOCKER_WATCHDOG_FAILED_FILE — container skip list (manual intervention needed) # WATCHDOG_STATE_FILE — active container watchdog strikes # SYS_WATCHDOG_STATE_FILE — active system watchdog strikes # BANDWIDTH_LOG — yesterday's transfer totals @@ -213,9 +213,9 @@ else fi # ── Container Skip List ─────────────────────────────────────────────────────────────────────── -if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then - SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE") - SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ') +if [[ -f "$DOCKER_WATCHDOG_FAILED_FILE" ]] && [[ -s "$DOCKER_WATCHDOG_FAILED_FILE" ]]; then + SKIP_COUNT=$(wc -l < "$DOCKER_WATCHDOG_FAILED_FILE") + SKIP_LIST=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" | tr '\n' ' ') DIGEST_LINES+=("$ICON_NOT_RUNNING Skip list: $SKIP_COUNT containers — $SKIP_LIST") ISSUES+=("Containers on skip list (manual intervention needed): $SKIP_LIST") SHOULD_SEND=true diff --git a/Old_Arch_Still_Works/continuous_scripts_status.sh b/Old_Arch_Still_Works/continuous_scripts_status.sh index 6627f3a..327c11a 100644 --- a/Old_Arch_Still_Works/continuous_scripts_status.sh +++ b/Old_Arch_Still_Works/continuous_scripts_status.sh @@ -189,14 +189,14 @@ if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then fi # Container skip list -if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then - SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE") +if [[ -f "$DOCKER_WATCHDOG_FAILED_FILE" ]] && [[ -s "$DOCKER_WATCHDOG_FAILED_FILE" ]]; then + SKIP_COUNT=$(wc -l < "$DOCKER_WATCHDOG_FAILED_FILE") echo "" echo " ⛔ Skip list ($SKIP_COUNT — manual intervention needed):" while IFS= read -r container; do [[ -z "$container" ]] && continue echo " → $container" - done < "$SYS_WATCHDOG_FAILED_FILE" + done < "$DOCKER_WATCHDOG_FAILED_FILE" else echo " ✅ Skip list: empty" fi diff --git a/Tools/watchdog_skip_list_manager.sh b/Tools/watchdog_skip_list_manager.sh index 8a2964c..51f1cc2 100644 --- a/Tools/watchdog_skip_list_manager.sh +++ b/Tools/watchdog_skip_list_manager.sh @@ -49,7 +49,7 @@ # STATE FILES # ============================================================================================== # -# SYS_WATCHDOG_FAILED_FILE — persistent container skip list (on /boot/config) +# DOCKER_WATCHDOG_FAILED_FILE — persistent container skip list (on /boot/config) # WATCHDOG_CONTAINER_RESTART_LOG — restart history used for loop detection # # ============================================================================================== @@ -128,7 +128,7 @@ detect_hosts [[ "$FORCE" == true ]] && warn "FORCE mode — confirmation prompt skipped" # Ensure state files exist -touch "$SYS_WATCHDOG_FAILED_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null +touch "$DOCKER_WATCHDOG_FAILED_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null # ============================================================================================== # ━━━ Status — always shown regardless of action ━━━ @@ -136,7 +136,7 @@ touch "$SYS_WATCHDOG_FAILED_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null echo "" echo "━━━ $ICON_WATCHDOG Skip List Status — $MY_ID ━━━" -SKIP_COUNT=$(grep -c "." "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || echo 0) +SKIP_COUNT=$(grep -c "." "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null || echo 0) SKIP_COUNT="${SKIP_COUNT//[^0-9]/}"; SKIP_COUNT="${SKIP_COUNT:-0}" RESTART_COUNT=$(wc -l < "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0) RESTART_COUNT="${RESTART_COUNT//[^0-9]/}"; RESTART_COUNT="${RESTART_COUNT:-0}" @@ -171,7 +171,7 @@ else echo " $ICON_WARN $container — not found on this server" ;; esac - done < "$SYS_WATCHDOG_FAILED_FILE" + done < "$DOCKER_WATCHDOG_FAILED_FILE" fi echo "" @@ -212,7 +212,7 @@ if [[ "$ACTION" == "clear-all" ]]; then exit 1 fi - > "$SYS_WATCHDOG_FAILED_FILE" + > "$DOCKER_WATCHDOG_FAILED_FILE" > "$WATCHDOG_CONTAINER_RESTART_LOG" warn "Skip list cleared ✅" warn "Restart history cleared ✅" @@ -221,7 +221,7 @@ if [[ "$ACTION" == "clear-all" ]]; then notify "Watchdog skip list cleared on $(hostname) ($MY_ID) — all containers will be monitored normally" \ "Watchdog Manager" "warning" else - warn "DRY RUN — would clear: $SYS_WATCHDOG_FAILED_FILE" + warn "DRY RUN — would clear: $DOCKER_WATCHDOG_FAILED_FILE" warn "DRY RUN — would clear: $WATCHDOG_CONTAINER_RESTART_LOG" fi fi @@ -240,11 +240,11 @@ if [[ "$ACTION" == "clear" ]]; then fi # Remove from skip list - if ! grep -q "^${TARGET_CONTAINER}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null; then + if ! grep -q "^${TARGET_CONTAINER}$" "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null; then warn "$TARGET_CONTAINER is not on the skip list" else if [[ "$DRY_RUN" == false ]]; then - sed -i "/^${TARGET_CONTAINER}$/d" "$SYS_WATCHDOG_FAILED_FILE" + sed -i "/^${TARGET_CONTAINER}$/d" "$DOCKER_WATCHDOG_FAILED_FILE" warn "$TARGET_CONTAINER removed from skip list ✅" else warn "DRY RUN — would remove $TARGET_CONTAINER from skip list" diff --git a/Watchdogs/docker_watchdog.sh b/Watchdogs/docker_watchdog.sh index 5811b99..5f23b0b 100755 --- a/Watchdogs/docker_watchdog.sh +++ b/Watchdogs/docker_watchdog.sh @@ -134,7 +134,7 @@ # ============================================================================================== # # WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots) -# SYS_WATCHDOG_FAILED_FILE — container skip list (STATE_DIR — survives reboots) +# DOCKER_WATCHDOG_FAILED_FILE — container skip list (STATE_DIR — survives reboots) # WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR) # RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag # @@ -255,7 +255,7 @@ validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" " # Ensure state files exist touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \ - "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null + "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null # Timeout for all docker commands — prevents hung daemon from stalling the watchdog DOCKER_TIMEOUT=15 @@ -306,14 +306,14 @@ set_strikes() { # Check if container is on the persistent skip list is_skipped() { - grep -q "^${1}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null + grep -q "^${1}$" "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null } # Add container to persistent skip list — manual intervention required to recover add_to_skip_list() { local container="$1" reason="$2" if ! is_skipped "$container"; then - echo "$container" >> "$SYS_WATCHDOG_FAILED_FILE" + echo "$container" >> "$DOCKER_WATCHDOG_FAILED_FILE" error "$container added to skip list — $reason" queue_notify "$container added to skip list on $(hostname) — $reason — manual intervention needed" "critical" fi @@ -321,7 +321,7 @@ add_to_skip_list() { # Remove container from skip list — called when container is seen running again remove_from_skip_list() { - sed -i "/^${1}$/d" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null + sed -i "/^${1}$/d" "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null warn "$1 recovered — removed from skip list ✅" }