massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
@@ -1,175 +1,259 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Watchdog Skip List Manager ---------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# =========================== Watchdog Skip List Manager =======================================
|
||||
# ==============================================================================================
|
||||
# View and manage the persistent container skip list used by docker_watchdog.sh.
|
||||
# Containers are added to the skip list when they exceed the restart loop limit.
|
||||
# They stay there until manually cleared or until found running again automatically.
|
||||
#
|
||||
# Usage:
|
||||
# watchdog_skip_list_manager.sh --status — show current skip list and restart history
|
||||
# watchdog_skip_list_manager.sh --clear-all — clear all skip lists and restart history
|
||||
# watchdog_skip_list_manager.sh --clear ContainerName — clear specific container
|
||||
# ── WHAT THE SKIP LIST IS ─────────────────────────────────────────────────────────────────────
|
||||
# docker_watchdog.sh adds a container to the skip list when it exceeds the restart loop
|
||||
# limit (WATCHDOG_CONTAINER_RESTART_LIMIT in WATCHDOG_CONTAINER_RESTART_WINDOW hours).
|
||||
# Once on the skip list the watchdog stops restarting it — prevents infinite restart loops.
|
||||
#
|
||||
# After clearing a container from the skip list:
|
||||
# Skip list persists on /boot/config — survives reboots.
|
||||
# Auto-clears when docker_watchdog.sh sees the container running on a cycle.
|
||||
# This script clears it manually when you have fixed the underlying problem.
|
||||
#
|
||||
# ── ACTIONS ───────────────────────────────────────────────────────────────────────────────────
|
||||
# --status — show skip list, container states, restart history
|
||||
# --clear ContainerName — clear a specific container from skip list + history
|
||||
# --clear-all — clear all skip lists and restart history
|
||||
#
|
||||
# ── AFTER CLEARING ────────────────────────────────────────────────────────────────────────────
|
||||
# 1. Fix whatever was causing the container to fail
|
||||
# 2. Start the container manually: docker start ContainerName
|
||||
# 3. The watchdog will monitor it normally on the next cycle
|
||||
# 2. Start it manually: docker start ContainerName
|
||||
# 3. docker_watchdog.sh monitors it normally on the next cycle
|
||||
# 4. If it crashes again → watchdog adds it back and notifies
|
||||
#
|
||||
# Files managed:
|
||||
# SYS_WATCHDOG_FAILED_FILE — persistent container skip list
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── SKIP LIST AUTO-CLEAR ──────────────────────────────────────────────────────────────────────
|
||||
# docker_watchdog.sh auto-clears a container from the skip list when it sees it running.
|
||||
# So if a container recovers on its own (Docker restart policy eventually works),
|
||||
# the watchdog will see it running, remove it from the skip list, and resume monitoring.
|
||||
# Manual clear only needed when container is stuck stopped and needs intervention.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents concurrent access with docker_watchdog.sh writing files
|
||||
# docker_watchdog check — warns if watchdog is running during clear (could re-add instantly)
|
||||
# DOCKER_TIMEOUT — docker inspect calls protected against daemon hangs
|
||||
# Confirmation required — interactive: YES | non-interactive: --force flag
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
#
|
||||
# ── FILES MANAGED ─────────────────────────────────────────────────────────────────────────────
|
||||
# SYS_WATCHDOG_FAILED_FILE — persistent container skip list
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# watchdog_skip_list_manager.sh — show status
|
||||
# watchdog_skip_list_manager.sh --status — show status explicitly
|
||||
# watchdog_skip_list_manager.sh --clear ContainerName — clear specific container
|
||||
# watchdog_skip_list_manager.sh --clear-all — clear everything
|
||||
# Any action supports --dry-run and --force
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
# Parse action from args
|
||||
ACTION=""
|
||||
# ── Parse action flags before parse_args ──────────────────────────────────────────────────────
|
||||
ACTION="status"
|
||||
TARGET_CONTAINER=""
|
||||
FORCE=false
|
||||
FILTERED_ARGS=()
|
||||
|
||||
for arg in "${PARSED_ARGS[@]}"; do
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--clear-all) ACTION="clear-all" ;;
|
||||
--clear) ACTION="clear" ;;
|
||||
--status) ACTION="status" ;;
|
||||
--force) FORCE=true ;;
|
||||
*)
|
||||
[[ "$ACTION" == "clear" && -z "$TARGET_CONTAINER" ]] && TARGET_CONTAINER="$arg"
|
||||
if [[ "$ACTION" == "clear" && -z "$TARGET_CONTAINER" ]]; then
|
||||
TARGET_CONTAINER="$arg"
|
||||
else
|
||||
FILTERED_ARGS+=("$arg")
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -z "$ACTION" ]] && ACTION="status"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID — used in output
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
[[ "$FORCE" == true ]] && warn "FORCE mode — confirmation prompt skipped"
|
||||
|
||||
# Ensure state files exist
|
||||
touch "$SYS_WATCHDOG_FAILED_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ STATUS ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Status — always shown regardless of action ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Skip List Status ━━━"
|
||||
echo "━━━ $ICON_WATCHDOG Skip List Status — $MY_ID ━━━"
|
||||
|
||||
SKIP_COUNT=$(grep -c "." "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || echo 0)
|
||||
SKIP_COUNT="${SKIP_COUNT//[^0-9]/}"; SKIP_COUNT="${SKIP_COUNT:-0}"
|
||||
RESTART_COUNT=$(wc -l < "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
|
||||
RESTART_COUNT="${RESTART_COUNT//[^0-9]/}"; RESTART_COUNT="${RESTART_COUNT:-0}"
|
||||
|
||||
# docker_watchdog.sh running check
|
||||
WATCHDOG_RUNNING=false
|
||||
if pgrep -f "docker_watchdog.sh" >/dev/null 2>&1; then
|
||||
WATCHDOG_RUNNING=true
|
||||
warn "docker_watchdog.sh is currently RUNNING"
|
||||
[[ "$ACTION" != "status" ]] && \
|
||||
warn "Clearing during an active cycle — watchdog may re-add container on next iteration"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [[ "$SKIP_COUNT" -eq 0 ]]; then
|
||||
success "Skip list is empty — all containers healthy"
|
||||
log "Skip list: empty — all containers monitored normally ✅"
|
||||
else
|
||||
warn "$SKIP_COUNT container(s) on skip list:"
|
||||
warn "$SKIP_COUNT container(s) on skip list — manual intervention needed:"
|
||||
echo ""
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
# Check if container is currently running
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
echo " $ICON_RUNNING $container — currently RUNNING (will auto-clear on next watchdog cycle)"
|
||||
elif [[ "$STATUS" == "false" ]]; then
|
||||
echo " $ICON_STOPPED $container — currently STOPPED — fix and start manually"
|
||||
else
|
||||
echo " $ICON_INFO $container — container not found"
|
||||
fi
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
case "$STATUS" in
|
||||
true)
|
||||
echo " $ICON_RUNNING $container — RUNNING (watchdog will auto-clear next cycle)"
|
||||
;;
|
||||
false)
|
||||
echo " $ICON_NOT_RUNNING $container — STOPPED — fix and start manually"
|
||||
;;
|
||||
*)
|
||||
echo " $ICON_WARN $container — not found on this server"
|
||||
;;
|
||||
esac
|
||||
done < "$SYS_WATCHDOG_FAILED_FILE"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Restart History ━━━"
|
||||
if [[ "$RESTART_COUNT" -eq 0 ]]; then
|
||||
success "No restart history"
|
||||
log "No restart history"
|
||||
else
|
||||
info "$RESTART_COUNT restart entries (window: ${WATCHDOG_CONTAINER_RESTART_WINDOW}h)"
|
||||
log "$RESTART_COUNT restart entries (window: ${WATCHDOG_CONTAINER_RESTART_WINDOW}h)"
|
||||
echo ""
|
||||
# Show per-container restart counts
|
||||
awk -F'|' '{counts[$1]++} END {for (c in counts) printf " %-30s %d restart(s)\n", c, counts[c]}' \
|
||||
"$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | sort
|
||||
awk -F'|' '{counts[$1]++} END {
|
||||
for (c in counts)
|
||||
printf " %-30s %d restart(s)\n", c, counts[c]
|
||||
}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | sort
|
||||
fi
|
||||
|
||||
[[ "$ACTION" == "status" ]] && exit 0
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ CLEAR ALL ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Clear All ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$ACTION" == "clear-all" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_TRASH Clear All Skip Lists ━━━"
|
||||
warn "This will clear the skip list and restart history for ALL containers"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
read -r -p "Type YES to confirm: " CONFIRM
|
||||
if [[ "$CONFIRM" != "YES" ]]; then
|
||||
info "Cancelled"
|
||||
exit 0
|
||||
if [[ "$FORCE" == true ]]; then
|
||||
log "FORCE flag set — skipping confirmation"
|
||||
elif [[ -t 0 ]]; then
|
||||
read -r -p "Type YES to confirm: " CONFIRM
|
||||
if [[ "$CONFIRM" != "YES" ]]; then
|
||||
warn "Cancelled"
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
error "Non-interactive mode — use --force flag to skip confirmation"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
> "$SYS_WATCHDOG_FAILED_FILE"
|
||||
> "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
success "Skip list cleared"
|
||||
success "Restart history cleared"
|
||||
notify "Watchdog skip list manually cleared on $(hostname) — all containers will be monitored normally" "Watchdog Manager" "normal"
|
||||
warn "Skip list cleared ✅"
|
||||
warn "Restart history cleared ✅"
|
||||
[[ "$WATCHDOG_RUNNING" == true ]] && \
|
||||
warn "Note: watchdog is running — containers will be monitored on next cycle"
|
||||
notify "Watchdog skip list cleared on $(hostname) ($MY_ID) — all containers will be monitored normally" \
|
||||
"Watchdog Manager" "warning"
|
||||
else
|
||||
warn "DRY RUN — would clear: $SYS_WATCHDOG_FAILED_FILE"
|
||||
warn "DRY RUN — would clear: $WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ CLEAR SPECIFIC CONTAINER ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Clear Specific Container ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$ACTION" == "clear" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_TRASH Clear Container: $TARGET_CONTAINER ━━━"
|
||||
|
||||
if [[ -z "$TARGET_CONTAINER" ]]; then
|
||||
error "No container specified. Usage: --clear ContainerName"
|
||||
error "No container specified"
|
||||
error "Usage: watchdog_skip_list_manager.sh --clear ContainerName"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Remove from skip list
|
||||
if ! grep -q "^${TARGET_CONTAINER}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null; then
|
||||
warn "$TARGET_CONTAINER is not on the skip list"
|
||||
else
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sed -i "/^${TARGET_CONTAINER}$/d" "$SYS_WATCHDOG_FAILED_FILE"
|
||||
success "$TARGET_CONTAINER removed from skip list"
|
||||
warn "$TARGET_CONTAINER removed from skip list ✅"
|
||||
else
|
||||
warn "DRY RUN — would remove $TARGET_CONTAINER from skip list"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Clear restart history for this container
|
||||
HIST_COUNT=$(grep -c "^${TARGET_CONTAINER}|" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
|
||||
HIST_COUNT=$(grep -c "^${TARGET_CONTAINER}|" \
|
||||
"$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
|
||||
HIST_COUNT="${HIST_COUNT//[^0-9]/}"; HIST_COUNT="${HIST_COUNT:-0}"
|
||||
|
||||
if [[ "$HIST_COUNT" -gt 0 ]]; then
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sed -i "/^${TARGET_CONTAINER}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
success "Cleared $HIST_COUNT restart history entries for $TARGET_CONTAINER"
|
||||
warn "Cleared $HIST_COUNT restart history entries for $TARGET_CONTAINER ✅"
|
||||
else
|
||||
warn "DRY RUN — would clear $HIST_COUNT restart history entries"
|
||||
fi
|
||||
else
|
||||
info "No restart history for $TARGET_CONTAINER"
|
||||
log "No restart history for $TARGET_CONTAINER"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "$ICON_INFO Next steps:"
|
||||
echo " 1. Fix whatever was causing $TARGET_CONTAINER to fail"
|
||||
echo " 2. Start it manually: docker start $TARGET_CONTAINER"
|
||||
echo " 3. Watchdog will monitor it normally on the next cycle"
|
||||
[[ "$WATCHDOG_RUNNING" == true ]] && \
|
||||
warn "Note: watchdog is running — $TARGET_CONTAINER may be re-added if still failing"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_INFO Next Steps ━━━"
|
||||
echo " 1. Fix whatever was causing $TARGET_CONTAINER to fail"
|
||||
echo " 2. Start it manually: docker start $TARGET_CONTAINER"
|
||||
echo " 3. docker_watchdog.sh monitors it on the next cycle"
|
||||
echo " 4. If it crashes again → watchdog adds it back and notifies"
|
||||
notify "$TARGET_CONTAINER cleared from watchdog skip list on $(hostname) ($MY_ID)" \
|
||||
"Watchdog Manager" "warning"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DONE ━━━━━"
|
||||
echo "━━━━━ $ICON_SUMMARY DONE — $MY_ID ━━━━━"
|
||||
Reference in New Issue
Block a user