#!/bin/bash # ----------------------------------------------------------------------------------------------- # --------------------------------- Watchdog Skip List Manager --------------------------------- # ----------------------------------------------------------------------------------------------- # View and manage the persistent container skip list used by docker_watchdog.sh. # Containers are added to the skip list when they exceed the restart loop limit. # They stay there until manually cleared or until found running again automatically. # # Usage: # watchdog_skip_list_manager.sh --status — show current skip list and restart history # watchdog_skip_list_manager.sh --clear-all — clear all skip lists and restart history # watchdog_skip_list_manager.sh --clear ContainerName — clear specific container # # After clearing a container from the skip list: # 1. Fix whatever was causing the container to fail # 2. Start the container manually: docker start ContainerName # 3. The watchdog will monitor it normally on the next cycle # # Files managed: # SYS_WATCHDOG_FAILED_FILE — persistent container skip list # WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection # ----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" parse_args "$@" # Parse action from args ACTION="" TARGET_CONTAINER="" for arg in "${PARSED_ARGS[@]}"; do case "$arg" in --clear-all) ACTION="clear-all" ;; --clear) ACTION="clear" ;; --status) ACTION="status" ;; *) [[ "$ACTION" == "clear" && -z "$TARGET_CONTAINER" ]] && TARGET_CONTAINER="$arg" ;; esac done [[ -z "$ACTION" ]] && ACTION="status" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_GEAR Setup ━━━" if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi success "Running as root" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" touch "$SYS_WATCHDOG_FAILED_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null # ----------------------------------------------------------------------------------------------- # ━━━ STATUS ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_WATCHDOG Skip List Status ━━━" SKIP_COUNT=$(grep -c "." "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || echo 0) RESTART_COUNT=$(wc -l < "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0) if [[ "$SKIP_COUNT" -eq 0 ]]; then success "Skip list is empty — all containers healthy" else warn "$SKIP_COUNT container(s) on skip list:" while IFS= read -r container; do [[ -z "$container" ]] && continue # Check if container is currently running STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") if [[ "$STATUS" == "true" ]]; then echo " $ICON_RUNNING $container — currently RUNNING (will auto-clear on next watchdog cycle)" elif [[ "$STATUS" == "false" ]]; then echo " $ICON_STOPPED $container — currently STOPPED — fix and start manually" else echo " $ICON_INFO $container — container not found" fi done < "$SYS_WATCHDOG_FAILED_FILE" fi echo "" echo "━━━ $ICON_WATCHDOG Restart History ━━━" if [[ "$RESTART_COUNT" -eq 0 ]]; then success "No restart history" else info "$RESTART_COUNT restart entries (window: ${WATCHDOG_CONTAINER_RESTART_WINDOW}h)" echo "" # Show per-container restart counts awk -F'|' '{counts[$1]++} END {for (c in counts) printf " %-30s %d restart(s)\n", c, counts[c]}' \ "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | sort fi [[ "$ACTION" == "status" ]] && exit 0 # ----------------------------------------------------------------------------------------------- # ━━━ CLEAR ALL ━━━ # ----------------------------------------------------------------------------------------------- if [[ "$ACTION" == "clear-all" ]]; then echo "" echo "━━━ $ICON_TRASH Clear All Skip Lists ━━━" warn "This will clear the skip list and restart history for ALL containers" if [[ "$DRY_RUN" == false ]]; then read -r -p "Type YES to confirm: " CONFIRM if [[ "$CONFIRM" != "YES" ]]; then info "Cancelled" exit 0 fi > "$SYS_WATCHDOG_FAILED_FILE" > "$WATCHDOG_CONTAINER_RESTART_LOG" success "Skip list cleared" success "Restart history cleared" notify "Watchdog skip list manually cleared on $(hostname) — all containers will be monitored normally" "Watchdog Manager" "normal" else warn "DRY RUN — would clear: $SYS_WATCHDOG_FAILED_FILE" warn "DRY RUN — would clear: $WATCHDOG_CONTAINER_RESTART_LOG" fi fi # ----------------------------------------------------------------------------------------------- # ━━━ CLEAR SPECIFIC CONTAINER ━━━ # ----------------------------------------------------------------------------------------------- if [[ "$ACTION" == "clear" ]]; then echo "" echo "━━━ $ICON_TRASH Clear Container: $TARGET_CONTAINER ━━━" if [[ -z "$TARGET_CONTAINER" ]]; then error "No container specified. Usage: --clear ContainerName" exit 1 fi if ! grep -q "^${TARGET_CONTAINER}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null; then warn "$TARGET_CONTAINER is not on the skip list" else if [[ "$DRY_RUN" == false ]]; then sed -i "/^${TARGET_CONTAINER}$/d" "$SYS_WATCHDOG_FAILED_FILE" success "$TARGET_CONTAINER removed from skip list" else warn "DRY RUN — would remove $TARGET_CONTAINER from skip list" fi fi # Clear restart history for this container HIST_COUNT=$(grep -c "^${TARGET_CONTAINER}|" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0) if [[ "$HIST_COUNT" -gt 0 ]]; then if [[ "$DRY_RUN" == false ]]; then sed -i "/^${TARGET_CONTAINER}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" success "Cleared $HIST_COUNT restart history entries for $TARGET_CONTAINER" else warn "DRY RUN — would clear $HIST_COUNT restart history entries" fi else info "No restart history for $TARGET_CONTAINER" fi echo "" echo "$ICON_INFO Next steps:" echo " 1. Fix whatever was causing $TARGET_CONTAINER to fail" echo " 2. Start it manually: docker start $TARGET_CONTAINER" echo " 3. Watchdog will monitor it normally on the next cycle" fi echo "" echo "━━━━━ $ICON_SUMMARY DONE ━━━━━"