#!/bin/bash
# ==============================================================================================
# =========================== Watchdog Skip List Manager =======================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# View and manage the persistent container skip list used by docker_watchdog.sh.
# docker_watchdog.sh adds a container to the skip list when it exceeds
# WATCHDOG_CONTAINER_RESTART_LIMIT restarts within WATCHDOG_CONTAINER_RESTART_WINDOW
# hours — prevents infinite restart loops on containers that keep crashing.
#
# Skip list persists on /boot/config (survives reboots). Auto-clears when
# docker_watchdog.sh sees the container running on a later cycle. Use this
# script to clear manually after fixing the underlying problem.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Skip List Lifecycle
#   1. Container crashes repeatedly → watchdog adds to skip list, notifies
#   2. Watchdog stops restarting the container on subsequent cycles
#   3a. If container recovers on its own (Docker restart policy), watchdog
#       sees it running, removes from skip list automatically
#   3b. If stuck stopped → fix the root cause, clear via this script, then
#       docker start ContainerName manually
#   4. Watchdog monitors normally on next cycle. If it crashes again → re-added.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Clearing Is an Assertion, Not a Fix
#   Removing a container from the skip list tells docker_watchdog.sh to start restarting
#   it again. It does nothing about why the container was crash-looping. Clearing before
#   the underlying fault is fixed just restarts the loop that put it there.
#
# The Skip List Is Protection, Not Punishment
#   A container lands here because restarting it repeatedly was making things worse, not
#   better. The entry exists so the watchdog stops burning cycles and notifications on
#   something only a human can fix.
#
# Persistent by Design
#   The list lives on /boot/config and survives reboots deliberately. A crash loop that
#   a reboot would clear is exactly the case where the watchdog would resume looping
#   after the reboot — persistence is what stops that.
#
# Auto-Clear Is the Normal Path
#   docker_watchdog.sh removes a container from the list on its own once it sees it
#   running healthily. This tool is for the case where you have fixed the problem and
#   do not want to wait for that, not the routine way entries leave the list.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   DOCKER_WATCHDOG_FAILED_FILE
#       Persistent skip list on /boot/config, shared with docker_watchdog.sh. Both must
#       agree on the path or the watchdog will not see what this tool changes.
#
#   WATCHDOG_CONTAINER_RESTART_LIMIT / WATCHDOG_CONTAINER_RESTART_WINDOW
#       The thresholds docker_watchdog.sh uses to decide a container belongs on the list.
#       Shown here for context — this tool does not apply them, it only views and edits
#       the resulting list.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Single Instance Lock
#   acquire_lock prevents concurrent access with docker_watchdog.sh writing
#   the same files.
#
# Active Watchdog Detection
#   Warns if docker_watchdog.sh is currently running when a clear is attempted
#   — the watchdog could re-add the container to the skip list within seconds.
#
# Docker Timeout
#   DOCKER_TIMEOUT caps docker inspect calls against a hung daemon.
#
# Confirmation Required
#   Interactive mode prompts for YES before clearing. Use --force for scripts.
#
# ==============================================================================================
# STATE FILES
# ==============================================================================================
#
#   DOCKER_WATCHDOG_FAILED_FILE        — persistent container skip list (on /boot/config)
#   WATCHDOG_CONTAINER_RESTART_LOG  — restart history used for loop detection
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# watchdog_skip_list_manager.sh [--status]
#     Show skip list, container states, and recent restart history.
#
# watchdog_skip_list_manager.sh --clear ContainerName
#     Remove a specific container from the skip list and clear its restart history.
#     Prompts for YES unless --force is passed.
#
# watchdog_skip_list_manager.sh --clear-all
#     Clear all skip lists and all restart history.
#     Prompts for YES unless --force is passed.
#
# All actions support --dry-run (show what would change) and --force (skip prompt).
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

DOCKER_TIMEOUT=15

# ── Parse action flags before parse_args ──────────────────────────────────────────────────────
ACTION="status"
TARGET_CONTAINER=""
FORCE=false
FILTERED_ARGS=()

for arg in "$@"; do
    case "$arg" in
        --clear-all)  ACTION="clear-all" ;;
        --clear)      ACTION="clear" ;;
        --status)     ACTION="status" ;;
        --force)      FORCE=true ;;
        *)
            if [[ "$ACTION" == "clear" && -z "$TARGET_CONTAINER" ]]; then
                TARGET_CONTAINER="$arg"
            else
                FILTERED_ARGS+=("$arg")
            fi
            ;;
    esac
done

parse_args "${FILTERED_ARGS[@]}"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi


acquire_lock

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

# detect_hosts() sets MY_ID — used in output
detect_hosts

log "$ICON_GEAR Config: action=${ACTION} container=${TARGET_CONTAINER:-all} skip-file=${DOCKER_WATCHDOG_FAILED_FILE}"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
[[ "$FORCE"   == true ]] && warn "FORCE mode — confirmation prompt skipped"

# Ensure state files exist
touch "$DOCKER_WATCHDOG_FAILED_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null

# ==============================================================================================
# ━━━ Status — always shown regardless of action ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_WATCHDOG Skip List Status — $MY_ID ━━━"

SKIP_COUNT=$(grep -c "." "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null || true)
SKIP_COUNT="${SKIP_COUNT//[^0-9]/}"; SKIP_COUNT="${SKIP_COUNT:-0}"
RESTART_COUNT=$(wc -l < "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
RESTART_COUNT="${RESTART_COUNT//[^0-9]/}"; RESTART_COUNT="${RESTART_COUNT:-0}"

# docker_watchdog.sh running check
WATCHDOG_RUNNING=false
if pgrep -f "docker_watchdog.sh" >/dev/null 2>&1; then
    WATCHDOG_RUNNING=true
    warn "docker_watchdog.sh is currently RUNNING"
    [[ "$ACTION" != "status" ]] && \
        warn "Clearing during an active cycle — watchdog may re-add container on next iteration"
fi

echo ""
if [[ "$SKIP_COUNT" -eq 0 ]]; then
    echo "Skip list: empty — all containers monitored normally ✅"
else
    warn "$SKIP_COUNT container(s) on skip list — manual intervention needed:"
    echo ""
    while IFS= read -r container; do
        [[ -z "$container" ]] && continue
        STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
            '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
        case "$STATUS" in
            true)
                echo "  $ICON_RUNNING $container — RUNNING (watchdog will auto-clear next cycle)"
                ;;
            false)
                echo "  $ICON_NOT_RUNNING $container — STOPPED — fix and start manually"
                ;;
            *)
                echo "  $ICON_WARN $container — not found on this server"
                ;;
        esac
    done < "$DOCKER_WATCHDOG_FAILED_FILE"
fi

echo ""
echo "━━━ $ICON_WATCHDOG Restart History ━━━"
if [[ "$RESTART_COUNT" -eq 0 ]]; then
    echo "No restart history"
else
    echo "$RESTART_COUNT restart entries (window: ${WATCHDOG_CONTAINER_RESTART_WINDOW}h)"
    echo ""
    awk -F'|' '{counts[$1]++} END {
        for (c in counts)
            printf "  %-30s %d restart(s)\n", c, counts[c]
    }' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | sort
fi

[[ "$ACTION" == "status" ]] && exit 0

# ==============================================================================================
# ━━━ Clear All ━━━
# ==============================================================================================
if [[ "$ACTION" == "clear-all" ]]; then
    echo ""
    echo "━━━ $ICON_TRASH Clear All Skip Lists ━━━"
    warn "This will clear the skip list and restart history for ALL containers"
    echo ""

    if [[ "$DRY_RUN" == false ]]; then
        if [[ "$FORCE" == true ]]; then
            log "FORCE flag set — skipping confirmation"
        elif [[ -t 0 ]]; then
            read -r -p "Type YES to confirm: " CONFIRM
            if [[ "$CONFIRM" != "YES" ]]; then
                warn "Cancelled"
                exit 0
            fi
        else
            error "Non-interactive mode — use --force flag to skip confirmation"
            exit 1
        fi

        > "$DOCKER_WATCHDOG_FAILED_FILE"
        > "$WATCHDOG_CONTAINER_RESTART_LOG"
        warn "Skip list cleared ✅"
        warn "Restart history cleared ✅"
        [[ "$WATCHDOG_RUNNING" == true ]] && \
            warn "Note: watchdog is running — containers will be monitored on next cycle"
        notify "Watchdog skip list cleared on $(hostname) ($MY_ID) — all containers will be monitored normally" \
            "Watchdog Manager" "warning"
    else
        warn "DRY RUN — would clear: $DOCKER_WATCHDOG_FAILED_FILE"
        warn "DRY RUN — would clear: $WATCHDOG_CONTAINER_RESTART_LOG"
    fi
fi

# ==============================================================================================
# ━━━ Clear Specific Container ━━━
# ==============================================================================================
if [[ "$ACTION" == "clear" ]]; then
    echo ""
    echo "━━━ $ICON_TRASH Clear Container: $TARGET_CONTAINER ━━━"

    if [[ -z "$TARGET_CONTAINER" ]]; then
        error "No container specified"
        error "Usage: watchdog_skip_list_manager.sh --clear ContainerName"
        exit 1
    fi

    # Remove from skip list
    if ! grep -q "^${TARGET_CONTAINER}$" "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null; then
        warn "$TARGET_CONTAINER is not on the skip list"
    else
        if [[ "$DRY_RUN" == false ]]; then
            sed -i "/^${TARGET_CONTAINER}$/d" "$DOCKER_WATCHDOG_FAILED_FILE"
            warn "$TARGET_CONTAINER removed from skip list ✅"
        else
            warn "DRY RUN — would remove $TARGET_CONTAINER from skip list"
        fi
    fi

    # Clear restart history for this container
    HIST_COUNT=$(grep -c "^${TARGET_CONTAINER}|" \
        "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
    HIST_COUNT="${HIST_COUNT//[^0-9]/}"; HIST_COUNT="${HIST_COUNT:-0}"

    if [[ "$HIST_COUNT" -gt 0 ]]; then
        if [[ "$DRY_RUN" == false ]]; then
            sed -i "/^${TARGET_CONTAINER}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG"
            warn "Cleared $HIST_COUNT restart history entries for $TARGET_CONTAINER ✅"
        else
            warn "DRY RUN — would clear $HIST_COUNT restart history entries"
        fi
    else
        log "No restart history for $TARGET_CONTAINER"
    fi

    [[ "$WATCHDOG_RUNNING" == true ]] && \
        warn "Note: watchdog is running — $TARGET_CONTAINER may be re-added if still failing"

    if [[ "$DRY_RUN" == false ]]; then
        echo ""
        echo "━━━ $ICON_INFO Next Steps ━━━"
        echo "  1. Fix whatever was causing $TARGET_CONTAINER to fail"
        echo "  2. Start it manually: docker start $TARGET_CONTAINER"
        echo "  3. docker_watchdog.sh monitors it on the next cycle"
        echo "  4. If it crashes again → watchdog adds it back and notifies"
        notify "$TARGET_CONTAINER cleared from watchdog skip list on $(hostname) ($MY_ID)" \
            "Watchdog Manager" "warning"
    fi
fi

echo ""
echo "━━━━━ $ICON_SUMMARY DONE — $MY_ID ━━━━━"