#!/bin/bash
# ==============================================================================================
# ============================ Watchdog Orchestrator ===========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. Replaces the
# continuous loops previously embedded in individual watchdog scripts — those
# are now single-pass; this orchestrator provides the cadence.
# Schedule: */15 * * * * (every 15 minutes)
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Order driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf.
# Default: resource_watchdog → docker_watchdog → system_watchdog →
#          unraid_api_key_renew → stability_watchdog
#
# ARRAY CHECK
#   Exits immediately if /mnt/user is not mounted as shfs. Watchdogs check
#   Docker containers and storage — meaningless without the array. Prevents
#   false positives and unnecessary reboots when array is stopped or stopping.
#
# STARTUP GRACE
#   No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds. Prevents
#   false positives from containers still starting at array launch. Each
#   sub-script enforces this independently — orchestrator exits early to avoid
#   log noise.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Pressure Before Healing
#   Resource Watchdog runs first — it frees RAM and CPU before any container
#   restart is attempted. Containers restarted into a resource-pressured system
#   just fail again. Docker Watchdog restarts with pressure already reduced.
#   System Watchdog checks component health after containers are healed.
#   Stability Watchdog reboots only when all prior layers could not resolve the
#   issue. API key renew is check-first and silent when valid.
#
# Never Queue
#   acquire_lock exits immediately if a prior cycle is still running. Prevents
#   pile-up when a cycle runs long (daemon restart attempt = 30s, etc.).
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Enforcement
#   Every watchdog launched from here requires root. Failing once at the top gives one
#   clear error instead of the same permission failure repeated per child.
#
# Lock Acquisition
#   acquire_lock in strict mode — if the previous cycle is still running, this one exits
#   rather than queuing. At a 15-minute cadence a waiting lock would pile up cycles
#   behind a slow watchdog and eventually run them all at once.
#
# Host Detection
#   detect_hosts() sets MY_ID for logs and notifications.
#
# Empty Job List Guard
#   Exits with an error and a notification if WATCHDOG_ORCHESTRATOR_SCRIPTS is empty.
#   Without it the cycle reports "0/0 passed" and exits 0 every cycle — indistinguishable
#   from a healthy run, while nothing at all is being monitored.
#
# Array Check
#   Exits early if /mnt/user is not shfs-mounted. Watchdogs that inspect shares would
#   otherwise read an unmounted array as missing data and act on it.
#
# Startup Grace
#   WATCHDOG_STARTUP_GRACE is respected before any checks run, so containers still
#   initialising after boot are not judged as unhealthy.
#
# Non-Fatal Steps
#   run_orch_child() records a failing or missing watchdog and continues. One broken
#   watchdog never suppresses the rest of the chain.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   WATCHDOG_ORCHESTRATOR_SCRIPTS         — watchdogs to run, in order
#   WATCHDOG_STARTUP_GRACE                — seconds after boot before checks activate
#   WATCHDOG_ORCHESTRATOR_HEARTBEAT       — periodic heartbeat log toggle
#   WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# watchdog_orchestrator.sh
#     Normal run (called by cron every 15 minutes).
#
# watchdog_orchestrator.sh --dry-run
#     Pass --dry-run to all sub-scripts.
#
# watchdog_orchestrator.sh --status
#     Show script paths and current grace state.
#
# watchdog_orchestrator.sh --log
#     Verbose output from all sub-scripts.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

source "$ECOSYSTEM_ROOT/load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

# Skip immediately if another cycle is still running — no pile-up
acquire_lock

detect_hosts

# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
# from a healthy run. Fail loudly instead of silently doing no work.
if [[ ${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} -eq 0 ]]; then
    error "WATCHDOG_ORCHESTRATOR_SCRIPTS is empty — no watchdogs will run"
    error "Check WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf"
    notify "watchdogs skipped on $(hostname) ($MY_ID) — WATCHDOG_ORCHESTRATOR_SCRIPTS is empty" \
        "$(basename "$0" .sh)" "warning"
    exit 1
fi

log "$ICON_GEAR Config: grace=${WATCHDOG_STARTUP_GRACE}s heartbeat=${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}/${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr scripts=${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"
log "$ICON_WATCHDOG Order: $(for s in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do printf '%s ' "${s##*/}"; done)"

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"

# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
_watchdog_display_name() {
    local path="$1"
    local base="${path##*/}"
    base="${base%.sh}"
    base="${base//_/ }"
    echo "$base" | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}'
}

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY WATCHDOG ORCHESTRATOR STATUS ━━━━━"
    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo ""

    UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
    if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
        warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
    else
        echo "Past startup grace — $(format_duration $UPTIME_S) uptime"
    fi

    echo ""
    echo "── Sub-scripts ──"
    for entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
        local_path="$ECOSYSTEM_ROOT/$entry"
        label="$(_watchdog_display_name "$entry")"
        if [[ -f "$local_path" ]]; then
            [[ -x "$local_path" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
            echo "  $icon $label — ${local_path##*/}"
        else
            echo "  $ICON_ERROR $label — NOT FOUND: $local_path"
        fi
    done

    echo ""
    echo "  Schedule:   */15 * * * * (every 15 minutes)"
    echo "  Heartbeat:  ${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true} / every ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Array Check ━━━
# ==============================================================================================
if ! platform_storage_healthy; then
    echo "Array not started — skipping watchdog cycle"
    exit 0
fi
log "$ICON_DISK Array: $(platform_storage_path) mounted ✅"

# ==============================================================================================
# ━━━ Startup Grace ━━━
# ==============================================================================================
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
    echo "Startup grace — $(format_duration $UPTIME_SECONDS) / $(format_duration $WATCHDOG_STARTUP_GRACE) — skipping cycle"
    exit 0
fi
log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)"

# ==============================================================================================
# ━━━ Run Watchdog Cycle ━━━
# ==============================================================================================
CYCLE_START=$(date +%s)
JOB_PASS=()
JOB_FAIL=()

for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
    [[ -z "$_entry" ]] && continue
    run_orch_child "$_entry"
done

CYCLE_END=$(date +%s)
DURATION=$(( CYCLE_END - CYCLE_START ))

# ==============================================================================================
# ━━━ Heartbeat ━━━
# ==============================================================================================
if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
    HB_SECONDS=$(( ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1} * 3600 ))
    HB_COUNT_FILE="${STATE_DIR:-/tmp}/watchdog_orch_hb.count"
    HB_COUNT=$(cat "$HB_COUNT_FILE" 2>/dev/null || echo 0)
    HB_COUNT=$(( HB_COUNT + 1 ))
    echo "$HB_COUNT" > "$HB_COUNT_FILE"
    # Each cron run = ~60s — use count × 60 as uptime approximation
    HB_ELAPSED=$(( HB_COUNT * 60 ))
    if [[ "$HB_SECONDS" -gt 0 ]] && (( HB_ELAPSED % HB_SECONDS < 60 )) && [[ "$HB_COUNT" -gt 1 ]]; then
        HB_HR=$(( HB_ELAPSED / 3600 ))
        log "♥ watchdog_orchestrator alive — $MY_ID — ~${HB_HR}hr ($(date '+%H:%M:%S'))"
    fi
fi

# ==============================================================================================
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
# ==============================================================================================
if [[ "${#JOB_FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
    for p in "${JOB_PASS[@]}"; do log "  $ICON_DONE  $p"; done
    for f in "${JOB_FAIL[@]}"; do error "  $ICON_ERROR $f"; done
    echo "$ICON_TIME  Duration: $(format_duration $DURATION)"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
else
    echo "$ICON_DONE Watchdog cycle — ${#JOB_PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
fi

if [[ "${#JOB_FAIL[@]}" -gt 0 ]]; then
    notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
        "Watchdog Orchestrator" "warning"
    exit 1
fi

exit 0