#!/bin/bash
# ==============================================================================================
# ========================= Monthly Maintenance Orchestrator ===================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Uptime-triggered monthly maintenance — runs heavy tasks that need a stable,
# settled system. Schedule: 0 0 15 * * (15th of each month at midnight)
# Fires only when BOTH gates pass:
#   1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days
#   2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run)
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Runs MONTHLY_MAINTENANCE_SCRIPTS sequentially when both gates pass.
# Silent exit 0 when either gate is not met — only outputs when maintenance fires.
# If uptime or interval gate is not met on the 15th, the run is skipped until
# next month.
#
# STATE FILE
#   MONTHLY_LAST_RUN_FILE lives on /boot/config — survives reboots, available
#   before the array starts. Written after each run (pass or partial fail).
#   Format: Unix timestamp. A reboot does NOT reset the last-run state — the
#   interval gate survives independently of the uptime gate.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Uptime Gate Ensures Stability
#   A scheduled reboot resets uptime. Monthly tasks (ZFS scrub, SMART long test)
#   need a stable, settled system — not one that just rebooted. Both gates must
#   pass before maintenance fires, ensuring the server has been healthy for a
#   full month.
#
# State Survives Reboots
#   MONTHLY_LAST_RUN_FILE is on /boot/config (USB flash), not on the array.
#   It is always available regardless of array state, so the interval gate is
#   never lost to a reboot.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Enforcement
#   ZFS scrub and SMART tests require root.
#
# Lock Acquisition
#   acquire_lock prevents concurrent monthly runs. These are long jobs — a scrub can run
#   for hours — and two at once would double the I/O cost for no benefit.
#
# Host Detection
#   detect_hosts() sets MY_ID for notifications and logs.
#
# Empty Job List Guard
#   Exits with an error and a notification if MONTHLY_MAINTENANCE_SCRIPTS is empty. A
#   monthly job that silently does nothing is the hardest kind to notice missing.
#
# Uptime Gate
#   MONTHLY_UPTIME_THRESHOLD_DAYS must be met before the run proceeds. Heavy full-disk
#   work immediately after a boot competes with everything else still starting up.
#
# Interval Gate
#   MONTHLY_RUN_INTERVAL_DAYS since the last successful run must have elapsed. The
#   schedule fires more often than the work should actually happen, so the gate — not
#   the cron entry — is what defines the real cadence.
#
# Force Override
#   --force bypasses both gates for a deliberate manual run.
#
# Non-Fatal Steps
#   A failing job is recorded and the remaining jobs still run.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   MONTHLY_MAINTENANCE_SCRIPTS    — ordered list of scripts to run
#   MONTHLY_UPTIME_THRESHOLD_DAYS  — minimum uptime in days before maintenance fires
#   MONTHLY_RUN_INTERVAL_DAYS      — minimum days since last run before running again
#   MONTHLY_LAST_RUN_FILE          — state file path (/boot/config — survives reboots)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# monthly_maintenance.sh
#     Normal run — uptime + interval gates enforced.
#
# monthly_maintenance.sh --dry-run
#     Preview gate state and scripts without running.
#
# monthly_maintenance.sh --status
#     Show gate state, last run, and configured scripts.
#
# monthly_maintenance.sh --force
#     Bypass uptime + interval gates (manual override).
#
# monthly_maintenance.sh --log
#     Verbose output.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

source "$ECOSYSTEM_ROOT/load_config.sh"

# ── Parse --force before common parse_args ────────────────────────────────────────────────────
FORCE_RUN=false
FILTERED_ARGS=()
for _arg in "$@"; do
    if [[ "$_arg" == "--force" ]]; then
        FORCE_RUN=true
    else
        FILTERED_ARGS+=("$_arg")
    fi
done
parse_args "${FILTERED_ARGS[@]}"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

detect_hosts

# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
# from a healthy run. Fail loudly instead of silently doing no work.
if [[ ${#MONTHLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
    error "MONTHLY_MAINTENANCE_SCRIPTS is empty — no monthly maintenance scripts will run"
    error "Check MONTHLY_MAINTENANCE_SCRIPTS in master.conf"
    notify "monthly maintenance scripts skipped on $(hostname) ($MY_ID) — MONTHLY_MAINTENANCE_SCRIPTS is empty" \
        "$(basename "$0" .sh)" "warning"
    exit 1
fi

[[ "$DRY_RUN" == true ]]   && warn "DRY RUN — no scripts will be executed"
[[ "$FORCE_RUN" == true ]] && warn "FORCE — uptime and interval gates bypassed"

# Defaults — overridden by master.conf values
MONTHLY_UPTIME_THRESHOLD_DAYS="${MONTHLY_UPTIME_THRESHOLD_DAYS:-30}"
MONTHLY_RUN_INTERVAL_DAYS="${MONTHLY_RUN_INTERVAL_DAYS:-30}"
MONTHLY_LAST_RUN_FILE="${MONTHLY_LAST_RUN_FILE:-${STATE_DIR:-/tmp}/monthly_maintenance_last_run.db}"

UPTIME_THRESHOLD_SECS=$(( MONTHLY_UPTIME_THRESHOLD_DAYS * 86400 ))
INTERVAL_SECS=$(( MONTHLY_RUN_INTERVAL_DAYS * 86400 ))
UPTIME_SECS=$(awk '{print int($1)}' /proc/uptime)
NOW=$(date +%s)

# ==============================================================================================
# ━━━ Gate Evaluation ━━━
# ==============================================================================================
_uptime_days=$(( UPTIME_SECS / 86400 ))
_uptime_hrs=$(( (UPTIME_SECS % 86400) / 3600 ))

UPTIME_GATE_PASS=false
if [[ "$UPTIME_SECS" -ge "$UPTIME_THRESHOLD_SECS" ]]; then
    UPTIME_GATE_PASS=true
fi

INTERVAL_GATE_PASS=false
LAST_RUN=0
DAYS_SINCE_LAST_RUN="never"
if [[ -f "$MONTHLY_LAST_RUN_FILE" ]]; then
    LAST_RUN=$(cat "$MONTHLY_LAST_RUN_FILE" 2>/dev/null || echo 0)
    ELAPSED=$(( NOW - LAST_RUN ))
    DAYS_SINCE=$(( ELAPSED / 86400 ))
    DAYS_SINCE_LAST_RUN="${DAYS_SINCE}d"
    if [[ "$ELAPSED" -ge "$INTERVAL_SECS" ]]; then
        INTERVAL_GATE_PASS=true
    fi
else
    INTERVAL_GATE_PASS=true  # never run
fi

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE STATUS ━━━━━"
    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo ""
    echo "── Gates ──"

    if [[ "$UPTIME_GATE_PASS" == true ]]; then
        echo "  $ICON_DONE Uptime: ${_uptime_days}d ${_uptime_hrs}h (threshold: ${MONTHLY_UPTIME_THRESHOLD_DAYS}d) ✅"
    else
        echo "  $ICON_WARN Uptime: ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d needed — NOT met"
    fi

    if [[ "$INTERVAL_GATE_PASS" == true ]]; then
        echo "  $ICON_DONE Interval: last run ${DAYS_SINCE_LAST_RUN} ago (threshold: ${MONTHLY_RUN_INTERVAL_DAYS}d) ✅"
    else
        echo "  $ICON_WARN Interval: last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d needed — NOT met"
    fi

    if [[ "$LAST_RUN" -gt 0 ]]; then
        echo "  Last run: $(date -d "@$LAST_RUN" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$LAST_RUN" '+%Y-%m-%d %H:%M:%S')"
    else
        echo "  Last run: never"
    fi

    echo ""
    echo "── Scripts ──"
    if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then
        echo "  (none configured — add to MONTHLY_MAINTENANCE_SCRIPTS in master.conf)"
    else
        for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
            [[ -z "$entry" ]] && continue
            read -r -a parts <<< "$entry"
            script_path="$ECOSYSTEM_ROOT/${parts[0]}"
            script_name=$(basename "${parts[0]}")
            if [[ -f "$script_path" ]]; then
                [[ -x "$script_path" ]] && echo "  $ICON_DONE $script_name" || echo "  $ICON_WARN $script_name (not executable)"
            else
                echo "  $ICON_ERROR $script_name — NOT FOUND: $script_path"
            fi
        done
    fi

    echo ""
    echo "  Schedule:   0 0 15 * * (15th of each month at midnight)"
    echo "  Force flag: --force bypasses both gates"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Gate Check — Silent Exit When Not Due ━━━
# ==============================================================================================
if [[ "$FORCE_RUN" != true ]]; then
    if [[ "$UPTIME_GATE_PASS" != true ]]; then
        echo "Uptime gate not met — ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d — no-op"
        exit 0
    fi
    if [[ "$INTERVAL_GATE_PASS" != true ]]; then
        echo "Interval gate not met — last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d — no-op"
        exit 0
    fi
fi

# ==============================================================================================
# ━━━ Pre-flight ━━━
# ==============================================================================================
if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then
    warn "No scripts in MONTHLY_MAINTENANCE_SCRIPTS — nothing to run"
    exit 0
fi

echo ""
echo "━━━ $ICON_START Monthly Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST  Uptime: ${_uptime_days}d ${_uptime_hrs}h | Last run: ${DAYS_SINCE_LAST_RUN} ago${FORCE_RUN:+ | FORCED}"
echo "$ICON_GEAR  Scripts: ${#MONTHLY_MAINTENANCE_SCRIPTS[@]}"
echo ""

START=$(date +%s)
JOB_PASS=()
JOB_FAIL=()
STEP=0

# ==============================================================================================
# ━━━ Run Sequence ━━━
# ==============================================================================================
for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
    [[ -z "$entry" ]] && continue
    (( STEP++ ))

    read -r -a parts <<< "$entry"
    script_name=$(basename "${parts[0]}")
    extra_args=("${parts[@]:1}")

    echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
    run_orch_child "$entry"
    echo ""
done

END=$(date +%s)

# ==============================================================================================
# ━━━ Record Last Run ━━━
# ==============================================================================================
# Write timestamp whether we passed or partially failed — prevents hammering broken scripts.
if [[ "$DRY_RUN" != true ]]; then
    echo "$NOW" > "$MONTHLY_LAST_RUN_FILE"
    log "Last run recorded: $(date -d "@$NOW" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$NOW" '+%Y-%m-%d %H:%M:%S')"
fi

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━"
echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME    Duration: $(format_duration $(( END - START )))"
[[ ${#JOB_PASS[@]} -gt 0 ]] && echo "$ICON_DONE    Passed:   ${JOB_PASS[*]}"
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR   Failed:   ${JOB_FAIL[*]}"
echo ""

# STEP is what this orchestrator expected to run, so it is the denominator that makes a skipped
# step visible rather than absent.
JOB_COUNT="$STEP"
orchestrator_summary "MONTHLY MAINTENANCE" "$START" "Monthly Maintenance"
exit $?