#!/bin/bash
# ==============================================================================================
# =========================== Intermediate Sync Maintenance ====================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# 4-hour orchestrator — arr library reconciliation, artwork fetching, and
# optional rsync. Schedule: 0 */4 * * * (every 4 hours)
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
#   1. conf_sync.sh                      — refresh partner conf cache in RAM (/tmp/varaverk/conf/),
#                                          both directions: pull theirs, push ours
#   2. arr_sync.sh                       — sync Lidarr/Sonarr/Radarr libraries across all nodes
#   3. Rsync window (optional)           — INTERMEDIATE_SYNC_SHARES, if any configured
#   4. INTERMEDIATE_MAINTENANCE_SCRIPTS  — artwork fetch and any future 4-hour jobs
#
# DRIVE TEMP HANDLING (rsync.sh exit codes):
#   exit 1 = temp WARN    → skip this share, continue to next
#   exit 2 = temp CRITICAL → abort ALL remaining syncs in this window
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Closes the Library Gap
#   Arr libraries need to converge more frequently than once a day. If a remote
#   node adds something at 2am, the next daily window is 23 hours away — remote
#   arrs search for content they don't know is already owned. Running every 4
#   hours closes that gap.
#
# Optional Rsync Layer
#   INTERMEDIATE_SYNC_SHARES is empty by default — the rsync step is skipped
#   entirely when nothing is configured. Add shares only if a subset of data
#   needs mid-day propagation. Full media share sync stays in the daily window.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Enforcement
#   Every script called from here requires root.
#
# Lock Acquisition
#   acquire_lock prevents concurrent intermediate windows.
#
# Host Detection
#   detect_hosts() aliases the correct per-host share lists.
#
# Empty Job List Guard
#   Exits with an error and a notification if INTERMEDIATE_MAINTENANCE_SCRIPTS is empty,
#   rather than running six no-op windows a day that all report success.
#
# Connectivity Check
#   check_connectivity is verified before any rsync, and skipped entirely when no shares
#   are configured — there is nothing to reach a partner for.
#
# Remote Rootfs Check
#   check_remote_rootfs aborts rsync if the remote rootfs is nearly full.
#
# Drive Temperature Escalation
#   rsync.sh's exit code is honoured per share: exit 1 skips that share, exit 2 aborts
#   every remaining sync in the window and notifies.
#
# Non-Fatal Jobs
#   A failing arr_sync warns but does not block rsync or the artwork fetch that follow it.
#
# Minimal on Success
#   Runs six times a day, so the full breakdown only prints on failure or with --log.
#   A quiet run is the normal outcome and should not fill the log.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# host*.conf
#
#   HOST*_INTERMEDIATE_SYNC_SHARES  — shares synced mid-day (empty = rsync skipped)
#
# master.conf
#
#   INTERMEDIATE_RSYNC_ENABLED      — enable/disable rsync section (default: true)
#   INTERMEDIATE_MAINTENANCE_SCRIPTS — jobs run after rsync
#   ARR_SYNC_ENABLED                — toggle inside arr_sync.sh
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# intermediate_sync_maintenance.sh
#     Normal run.
#
# intermediate_sync_maintenance.sh --dry-run
#     Preview without changes.
#
# intermediate_sync_maintenance.sh --log
#     Verbose per-job output.
#
# intermediate_sync_maintenance.sh --status
#     Show configured shares/jobs and exit.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi


acquire_lock

detect_hosts

# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
# from a healthy run. Fail loudly instead of silently doing no work.
if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
    error "INTERMEDIATE_MAINTENANCE_SCRIPTS is empty — no intermediate maintenance scripts will run"
    error "Check INTERMEDIATE_MAINTENANCE_SCRIPTS in master.conf"
    notify "intermediate maintenance scripts skipped on $(hostname) ($MY_ID) — INTERMEDIATE_MAINTENANCE_SCRIPTS is empty" \
        "$(basename "$0" .sh)" "warning"
    exit 1
fi
resolve_remote_ip

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC STATUS ━━━━━"
    echo "$ICON_HOST   Identity:        $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_NET    Remote:          $REMOTE_ID ($REMOTE_SERVER_NAME)"
    echo "$ICON_SYNC   Rsync enabled:   ${RSYNC_ENABLED:-true}"
    echo "$ICON_SYNC   Interm. enabled: ${INTERMEDIATE_RSYNC_ENABLED:-true}"
    echo "$ICON_GEAR   Arr sync:        ${ARR_SYNC_ENABLED:-true}"
    echo ""
    echo "━━━ Intermediate Sync Shares ━━━"
    if [[ ${#INTERMEDIATE_SYNC_SHARES[@]} -eq 0 ]]; then
        echo "  None configured — add to INTERMEDIATE_SYNC_SHARES in master.conf to enable"
    else
        for share in "${INTERMEDIATE_SYNC_SHARES[@]}"; do
            [[ -n "$share" ]] && echo "  $ICON_SYNC $(basename "$share") ($share)"
        done
    fi
    echo ""
    echo "━━━ Intermediate Maintenance Scripts ━━━"
    if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
        echo "  None configured"
    else
        for entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
            [[ -n "$entry" ]] && echo "  $ICON_GEAR ${entry##*/}"
        done
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

WINDOW_START=$(date +%s)
JOB_PASS=()
JOB_FAIL=()
PASS=()
FAIL=()
SHARE_TIMES=()

echo ""
echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

# ==============================================================================================
# ━━━ Conf Sync ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Conf Sync ━━━"

# Full sync, not --pull-only. The push half was written as an event-driven fast path for the
# conf-save hook, but no such hook was ever built — so outside array start nothing pushed this
# host's conf to its partners at all, and a partner's copy of our conf stayed at whatever it was
# when we last rebooted. Pull alone kept our view of them fresh while their view of us decayed.
CONF_SYNC_SCRIPT="$ECOSYSTEM_ROOT/System_Essentials/conf_sync.sh"
if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
    warn "conf_sync.sh not found — skipping partner conf refresh"
else
    _conf_args=()
    [[ "$DRY_RUN" == true ]] && _conf_args+=("--dry-run")
    if bash "$CONF_SYNC_SCRIPT" "${_conf_args[@]}"; then
        echo "Partner conf cache refreshed ✅"
        JOB_PASS+=("conf_sync.sh")
    else
        warn "Partner conf sync failed — cache may be stale"
        JOB_FAIL+=("conf_sync.sh")
    fi
    unset _conf_args
fi

# ==============================================================================================
# ━━━ Arr Sync ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Arr Sync ━━━"

ARR_SYNC_SCRIPT="$ECOSYSTEM_ROOT/Arrs_Stack/arr_sync.sh"
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
    echo "ARR_SYNC_ENABLED=false — skipping"
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
    warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
    JOB_FAIL+=("arr_sync.sh")
else
    _arr_sync_args=()
    [[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
    if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
        echo "Arr sync complete ✅"
        JOB_PASS+=("arr_sync.sh")
    else
        warn "Arr sync completed with errors — continuing"
        JOB_FAIL+=("arr_sync.sh")
    fi
    unset _arr_sync_args
fi

# ==============================================================================================
# ━━━ Rsync (optional) ━━━
# ==============================================================================================
SHARE_COUNT=${#INTERMEDIATE_SYNC_SHARES[@]}

echo ""
echo "━━━ $ICON_SYNC Mid-day Share Sync — $SHARE_COUNT share(s) ━━━"

TOTAL_START=$(date +%s)
SHARE_INDEX=0
ABORT_ALL_SYNCS=false

if [[ "$SHARE_COUNT" -eq 0 ]]; then
    echo "No INTERMEDIATE_SYNC_SHARES configured — skipping"
    echo "Add shares to INTERMEDIATE_SYNC_SHARES in master.conf to enable mid-day sync"
elif ! check_rsync_enabled "INTERMEDIATE"; then
    warn "Intermediate rsync disabled — skipping all $SHARE_COUNT share sync(s)"
else
    check_connectivity
    check_remote_rootfs

    RSYNC_DRY=""
    [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"

    for SHARE in "${INTERMEDIATE_SYNC_SHARES[@]}"; do
        (( SHARE_INDEX++ ))
        SHARE_NAME=$(basename "$SHARE")
        SHARE_START=$(date +%s)

        echo ""
        echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"

        if [[ "$ABORT_ALL_SYNCS" == true ]]; then
            warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
            FAIL+=("$SHARE_NAME:temp-critical")
            continue
        fi

        bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
        RSYNC_EXIT=$?

        SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")

        case "$RSYNC_EXIT" in
            0)
                PASS+=("$SHARE_NAME")
                echo "$SHARE_NAME — done ✅"
                ;;
            1)
                FAIL+=("$SHARE_NAME:temp-warn")
                warn "$SHARE_NAME skipped — drive temps too high"
                ;;
            2)
                FAIL+=("$SHARE_NAME:temp-critical")
                ABORT_ALL_SYNCS=true
                error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
                notify "Intermediate sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
                    "Intermediate Sync" "warning"
                ;;
            *)
                FAIL+=("$SHARE_NAME")
                error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
                ;;
        esac
    done
fi

TOTAL_END=$(date +%s)

# ==============================================================================================
# ━━━ Maintenance Jobs ━━━
# ==============================================================================================
if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_GEAR Maintenance Jobs ━━━"
    for script_entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
        [[ -z "$script_entry" ]] && continue
        echo ""
        run_orch_child "$script_entry"
    done
fi

WINDOW_END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))

# Full breakdown on failure or --log; minimal one-liner otherwise (4-hour cadence — keep it quiet).
SHOW_FULL=false
[[ "$TOTAL_FAIL" -gt 0 || "$ENABLE_LOGGING" == true ]] && SHOW_FULL=true

if [[ "$SHOW_FULL" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━"
    echo "$ICON_HOST   Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_TIME   Window:   $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
    echo "$ICON_TIME   Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
    echo ""

    if [[ "$SHARE_COUNT" -gt 0 ]]; then
        echo "$ICON_SYNC  Shares ($SHARE_COUNT):"
        for entry in "${SHARE_TIMES[@]}"; do
            sname="${entry%%:*}"
            sdur="${entry##*:}"
            if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
                echo "  $ICON_ERROR $sname — $(format_duration "$sdur")"
            else
                echo "  $ICON_DONE  $sname — $(format_duration "$sdur")"
            fi
        done
        echo "  Passed: ${#PASS[@]}/$SHARE_COUNT  Failed: ${#FAIL[@]}/$SHARE_COUNT"
        echo ""
    fi

    if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
        echo "$ICON_GEAR  Jobs:"
        for job in "${JOB_PASS[@]}"; do echo "  $ICON_DONE  $job"; done
        for job in "${JOB_FAIL[@]}"; do echo "  $ICON_ERROR $job"; done
        echo ""
    fi
fi

# Standard ending, quiet mode — 4-hour cadence, so an OK cycle is one parseable line and
# anything skipped or failed expands to the full block on its own.
_mode=quiet; [[ "$ENABLE_LOGGING" == true ]] && _mode=full
orchestrator_summary "INTERMEDIATE SYNC" "$WINDOW_START" "Intermediate Sync" "$_mode"
exit $?