#!/bin/bash
# ==============================================================================================
# ============================= Weekly Sync Maintenance ========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Weekly maintenance window orchestrator — clean Emby sync, container updates,
# and weekly restarts. Schedule: 30 2 * * 0 (Sunday 2:30am)
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
#   1. Stop local containers       — Emby + auth stack stopped locally
#   2. Stop remote containers      — Emby + auth stack stopped remotely via SSH
#   3. Pull updates locally        — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
#   4. Pull updates remotely       — if WEEKLY_SYNC_UPDATES_REMOTE=true
#   5. rsync WEEKLY_SYNC_SHARES    — full clean mirror, containers stopped both sides
#   6. Start remote containers     — correct order, delayed start respected
#   7. Start local containers      — rebuild if new image pulled, docker start otherwise
#   8. WEEKLY_MAINTENANCE_SCRIPTS  — weekly restarts etc. (docker_weekly_restart.sh)
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Weekly Cadence Preserves Cache
#   Emby builds a warm image cache on HOST2 throughout the week. Syncing nightly
#   resets that cache — cold loads every morning for users. Weekly sync keeps
#   the cache warm for 6 days, resets Sunday night while users sleep.
#   play_state_sync covers watch/resume state every 30 min between weekly syncs.
#
# Zero-Downtime Updates
#   Containers are already stopped for the sync window — image pulls happen at
#   zero extra downtime. Both servers start on identical image versions after
#   the window completes.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Enforcement
#   Container stop/start and rsync both require root.
#
# Lock Acquisition
#   acquire_lock prevents concurrent weekly windows. This window stops Emby and the auth
#   stack — two overlapping runs would fight over the same critical containers.
#
# Host Detection
#   detect_hosts() sets MY_ID for the banner, summary and notifications.
#
# Empty Job List Guard
#   Exits with an error and a notification if WEEKLY_MAINTENANCE_SCRIPTS is empty, rather
#   than taking the weekly outage window and doing nothing with it.
#
# Connectivity Check
#   check_connectivity verifies the remote before any remote operation is attempted.
#
# Remote Rootfs Check
#   check_remote_rootfs aborts rsync if the remote rootfs is nearly full.
#
# Timeout Protection
#   DOCKER_TIMEOUT bounds every docker call and SSH_TIMEOUT every SSH call, so neither a
#   hung daemon nor an unresponsive partner can hold the weekly window open indefinitely.
#
# Non-Fatal Steps
#   A failing job is recorded and the remaining jobs still run.
#
# Silent on Success
#   Runs weekly; only failures warrant a notification.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   WEEKLY_SYNC_SHARES          — shares synced during window
#   WEEKLY_MAINTENANCE_SCRIPTS  — scripts run after sync
#   WEEKLY_SYNC_UPDATES         — toggle local container updates
#   WEEKLY_SYNC_UPDATES_REMOTE  — toggle remote container updates
#   WEEKLY_RSYNC_ENABLED        — enable/disable rsync section
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# weekly_sync_maintenance.sh
#     Normal run.
#
# weekly_sync_maintenance.sh --dry-run
#     Preview without stopping containers or syncing.
#
# weekly_sync_maintenance.sh --log
#     Verbose per-share/per-job output.
#
# weekly_sync_maintenance.sh --status
#     Show configuration and exit.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"

parse_args "$@"

DOCKER_TIMEOUT=30   # container stop/start needs longer than normal
SSH_TIMEOUT=30      # remote pulls can be slow

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi


acquire_lock

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts

# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
# from a healthy run. Fail loudly instead of silently doing no work.
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
    error "WEEKLY_MAINTENANCE_SCRIPTS is empty — no weekly maintenance scripts will run"
    error "Check WEEKLY_MAINTENANCE_SCRIPTS in master.conf"
    notify "weekly maintenance scripts skipped on $(hostname) ($MY_ID) — WEEKLY_MAINTENANCE_SCRIPTS is empty" \
        "$(basename "$0" .sh)" "warning"
    exit 1
fi
resolve_remote_ip

WINDOW_START=$(date +%s)
PASS=()
FAIL=()
JOB_PASS=()
JOB_FAIL=()

# Load container lists from profile config
read -r -a MAINTENANCE_CONTAINERS <<< \
    "${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━"
    echo "$ICON_HOST   Identity:        $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_NET    Remote:          $REMOTE_ID ($REMOTE_SERVER_NAME)"
    echo "$ICON_SYNC   Rsync enabled:   ${RSYNC_ENABLED:-true}"
    echo "$ICON_SYNC   Weekly enabled:  ${WEEKLY_RSYNC_ENABLED:-false}"
    echo "$ICON_GEAR   Local updates:   ${WEEKLY_SYNC_UPDATES:-false}"
    echo "$ICON_GEAR   Remote updates:  ${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
    echo ""
    echo "━━━ Weekly Sync Shares ━━━"
    if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then
        warn "  No WEEKLY_SYNC_SHARES configured"
    else
        for share in "${WEEKLY_SYNC_SHARES[@]}"; do
            [[ -n "$share" ]] && echo "  $ICON_SYNC $(basename "$share") ($share)"
        done
    fi
    echo ""
    echo "━━━ Weekly Maintenance Scripts ━━━"
    if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
        echo "  None configured"
    else
        for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
            [[ -n "$entry" ]] && echo "  $ICON_GEAR ${entry##*/}"
        done
    fi
    echo ""
    echo "━━━ Containers (from profile config) ━━━"
    for c in "${MAINTENANCE_CONTAINERS[@]}"; do
        [[ -n "$c" ]] && echo "  $ICON_CONTAINERS $c"
    done
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Pre-flight Checks ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo ""
echo "━━━ $ICON_SHIELD Pre-flight ━━━"

check_connectivity
check_remote_rootfs

# ==============================================================================================
# ━━━ Stop Containers ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — containers will not be stopped"
else
    # Load container lists for stop functions
    read -r -a CRITICAL_CONTAINER_NAMES <<< \
        "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
    read -r -a LOCAL_CRITICAL_CONTAINER_NAMES <<< \
        "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
    read -r -a DELAYED_CONTAINERS       <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
    CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"

    stop_local_containers
    stop_containers
fi

# ==============================================================================================
# ━━━ Container Updates ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Container Updates ━━━"

declare -A _weekly_needs_rebuild=()

if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
    if [[ "$DRY_RUN" == true ]]; then
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
        done
    else
        log "Pulling local container updates..."
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -z "$c" ]] && continue
            IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
                "$c" --format '{{.Config.Image}}' 2>/dev/null)
            if [[ -z "$IMAGE" ]]; then
                log "$c — not found locally, skipping update"
                continue
            fi
            _old_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
            log "Pulling $IMAGE for $c..."
            if docker pull "$IMAGE" >/dev/null 2>&1; then
                _new_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")
                if [[ -n "$_old_id" && "$_old_id" != "$_new_id" ]]; then
                    log "$c — new image (${_old_id:7:12} → ${_new_id:7:12}) — will rebuild after sync"
                    _weekly_needs_rebuild["$c"]=1
                else
                    log "$c — already current"
                fi
            else
                warn "$c — pull failed, will start on existing image"
            fi
        done
        unset _old_id _new_id
    fi
else
    echo "WEEKLY_SYNC_UPDATES=false — skipping local updates"
fi

if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
    else
        log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -z "$c" ]] && continue
            IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
                -o ConnectTimeout="$SSH_TIMEOUT" \
                root@"$REMOTE_SERVER" \
                "docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
            if [[ -z "$IMAGE" ]]; then
                log "$c — not found on remote, skipping update"
                continue
            fi
            log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
            if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
                -o ConnectTimeout="$SSH_TIMEOUT" \
                root@"$REMOTE_SERVER" \
                "docker pull $IMAGE" >/dev/null 2>&1; then
                echo "$c — remote image updated ✅"
            else
                warn "$c — remote pull failed, will start on existing image"
            fi
        done
    fi
else
    echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
fi

# ==============================================================================================
# ━━━ Critical Shares Sync ━━━
# ==============================================================================================
SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]}

echo ""
echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━"

SYNC_START=$(date +%s)
JOB_NUM=0

if ! check_rsync_enabled "WEEKLY"; then
    warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)"
    warn "Proceeding to container start and maintenance scripts..."
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
    warn "No WEEKLY_SYNC_SHARES configured — skipping sync"
    warn "Check WEEKLY_SYNC_SHARES in master.conf"
else
    RSYNC_DRY=""
    [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"

    for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do
        (( JOB_NUM++ ))
        JOB_NAME=$(basename "$JOB")

        echo ""
        echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"

        JOB_START=$(date +%s)
        bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY
        EXIT_CODE=$?
        JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START )))

        if [[ "$EXIT_CODE" -eq 0 ]]; then
            PASS+=("$JOB_NAME")
            echo "$JOB_NAME — done in $JOB_DUR ✅"
        else
            FAIL+=("$JOB_NAME")
            error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)"
        fi
        echo ""
    done
fi

SYNC_END=$(date +%s)

# ==============================================================================================
# ━━━ Start Containers ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — containers will not be started"
else
    start_containers

    # Local start — rebuild containers that received a new image, docker start the rest
    if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then
        log "No local containers to restart."
    else
        for _c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do
            [[ -z "$_c" ]] && continue
            _needs_delay=false
            for _d in "${DELAYED_CONTAINERS[@]}"; do
                [[ "$_c" == "$_d" ]] && _needs_delay=true && break
            done
            [[ "$_needs_delay" == true ]] && {
                info "Waiting ${CONTAINER_DELAY}s before starting $_c..."
                sleep "$CONTAINER_DELAY"
            }
            if [[ -n "${_weekly_needs_rebuild[$_c]:-}" ]]; then
                log "Rebuilding $_c on new image..."
                if platform_rebuild_container "$_c"; then
                    echo "$_c rebuilt on new image ✅"
                else
                    warn "$_c rebuild failed — falling back to docker start"
                    docker start "$_c" >/dev/null 2>&1 || error "Failed to start $_c"
                fi
            else
                docker start "$_c" >/dev/null 2>&1 && echo "$_c started" || error "Failed to start $_c"
            fi
        done
        unset _c _d _needs_delay
    fi
fi

# ==============================================================================================
# ━━━ Post-sync Jobs ━━━
# ==============================================================================================
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
    for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
        [[ -z "$script_entry" ]] && continue
        echo ""
        run_orch_child "$script_entry"
    done
fi

WINDOW_END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
echo "$ICON_HOST   Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME   Window:    $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
echo "$ICON_TIME   Duration:  $(format_duration $(( WINDOW_END - WINDOW_START )))"
echo "$ICON_GEAR   Updates:   local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
echo ""

echo "$ICON_SYNC  Sync jobs ($SHARE_COUNT):"
for job in "${PASS[@]}"; do echo "  $ICON_DONE  $job"; done
for job in "${FAIL[@]}"; do echo "  $ICON_ERROR $job"; done
echo "  Passed: ${#PASS[@]}  Failed: ${#FAIL[@]}"

if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
    echo ""
    echo "$ICON_GEAR  Post-sync jobs:"
    for job in "${JOB_PASS[@]}"; do echo "  $ICON_DONE  $job"; done
    for job in "${JOB_FAIL[@]}"; do echo "  $ICON_ERROR $job"; done
fi

TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))

echo ""
if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
    echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
else
    warn "Status: $TOTAL_FAIL failure(s)"
    notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
        "Weekly Maintenance" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
exit 0