Varaverk/Orchestrators/critical_sync_maintenance.sh

#!/bin/bash
# ==============================================================================================
# ============================= Critical Sync Maintenance ======================================
# ==============================================================================================
# Orchestrator for time-sensitive syncs that run every 15 minutes.
# Keeps the mirror current between the less frequent daily and weekly windows.
# Schedule: */15 * * * * (every 15 minutes via User Scripts plugin)
#
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
#   1. Critical-Data rsync    — auth stack, NPM config, certs (containers stopped both sides)
#   2. emby-failover rsync    — dirty Emby sync (watch states, library — Emby stays running)
#   3. CRITICAL_MAINTENANCE_SCRIPTS — any scripts configured for critical window
#   4. partnership --check    — read both state files, detect changes, act accordingly
#
# ── WHY EVERY 15 MINUTES ──────────────────────────────────────────────────────────────────────
#   Auth stack changes (new users, proxy rules, certs) propagate within 15min ✅
#   Emby watch states stay in sync — mirror users see correct playback position ✅
#   Partnership state changes detected and acted on quickly ✅
#   Lock prevents: daily rsync doing Critical-Data mid-critical window ✅
#
# ── RSYNC GATE ────────────────────────────────────────────────────────────────────────────────
#   RSYNC_ENABLED=false           → skips all syncs (global gate)
#   CRITICAL_RSYNC_ENABLED=false  → skips critical syncs only (per-orchestrator gate)
#   partnership --check always runs regardless — state check doesn't need rsync
#
# ── LOCK BEHAVIOUR ────────────────────────────────────────────────────────────────────────────
#   acquire_lock "strict" — if previous 15min run still going, skip this cycle entirely
#   Critical-Data taking > 15min is a problem worth knowing about
#   Strict mode prevents pile-up without waiting — log and move on ✅
#
# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
#   Runs 96 times per day — clean runs must produce zero output ✅
#   Only failures and notable events produce visible output
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   CRITICAL_RSYNC_ENABLED       — enable/disable rsync section
#   CRITICAL_SYNC_SHARES         — shares synced every 15min (HOST*_CRITICAL_SYNC_SHARES)
#   CRITICAL_MAINTENANCE_SCRIPTS — scripts run in critical window (optional)
#   PARTNERSHIP_ENABLED          — enable/disable partnership check
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   critical_sync_maintenance.sh              — normal run
#   critical_sync_maintenance.sh --dry-run    — preview syncs without transferring
#   critical_sync_maintenance.sh --log        — verbose per-share output
#   critical_sync_maintenance.sh --status     — show configuration and exit
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

validate_unraid_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock "strict"

detect_hosts
resolve_remote_ip

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC STATUS ━━━━━"
    echo "$ICON_HOST    Identity:           $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_NET     Remote:             $REMOTE_ID ($REMOTE_SERVER_NAME)"
    echo "$ICON_SYNC    Rsync enabled:      ${RSYNC_ENABLED:-true}"
    echo "$ICON_SYNC    Critical enabled:   ${CRITICAL_RSYNC_ENABLED:-false}"
    echo "$ICON_SHIELD  Partnership:        ${PARTNERSHIP_ENABLED:-false}"
    echo ""
    echo "━━━ Critical Sync Shares ━━━"
    if [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
        warn "  No CRITICAL_SYNC_SHARES configured"
    else
        for share in "${CRITICAL_SYNC_SHARES[@]}"; do
            [[ -z "$share" ]] && continue
            SHARE_PATH="${share%%|*}"
            SHARE_PROFILE="${share##*|}"
            SHARE_NAME=$(basename "$SHARE_PATH")
            [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]] && \
                echo "  $ICON_SYNC $SHARE_NAME — no profile" || \
                echo "  $ICON_SYNC $SHARE_NAME — profile: $SHARE_PROFILE"
        done
    fi
    echo ""
    echo "━━━ Critical Maintenance Scripts ━━━"
    if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
        echo "  None configured"
    else
        for entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
            [[ -z "$entry" || "$entry" == \#* ]] && continue
            echo "  $ICON_GEAR $(basename "${entry%% *}")"
        done
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Critical Shares Sync ━━━
# ==============================================================================================
START=$(date +%s)
RSYNC_OK=false
PASS=()
FAIL=()

if ! check_rsync_enabled "CRITICAL"; then
    log "Critical rsync disabled — skipping sync, running partnership check only"
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
    warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID"
    warn "Check HOST*_CRITICAL_SYNC_SHARES in master_host*.conf"
else
    log "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')"

    # Build dry-run flag to pass through
    RSYNC_DRY=""
    [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"

    for share in "${CRITICAL_SYNC_SHARES[@]}"; do
        [[ -z "$share" ]] && continue

        # Parse optional profile flag: "/path/to/share|profile-name"
        SHARE_PATH="${share%%|*}"
        SHARE_PROFILE="${share##*|}"
        SHARE_NAME=$(basename "$SHARE_PATH")

        SHARE_START=$(date +%s)

        if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then
            bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" $RSYNC_DRY
        else
            bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" \
                --profile="$SHARE_PROFILE" $RSYNC_DRY
        fi

        RSYNC_EXIT=$?
        SHARE_DUR=$(format_duration $(( $(date +%s) - SHARE_START )))

        if [[ "$RSYNC_EXIT" -eq 0 ]]; then
            PASS+=("$SHARE_NAME")
            log "$SHARE_NAME — done in $SHARE_DUR ✅"
            RSYNC_OK=true
        else
            FAIL+=("$SHARE_NAME")
            error "$SHARE_NAME — failed after $SHARE_DUR (exit $RSYNC_EXIT)"
        fi
    done
fi

# ==============================================================================================
# ━━━ Critical Maintenance Scripts ━━━
# ==============================================================================================
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
    for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
        [[ -z "$script_entry" || "$script_entry" == \#* ]] && continue

        SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
        SCRIPT_ARGS="${script_entry#* }"
        [[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
        [[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run"

        SCRIPT_NAME=$(basename "$SCRIPT_PATH")

        if [[ ! -f "$SCRIPT_PATH" ]]; then
            warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
            continue
        fi

        log "Running: $SCRIPT_NAME"
        bash "$SCRIPT_PATH" $SCRIPT_ARGS
        EXIT_CODE=$?
        [[ "$EXIT_CODE" -ne 0 ]] && \
            warn "$SCRIPT_NAME exited with code $EXIT_CODE"
    done
fi

# ==============================================================================================
# ━━━ Partnership Check ━━━
# ==============================================================================================
if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
    PARTNER_DRY=""
    [[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run"

    if [[ "$RSYNC_OK" == true ]]; then
        bash "$SCRIPT_DIR/partnership_manage.sh" \
            --check --remote-seen $PARTNER_DRY
    else
        bash "$SCRIPT_DIR/partnership_manage.sh" \
            --check --remote-unseen $PARTNER_DRY
    fi
else
    log "Partnership disabled — skipping check"
fi

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
END=$(date +%s)
DURATION=$(format_duration $(( END - START )))

# Silent when healthy — only show summary if there were failures or notable events
if [[ ${#FAIL[@]} -gt 0 ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
    echo "$ICON_HOST   Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_TIME   Duration: $DURATION"
    [[ ${#PASS[@]} -gt 0 ]] && log "Synced:  ${PASS[*]}"
    echo "$ICON_ERROR  Failed:  ${FAIL[*]}"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
        "Critical Sync" "warning"
    exit 1
else
    log "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)"
fi

exit 0