#!/bin/bash # ============================================================================================== # ============================= Critical Sync Maintenance ====================================== # ============================================================================================== # Orchestrator for time-sensitive syncs that run every 15 minutes. # Keeps the mirror current between the less frequent daily and weekly windows. # Schedule: */15 * * * * (every 15 minutes via User Scripts plugin) # # ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── # 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides) # 2. emby-fallback rsync — dirty Emby sync (watch states, library — Emby stays running) # 3. CRITICAL_MAINTENANCE_SCRIPTS — any scripts configured for critical window # 4. partnership --check — read both state files, detect changes, act accordingly # # ── WHY EVERY 15 MINUTES ────────────────────────────────────────────────────────────────────── # Auth stack changes (new users, proxy rules, certs) propagate within 15min ✅ # Emby watch states stay in sync — mirror users see correct playback position ✅ # Partnership state changes detected and acted on quickly ✅ # Lock prevents: daily rsync doing Critical-Data mid-critical window ✅ # # ── RSYNC GATE ──────────────────────────────────────────────────────────────────────────────── # RSYNC_ENABLED=false → skips all syncs (global gate) # CRITICAL_RSYNC_ENABLED=false → skips critical syncs only (per-orchestrator gate) # partnership --check always runs regardless — state check doesn't need rsync # # ── LOCK BEHAVIOUR ──────────────────────────────────────────────────────────────────────────── # acquire_lock "strict" — if previous 15min run still going, skip this cycle entirely # Critical-Data taking > 15min is a problem worth knowing about # Strict mode prevents pile-up without waiting — log and move on ✅ # # ── SILENT WHEN HEALTHY ─────────────────────────────────────────────────────────────────────── # Runs 96 times per day — clean runs must produce zero output ✅ # Only failures and notable events produce visible output # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # CRITICAL_RSYNC_ENABLED — enable/disable rsync section # CRITICAL_SYNC_SHARES — shares synced every 15min (HOST*_CRITICAL_SYNC_SHARES) # CRITICAL_MAINTENANCE_SCRIPTS — scripts run in critical window (optional) # PARTNERSHIP_ENABLED — enable/disable partnership check # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # critical_sync_maintenance.sh — normal run # critical_sync_maintenance.sh --dry-run — preview syncs without transferring # critical_sync_maintenance.sh --log — verbose per-share output # critical_sync_maintenance.sh --status — show configuration and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi validate_unraid_cmd \ "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ "" "" \ "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" acquire_lock "strict" detect_hosts resolve_remote_ip [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)" echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}" echo "$ICON_SYNC Critical enabled: ${CRITICAL_RSYNC_ENABLED:-false}" echo "$ICON_SHIELD Partnership: ${PARTNERSHIP_ENABLED:-false}" echo "" echo "━━━ Critical Sync Shares ━━━" if [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then warn " No CRITICAL_SYNC_SHARES configured" else for share in "${CRITICAL_SYNC_SHARES[@]}"; do [[ -z "$share" ]] && continue SHARE_PATH="${share%%|*}" SHARE_PROFILE="${share##*|}" SHARE_NAME=$(basename "$SHARE_PATH") [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]] && \ echo " $ICON_SYNC $SHARE_NAME — no profile" || \ echo " $ICON_SYNC $SHARE_NAME — profile: $SHARE_PROFILE" done fi echo "" echo "━━━ Critical Maintenance Scripts ━━━" if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then echo " None configured" else for entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do [[ -z "$entry" || "$entry" == \#* ]] && continue echo " $ICON_GEAR $(basename "${entry%% *}")" done fi echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ━━━ Critical Shares Sync ━━━ # ============================================================================================== START=$(date +%s) RSYNC_OK=false PASS=() FAIL=() if ! check_rsync_enabled "CRITICAL"; then echo "Critical rsync disabled — skipping sync, running partnership check only" elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID" warn "Check HOST*_CRITICAL_SYNC_SHARES in host*.conf" else echo "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')" # Build dry-run flag to pass through RSYNC_DRY="" [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run" for share in "${CRITICAL_SYNC_SHARES[@]}"; do [[ -z "$share" ]] && continue # Parse optional profile flag: "/path/to/share|profile-name" SHARE_PATH="${share%%|*}" SHARE_PROFILE="${share##*|}" SHARE_NAME=$(basename "$SHARE_PATH") SHARE_START=$(date +%s) if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" $RSYNC_DRY else bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" \ --profile="$SHARE_PROFILE" $RSYNC_DRY fi RSYNC_EXIT=$? SHARE_DUR=$(format_duration $(( $(date +%s) - SHARE_START ))) if [[ "$RSYNC_EXIT" -eq 0 ]]; then PASS+=("$SHARE_NAME") log "$SHARE_NAME — done in $SHARE_DUR ✅" RSYNC_OK=true else FAIL+=("$SHARE_NAME") error "$SHARE_NAME — failed after $SHARE_DUR (exit $RSYNC_EXIT)" fi done fi # ============================================================================================== # ━━━ Critical Maintenance Scripts ━━━ # ============================================================================================== if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do [[ -z "$script_entry" || "$script_entry" == \#* ]] && continue SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}" SCRIPT_ARGS="${script_entry#* }" [[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS="" [[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run" SCRIPT_NAME=$(basename "$SCRIPT_PATH") if [[ ! -f "$SCRIPT_PATH" ]]; then warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping" continue fi log "Running: $SCRIPT_NAME" bash "$SCRIPT_PATH" $SCRIPT_ARGS EXIT_CODE=$? [[ "$EXIT_CODE" -ne 0 ]] && \ warn "$SCRIPT_NAME exited with code $EXIT_CODE" done fi # ============================================================================================== # ━━━ Partnership Check ━━━ # ============================================================================================== if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then PARTNER_DRY="" [[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run" if [[ "$RSYNC_OK" == true ]]; then bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \ --check --remote-seen $PARTNER_DRY else bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \ --check --remote-unseen $PARTNER_DRY fi else echo "Partnership disabled — skipping check" fi # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== END=$(date +%s) DURATION=$(format_duration $(( END - START ))) # Silent when healthy — only show summary if there were failures or notable events if [[ ${#FAIL[@]} -gt 0 ]]; then echo "" echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $DURATION" [[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}" echo "$ICON_ERROR Failed: ${FAIL[*]}" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \ "Critical Sync" "warning" exit 1 else echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)" fi exit 0