massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
@@ -1,69 +1,136 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------------------- Critical Sync Maintenance --------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ============================= Critical Sync Maintenance ======================================
|
||||
# ==============================================================================================
|
||||
# Orchestrator for time-sensitive syncs that run every 15 minutes.
|
||||
# Keeps the mirror current between the less frequent daily and weekly windows.
|
||||
# Schedule: */15 * * * * (every 15 minutes)
|
||||
# Schedule: */15 * * * * (every 15 minutes via User Scripts plugin)
|
||||
#
|
||||
# Execution order:
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides)
|
||||
# 2. emby-failover rsync — dirty Emby sync (watch states, library — Emby stays running)
|
||||
# 3. partnership --check — read both state files, detect changes, act accordingly
|
||||
# 3. CRITICAL_MAINTENANCE_SCRIPTS — any scripts configured for critical window
|
||||
# 4. partnership --check — read both state files, detect changes, act accordingly
|
||||
#
|
||||
# Why every 15 minutes:
|
||||
# ── WHY EVERY 15 MINUTES ──────────────────────────────────────────────────────────────────────
|
||||
# Auth stack changes (new users, proxy rules, certs) propagate within 15min ✅
|
||||
# Emby watch states stay in sync — mirror users see correct playback position ✅
|
||||
# Partnership state changes detected and acted on quickly ✅
|
||||
# Lock prevents: daily rsync doing Critical-Data mid-critical window ✅
|
||||
#
|
||||
# Rsync gate:
|
||||
# RSYNC_ENABLED=false → skips all syncs (global gate)
|
||||
# CRITICAL_RSYNC_ENABLED=false → skips critical syncs (per-orchestrator)
|
||||
# partnership --check still runs regardless of rsync gate
|
||||
# (state check doesn't need rsync to work)
|
||||
# ── RSYNC GATE ────────────────────────────────────────────────────────────────────────────────
|
||||
# RSYNC_ENABLED=false → skips all syncs (global gate)
|
||||
# CRITICAL_RSYNC_ENABLED=false → skips critical syncs only (per-orchestrator gate)
|
||||
# partnership --check always runs regardless — state check doesn't need rsync
|
||||
#
|
||||
# Lock behavior:
|
||||
# acquire_lock "strict" — if previous 15min run still going, skip this cycle
|
||||
# ── LOCK BEHAVIOUR ────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock "strict" — if previous 15min run still going, skip this cycle entirely
|
||||
# Critical-Data taking > 15min is a problem worth knowing about
|
||||
# Lock prevents pile-up ✅
|
||||
# Strict mode prevents pile-up without waiting — log and move on ✅
|
||||
#
|
||||
# Configuration in Master.conf:
|
||||
# CRITICAL_RSYNC_ENABLED — enable/disable rsync section
|
||||
# PARTNERSHIP_ENABLED — enable/disable partnership check
|
||||
# CRITICAL_SYNC_SHARES — shares synced every 15min
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
|
||||
# Runs 96 times per day — clean runs must produce zero output ✅
|
||||
# Only failures and notable events produce visible output
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# CRITICAL_RSYNC_ENABLED — enable/disable rsync section
|
||||
# CRITICAL_SYNC_SHARES — shares synced every 15min (HOST*_CRITICAL_SYNC_SHARES)
|
||||
# CRITICAL_MAINTENANCE_SCRIPTS — scripts run in critical window (optional)
|
||||
# PARTNERSHIP_ENABLED — enable/disable partnership check
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# critical_sync_maintenance.sh — normal run
|
||||
# critical_sync_maintenance.sh --dry-run — preview syncs without transferring
|
||||
# critical_sync_maintenance.sh --log — verbose per-share output
|
||||
# critical_sync_maintenance.sh --status — show configuration and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock "strict"
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_SYNC Critical enabled: ${CRITICAL_RSYNC_ENABLED:-false}"
|
||||
echo "$ICON_SHIELD Partnership: ${PARTNERSHIP_ENABLED:-false}"
|
||||
echo ""
|
||||
echo "━━━ Critical Sync Shares ━━━"
|
||||
if [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||
warn " No CRITICAL_SYNC_SHARES configured"
|
||||
else
|
||||
for share in "${CRITICAL_SYNC_SHARES[@]:-}"; do
|
||||
[[ -z "$share" ]] && continue
|
||||
SHARE_PATH="${share%%|*}"
|
||||
SHARE_PROFILE="${share##*|}"
|
||||
SHARE_NAME=$(basename "$SHARE_PATH")
|
||||
[[ "$SHARE_PATH" == "$SHARE_PROFILE" ]] && \
|
||||
echo " $ICON_SYNC $SHARE_NAME — no profile" || \
|
||||
echo " $ICON_SYNC $SHARE_NAME — profile: $SHARE_PROFILE"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo "━━━ Critical Maintenance Scripts ━━━"
|
||||
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
||||
echo " None configured"
|
||||
else
|
||||
for entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]:-}"; do
|
||||
[[ -z "$entry" || "$entry" == \#* ]] && continue
|
||||
echo " $ICON_GEAR $(basename "${entry%% *}")"
|
||||
done
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Critical Shares Sync ━━━
|
||||
# ==============================================================================================
|
||||
START=$(date +%s)
|
||||
RSYNC_OK=false
|
||||
PASS=()
|
||||
FAIL=()
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Critical Shares Sync ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Critical Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
if ! check_rsync_enabled "CRITICAL"; then
|
||||
warn "Critical rsync disabled — skipping sync, running partnership check only"
|
||||
log "Critical rsync disabled — skipping sync, running partnership check only"
|
||||
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
||||
warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID"
|
||||
warn "Check HOST*_CRITICAL_SYNC_SHARES in master_host*.conf"
|
||||
else
|
||||
log "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')"
|
||||
|
||||
# Build dry-run flag to pass through
|
||||
RSYNC_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
||||
|
||||
for share in "${CRITICAL_SYNC_SHARES[@]:-}"; do
|
||||
[[ -z "$share" ]] && continue
|
||||
|
||||
@@ -72,101 +139,94 @@ else
|
||||
SHARE_PROFILE="${share##*|}"
|
||||
SHARE_NAME=$(basename "$SHARE_PATH")
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC $SHARE_NAME ━━━"
|
||||
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then
|
||||
# No profile specified
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH"
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" $RSYNC_DRY
|
||||
else
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" --profile="$SHARE_PROFILE"
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" \
|
||||
--profile="$SHARE_PROFILE" $RSYNC_DRY
|
||||
fi
|
||||
|
||||
RSYNC_EXIT=$?
|
||||
SHARE_END=$(date +%s)
|
||||
SHARE_DUR=$(format_duration $(( SHARE_END - SHARE_START )))
|
||||
SHARE_DUR=$(format_duration $(( $(date +%s) - SHARE_START )))
|
||||
|
||||
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
|
||||
PASS+=("$SHARE_NAME")
|
||||
success "$SHARE_NAME — done in $SHARE_DUR ✅"
|
||||
log "$SHARE_NAME — done in $SHARE_DUR ✅"
|
||||
RSYNC_OK=true
|
||||
else
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME — failed after $SHARE_DUR"
|
||||
error "$SHARE_NAME — failed after $SHARE_DUR (exit $RSYNC_EXIT)"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CLEAN Critical Maintenance Scripts ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Critical Maintenance — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
||||
log "No CRITICAL_MAINTENANCE_SCRIPTS defined — skipping"
|
||||
else
|
||||
# ==============================================================================================
|
||||
# ━━━ Critical Maintenance Scripts ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
# Strip leading comment lines
|
||||
[[ "$script_entry" == \#* ]] && continue
|
||||
[[ -z "$script_entry" || "$script_entry" == \#* ]] && continue
|
||||
|
||||
SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
|
||||
SCRIPT_ARGS="${script_entry#* }"
|
||||
[[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
|
||||
[[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run"
|
||||
|
||||
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
|
||||
echo " → $SCRIPT_NAME"
|
||||
|
||||
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
||||
warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Running: $SCRIPT_NAME"
|
||||
bash "$SCRIPT_PATH" $SCRIPT_ARGS
|
||||
EXIT_CODE=$?
|
||||
if [[ "$EXIT_CODE" -ne 0 ]]; then
|
||||
[[ "$EXIT_CODE" -ne 0 ]] && \
|
||||
warn "$SCRIPT_NAME exited with code $EXIT_CODE"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SHIELD Partnership Check ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Partnership Check ━━━"
|
||||
# ==============================================================================================
|
||||
# ━━━ Partnership Check ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
|
||||
PARTNER_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run"
|
||||
|
||||
if [[ "${PARTNERSHIP_ENABLED:-false}" == false ]]; then
|
||||
log "Partnership disabled — skipping check"
|
||||
else
|
||||
# Pass rsync outcome to --check so it can update last_seen_remote
|
||||
if [[ "$RSYNC_OK" == true ]]; then
|
||||
bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-seen
|
||||
bash "$SCRIPT_DIR/partnership_manage.sh" \
|
||||
--check --remote-seen $PARTNER_DRY
|
||||
else
|
||||
bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-unseen
|
||||
bash "$SCRIPT_DIR/partnership_manage.sh" \
|
||||
--check --remote-unseen $PARTNER_DRY
|
||||
fi
|
||||
else
|
||||
log "Partnership disabled — skipping check"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
END=$(date +%s)
|
||||
DURATION=$(format_duration $(( END - START )))
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
|
||||
if [[ ${#PASS[@]} -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Synced: ${PASS[*]}"
|
||||
fi
|
||||
# Silent when healthy — only show summary if there were failures or notable events
|
||||
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $DURATION"
|
||||
[[ ${#PASS[@]} -gt 0 ]] && log "Synced: ${PASS[*]}"
|
||||
echo "$ICON_ERROR Failed: ${FAIL[*]}"
|
||||
notify "Critical sync failed on $(hostname) — ${FAIL[*]}" "Critical Sync" "warning"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||||
"Critical Sync" "warning"
|
||||
exit 1
|
||||
else
|
||||
log "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)"
|
||||
fi
|
||||
if [[ ${#PASS[@]} -eq 0 ]] && [[ ${#FAIL[@]} -eq 0 ]]; then
|
||||
echo "$ICON_SKIP Rsync: disabled"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
exit 0
|
||||
Reference in New Issue
Block a user