172 lines
6.6 KiB
Bash
172 lines
6.6 KiB
Bash
#!/bin/bash
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ----------------------------- Critical Sync Maintenance --------------------------------------
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Orchestrator for time-sensitive syncs that run every 15 minutes.
|
|
# Keeps the mirror current between the less frequent daily and weekly windows.
|
|
# Schedule: */15 * * * * (every 15 minutes)
|
|
#
|
|
# Execution order:
|
|
# 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides)
|
|
# 2. emby-failover rsync — dirty Emby sync (watch states, library — Emby stays running)
|
|
# 3. partnership --check — read both state files, detect changes, act accordingly
|
|
#
|
|
# Why every 15 minutes:
|
|
# Auth stack changes (new users, proxy rules, certs) propagate within 15min ✅
|
|
# Emby watch states stay in sync — mirror users see correct playback position ✅
|
|
# Partnership state changes detected and acted on quickly ✅
|
|
#
|
|
# Rsync gate:
|
|
# RSYNC_ENABLED=false → skips all syncs (global gate)
|
|
# CRITICAL_RSYNC_ENABLED=false → skips critical syncs (per-orchestrator)
|
|
# partnership --check still runs regardless of rsync gate
|
|
# (state check doesn't need rsync to work)
|
|
#
|
|
# Lock behavior:
|
|
# acquire_lock "strict" — if previous 15min run still going, skip this cycle
|
|
# Critical-Data taking > 15min is a problem worth knowing about
|
|
# Lock prevents pile-up ✅
|
|
#
|
|
# Configuration in Master.conf:
|
|
# CRITICAL_RSYNC_ENABLED — enable/disable rsync section
|
|
# PARTNERSHIP_ENABLED — enable/disable partnership check
|
|
# CRITICAL_SYNC_SHARES — shares synced every 15min
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../Master.conf"
|
|
source "$SCRIPT_DIR/../common.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_GEAR Setup ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
acquire_lock "strict"
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
START=$(date +%s)
|
|
RSYNC_OK=false
|
|
PASS=()
|
|
FAIL=()
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SYNC Critical Shares Sync ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Critical Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
|
|
if ! check_rsync_enabled "CRITICAL"; then
|
|
warn "Critical rsync disabled — skipping sync, running partnership check only"
|
|
else
|
|
for share in "${CRITICAL_SYNC_SHARES[@]:-}"; do
|
|
[[ -z "$share" ]] && continue
|
|
|
|
# Parse optional profile flag: "/path/to/share|profile-name"
|
|
SHARE_PATH="${share%%|*}"
|
|
SHARE_PROFILE="${share##*|}"
|
|
SHARE_NAME=$(basename "$SHARE_PATH")
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC $SHARE_NAME ━━━"
|
|
|
|
SHARE_START=$(date +%s)
|
|
|
|
if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then
|
|
# No profile specified
|
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH"
|
|
else
|
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" --profile="$SHARE_PROFILE"
|
|
fi
|
|
|
|
RSYNC_EXIT=$?
|
|
SHARE_END=$(date +%s)
|
|
SHARE_DUR=$(format_duration $(( SHARE_END - SHARE_START )))
|
|
|
|
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
|
|
PASS+=("$SHARE_NAME")
|
|
success "$SHARE_NAME — done in $SHARE_DUR ✅"
|
|
RSYNC_OK=true
|
|
else
|
|
FAIL+=("$SHARE_NAME")
|
|
error "$SHARE_NAME — failed after $SHARE_DUR"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_CLEAN Critical Maintenance Scripts ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_CLEAN Critical Maintenance — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
|
|
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
|
log "No CRITICAL_MAINTENANCE_SCRIPTS defined — skipping"
|
|
else
|
|
for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$script_entry" ]] && continue
|
|
# Strip leading comment lines
|
|
[[ "$script_entry" == \#* ]] && continue
|
|
|
|
SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
|
|
SCRIPT_ARGS="${script_entry#* }"
|
|
[[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
|
|
|
|
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
|
|
echo " → $SCRIPT_NAME"
|
|
|
|
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
|
warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
|
|
continue
|
|
fi
|
|
|
|
bash "$SCRIPT_PATH" $SCRIPT_ARGS
|
|
EXIT_CODE=$?
|
|
if [[ "$EXIT_CODE" -ne 0 ]]; then
|
|
warn "$SCRIPT_NAME exited with code $EXIT_CODE"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SHIELD Partnership Check ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Partnership Check ━━━"
|
|
|
|
if [[ "${PARTNERSHIP_ENABLED:-false}" == false ]]; then
|
|
log "Partnership disabled — skipping check"
|
|
else
|
|
# Pass rsync outcome to --check so it can update last_seen_remote
|
|
if [[ "$RSYNC_OK" == true ]]; then
|
|
bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-seen
|
|
else
|
|
bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-unseen
|
|
fi
|
|
fi
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SUMMARY Summary ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
END=$(date +%s)
|
|
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
|
|
if [[ ${#PASS[@]} -gt 0 ]]; then
|
|
echo "$ICON_SUCCESS Synced: ${PASS[*]}"
|
|
fi
|
|
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
|
echo "$ICON_ERROR Failed: ${FAIL[*]}"
|
|
notify "Critical sync failed on $(hostname) — ${FAIL[*]}" "Critical Sync" "warning"
|
|
fi
|
|
if [[ ${#PASS[@]} -eq 0 ]] && [[ ${#FAIL[@]} -eq 0 ]]; then
|
|
echo "$ICON_SKIP Rsync: disabled"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |