153 lines
5.7 KiB
Bash
153 lines
5.7 KiB
Bash
#!/bin/bash
|
|
# -----------------------------------------------------------------------------------------------
|
|
# --------------------------------- Critical Shares Full Sync ----------------------------------
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Runs clean nightly sync for Emby and the auth stack (Critical-Data).
|
|
# Both require containers stopped for a consistent, safe state sync.
|
|
#
|
|
# Why this exists as a separate orchestrator from media_shares_sync.sh:
|
|
# media_shares_sync.sh handles media shares — large, runs at 1am, no container stops needed
|
|
# This script handles appdata that needs containers stopped for clean state:
|
|
#
|
|
# Emby:
|
|
# Hourly dirty sync runs continuously (containers up, WAL excluded)
|
|
# Nightly clean sync stops Emby → syncs full clean state → restarts Emby
|
|
# Ensures HOST2 has a fully consistent Emby database nightly
|
|
#
|
|
# Critical-Data (auth stack):
|
|
# NPM, Authelia, Mariadb-Authelia, Redis-Authelia, LLDAP
|
|
# Containers stopped during sync — databases flush cleanly
|
|
# HOST2 gets a clean auth state nightly
|
|
# Users, groups, proxy rules, SSL certs all consistent
|
|
# Adding a user on HOST1 → propagates to HOST2 overnight automatically
|
|
#
|
|
# Schedule: 2:30am daily — after media_shares_sync.sh (1am) finishes
|
|
# Container stop time is brief — Emby ~30s, auth stack ~15s
|
|
# Users experience a short Emby interruption at 2:30am — acceptable tradeoff
|
|
#
|
|
# For most users: this script ensures failover always has a clean working auth state
|
|
# without needing to think about dirty writes or WAL files.
|
|
#
|
|
# All configuration in Master.conf — rsync profiles handle container stops automatically.
|
|
# Supports --dry-run to walk through without stopping containers or syncing.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../Master.conf"
|
|
source "$SCRIPT_DIR/../common.sh"
|
|
|
|
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_GEAR Setup ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
success "Running as root"
|
|
|
|
acquire_lock
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
|
|
|
|
check_connectivity
|
|
check_remote_rootfs
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Tracking
|
|
# -----------------------------------------------------------------------------------------------
|
|
PASS=()
|
|
FAIL=()
|
|
SHARE_TIMES=()
|
|
TOTAL_START=$(date +%s)
|
|
|
|
SYNC_JOBS=(
|
|
"/mnt/user/Media_Server/Emby"
|
|
"/mnt/user/appdata-Failover/Critical-Data"
|
|
)
|
|
|
|
SHARE_COUNT=${#SYNC_JOBS[@]}
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SYNC Nightly Clean Sync ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Nightly Clean Sync Starting — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_SUMMARY Jobs: $SHARE_COUNT"
|
|
echo ""
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — containers will not be stopped"
|
|
|
|
JOB_NUM=0
|
|
for JOB in "${SYNC_JOBS[@]}"; do
|
|
((JOB_NUM++))
|
|
JOB_NAME=$(basename "$JOB")
|
|
echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"
|
|
|
|
JOB_START=$(date +%s)
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
bash "$RSYNC_SCRIPT" "$JOB" --dry-run
|
|
else
|
|
bash "$RSYNC_SCRIPT" "$JOB"
|
|
fi
|
|
|
|
EXIT_CODE=$?
|
|
JOB_END=$(date +%s)
|
|
JOB_DURATION=$(format_duration $(( JOB_END - JOB_START )))
|
|
|
|
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
|
PASS+=("$JOB_NAME")
|
|
success "$JOB_NAME — $ICON_SUCCESS done in $JOB_DURATION"
|
|
else
|
|
FAIL+=("$JOB_NAME")
|
|
error "$JOB_NAME — $ICON_ERROR failed after $JOB_DURATION"
|
|
fi
|
|
|
|
echo ""
|
|
done
|
|
|
|
TOTAL_END=$(date +%s)
|
|
TOTAL_DURATION=$(format_duration $(( TOTAL_END - TOTAL_START )))
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SUMMARY Summary ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo "━━━━━ $ICON_SUMMARY NIGHTLY SYNC SUMMARY ━━━━━"
|
|
echo "$ICON_TIME Duration: $TOTAL_DURATION"
|
|
echo "$ICON_SUCCESS Passed: ${#PASS[@]} $ICON_ERROR Failed: ${#FAIL[@]}"
|
|
echo ""
|
|
|
|
if [[ ${#PASS[@]} -gt 0 ]]; then
|
|
for job in "${PASS[@]}"; do echo " $ICON_SUCCESS $job"; done
|
|
fi
|
|
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
|
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
|
fi
|
|
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
|
elif [[ ${#FAIL[@]} -eq 0 ]]; then
|
|
echo "$ICON_DONE Status: $ICON_SUCCESS ALL JOBS COMPLETE"
|
|
notify "Nightly sync complete on $(hostname) — Emby + auth stack synced cleanly" "Nightly Sync" "normal"
|
|
else
|
|
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAIL[@]} JOB(S) FAILED"
|
|
notify "Nightly sync failed on $(hostname) — failed: ${FAIL[*]}" "Nightly Sync" "warning"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |