252 lines
10 KiB
Bash
252 lines
10 KiB
Bash
#!/bin/bash
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ----------------------------- Critical Shares Maintenance ------------------------------------
|
|
# -----------------------------------------------------------------------------------------------
|
|
# Maintenance window orchestrator for Emby and the auth stack (Critical-Data).
|
|
# Both local and remote containers are stopped for the entire window — clean state
|
|
# guaranteed for sync, updates, and restart.
|
|
#
|
|
# What it does (in order):
|
|
# 1. Stop local containers — auth stack + Emby stopped locally
|
|
# 2. Stop remote containers — auth stack + Emby stopped remotely via SSH
|
|
# 3. Pull updates locally — if CRITICAL_SYNC_UPDATES=true
|
|
# 4. Pull updates remotely — if CRITICAL_SYNC_UPDATES_REMOTE=true
|
|
# 5. rsync Emby — clean full sync, both sides down
|
|
# 6. rsync Critical-Data — clean full sync, databases fully flushed
|
|
# 7. Start remote containers — starts on new images, correct order
|
|
# 8. Start local containers — starts on new images, correct order
|
|
#
|
|
# Why this is better than separate update + sync jobs:
|
|
# Containers are already stopped for the sync — no extra downtime for updates
|
|
# Both sides on identical image versions after restart
|
|
# Databases synced before first start on new version — clean state guaranteed
|
|
# One maintenance window handles sync + updates + ordered restart
|
|
#
|
|
# Toggle updates on/off in Master.conf:
|
|
# CRITICAL_SYNC_UPDATES=true/false — local updates
|
|
# CRITICAL_SYNC_UPDATES_REMOTE=true/false — remote updates
|
|
# Both false = sync only (sync only, no updates)
|
|
#
|
|
# Container lists and startup order from PROFILE_CRITICAL_CONTAINER_NAMES
|
|
# Delayed containers (Authelia, Authelia-Secondary) respected on restart
|
|
# Containers not found on a server skipped gracefully
|
|
# Only containers that were running get restarted — stopped containers stay stopped
|
|
#
|
|
# Recommended schedule: 30 2 * * 0 (Sunday 2:30am)
|
|
# All configuration in Master.conf.
|
|
# Supports --dry-run to walk through without stopping containers, syncing, or updating.
|
|
# -----------------------------------------------------------------------------------------------
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../Master.conf"
|
|
source "$SCRIPT_DIR/../common.sh"
|
|
|
|
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_GEAR Setup ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
|
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
success "Running as root"
|
|
|
|
acquire_lock
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
# Load container list from emby profile — used for update pulls
|
|
read -r -a MAINTENANCE_CONTAINERS <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
|
|
|
|
check_connectivity
|
|
check_remote_rootfs
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — containers will not be stopped"
|
|
else
|
|
# Load critical-data + emby container list for stops
|
|
read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
|
|
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
|
|
CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"
|
|
|
|
stop_local_containers
|
|
stop_containers
|
|
fi
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_GEAR Container Updates ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Container Updates ━━━"
|
|
|
|
if [[ "$CRITICAL_SYNC_UPDATES" == true ]]; then
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would pull updates for local containers"
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
warn "DRY RUN — would docker pull: $c"
|
|
done
|
|
else
|
|
info "Pulling local container updates..."
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
# Get image name from running or stopped container
|
|
IMAGE=$(docker inspect "$c" --format '{{.Config.Image}}' 2>/dev/null)
|
|
if [[ -z "$IMAGE" ]]; then
|
|
log "$c — not found locally, skipping update"
|
|
continue
|
|
fi
|
|
info "Pulling $IMAGE for $c..."
|
|
if docker pull "$IMAGE" >/dev/null 2>&1; then
|
|
success "$c — image updated"
|
|
else
|
|
warn "$c — pull failed, will start on existing image"
|
|
fi
|
|
done
|
|
fi
|
|
else
|
|
info "CRITICAL_SYNC_UPDATES=false — skipping local updates"
|
|
fi
|
|
|
|
if [[ "$CRITICAL_SYNC_UPDATES_REMOTE" == true ]]; then
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
|
|
else
|
|
info "Pulling remote container updates on $REMOTE_SERVER_NAME..."
|
|
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
|
[[ -z "$c" ]] && continue
|
|
IMAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
|
|
"docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
|
|
if [[ -z "$IMAGE" ]]; then
|
|
log "$c — not found on remote, skipping update"
|
|
continue
|
|
fi
|
|
info "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
|
|
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker pull $IMAGE" >/dev/null 2>&1; then
|
|
success "$c — remote image updated"
|
|
else
|
|
warn "$c — remote pull failed, will start on existing image"
|
|
fi
|
|
done
|
|
fi
|
|
else
|
|
info "CRITICAL_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
|
fi
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SYNC Critical Shares Sync ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
PASS=()
|
|
FAIL=()
|
|
TOTAL_START=$(date +%s)
|
|
|
|
SYNC_JOBS=(
|
|
"/mnt/user/Media_Server/Emby"
|
|
"/mnt/user/appdata-Failover/Critical-Data"
|
|
)
|
|
|
|
SHARE_COUNT=${#SYNC_JOBS[@]}
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Critical Shares Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_SUMMARY Jobs: $SHARE_COUNT"
|
|
echo ""
|
|
|
|
JOB_NUM=0
|
|
for JOB in "${SYNC_JOBS[@]}"; do
|
|
((JOB_NUM++))
|
|
JOB_NAME=$(basename "$JOB")
|
|
echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"
|
|
|
|
JOB_START=$(date +%s)
|
|
|
|
# Containers already stopped — rsync profile won't try to stop them again
|
|
# Pass --no-container-stop flag would be ideal but profiles handle this naturally
|
|
# since containers are already stopped, stop_containers finds nothing running
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
bash "$RSYNC_SCRIPT" "$JOB" --dry-run
|
|
else
|
|
bash "$RSYNC_SCRIPT" "$JOB"
|
|
fi
|
|
|
|
EXIT_CODE=$?
|
|
JOB_END=$(date +%s)
|
|
JOB_DURATION=$(format_duration $(( JOB_END - JOB_START )))
|
|
|
|
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
|
PASS+=("$JOB_NAME")
|
|
success "$JOB_NAME — $ICON_SUCCESS done in $JOB_DURATION"
|
|
else
|
|
FAIL+=("$JOB_NAME")
|
|
error "$JOB_NAME — $ICON_ERROR failed after $JOB_DURATION"
|
|
fi
|
|
|
|
echo ""
|
|
done
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — containers will not be started"
|
|
else
|
|
start_containers
|
|
start_local_containers
|
|
fi
|
|
|
|
TOTAL_END=$(date +%s)
|
|
TOTAL_DURATION=$(format_duration $(( TOTAL_END - TOTAL_START )))
|
|
|
|
# -----------------------------------------------------------------------------------------------
|
|
# ━━━ $ICON_SUMMARY Summary ━━━
|
|
# -----------------------------------------------------------------------------------------------
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY CRITICAL SHARES MAINTENANCE SUMMARY ━━━━━"
|
|
echo "$ICON_TIME Duration: $TOTAL_DURATION"
|
|
echo "$ICON_GEAR Updates: local=$CRITICAL_SYNC_UPDATES remote=$CRITICAL_SYNC_UPDATES_REMOTE"
|
|
echo "$ICON_SUCCESS Passed: ${#PASS[@]} $ICON_ERROR Failed: ${#FAIL[@]}"
|
|
echo ""
|
|
|
|
if [[ ${#PASS[@]} -gt 0 ]]; then
|
|
for job in "${PASS[@]}"; do echo " $ICON_SUCCESS $job"; done
|
|
fi
|
|
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
|
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
|
fi
|
|
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
|
elif [[ ${#FAIL[@]} -eq 0 ]]; then
|
|
echo "$ICON_DONE Status: $ICON_SUCCESS ALL JOBS COMPLETE"
|
|
notify "Critical maintenance complete on $(hostname) — synced + updated (local=$CRITICAL_SYNC_UPDATES remote=$CRITICAL_SYNC_UPDATES_REMOTE)" "Critical Maintenance" "normal"
|
|
else
|
|
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAIL[@]} JOB(S) FAILED"
|
|
notify "Critical maintenance failed on $(hostname) — failed: ${FAIL[*]}" "Critical Maintenance" "warning"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |