FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
112793055fe965da1cb409d19030eccc4be073dd
Varaverk/Orchestrators/weekly_sync_maintenance.sh
T
FailedProxy 94f90a4579 padded partnership set up
2026-04-29 20:09:28 -04:00

315 lines
13 KiB
Bash
Raw Blame History

#!/bin/bash
# -----------------------------------------------------------------------------------------------
# ----------------------------- Weekly Sync Maintenance ----------------------------------------
# -----------------------------------------------------------------------------------------------
# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts.
# Schedule: 30 2 * * 0 (Sunday 2:30am — fits before 3am network reboot)
#
# Execution order:
# 1. Stop local containers — Emby + auth stack stopped locally
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
# 5. rsync Emby — full clean mirror, both instances stopped
# 6. rsync Critical-Data — auth stack clean sync, databases flushed
# 7. Start remote containers — correct order, delayed start respected
# 8. Start local containers — correct order, delayed start respected
# 9. docker_weekly_restart.sh — weekly container restarts
#
# Synced shares (WEEKLY_SYNC_SHARES in Master.conf):
# /mnt/user/Media_Server/Emby — emby profile — full mirror, cache resets weekly
# /mnt/user/appdata-Failover/Critical-Data — critical-data — auth stack clean state
#
# Why weekly instead of nightly for Emby:
# Emby builds a warm image cache on HOST2 throughout the week
# Syncing nightly resets cache — cold loads every morning for users
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep
# emby-failover dirty sync covers watch states + library every 30-60min between syncs
#
# Container updates during the window:
# Containers already stopped for sync — updates pull at zero extra downtime
# Both servers start on identical image versions after the window completes
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in Master.conf
#
# What triggers weekly_health_digest.sh:
# NOT this script — weekly_health_digest.sh runs on its own Saturday schedule
#
# Configuration in Master.conf:
# WEEKLY_SYNC_SHARES — shares synced during the maintenance window
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync (docker_weekly_restart)
# WEEKLY_SYNC_UPDATES — toggle container updates on/off
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates on/off
# -----------------------------------------------------------------------------------------------
# All configuration in Master.conf.
# Supports --dry-run to walk through without stopping containers, syncing, or updating.
# -----------------------------------------------------------------------------------------------
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../Master.conf"
source "$SCRIPT_DIR/../common.sh"
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
parse_args "$@"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Setup ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Setup ━━━"
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
success "Running as root"
acquire_lock
detect_hosts
resolve_remote_ip
# Load container list from emby profile — used for update pulls
read -r -a MAINTENANCE_CONTAINERS <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
check_connectivity
check_remote_rootfs
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — containers will not be stopped"
else
# Load critical-data + emby container list for stops
read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"
stop_local_containers
stop_containers
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Container Updates ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_GEAR Container Updates ━━━"
if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would pull updates for local containers"
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
[[ -z "$c" ]] && continue
warn "DRY RUN — would docker pull: $c"
done
else
info "Pulling local container updates..."
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
[[ -z "$c" ]] && continue
# Get image name from running or stopped container
IMAGE=$(docker inspect "$c" --format '{{.Config.Image}}' 2>/dev/null)
if [[ -z "$IMAGE" ]]; then
log "$c — not found locally, skipping update"
continue
fi
info "Pulling $IMAGE for $c..."
if docker pull "$IMAGE" >/dev/null 2>&1; then
success "$c — image updated"
else
warn "$c — pull failed, will start on existing image"
fi
done
fi
else
info "WEEKLY_SYNC_UPDATES=false — skipping local updates"
fi
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
else
info "Pulling remote container updates on $REMOTE_SERVER_NAME..."
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
[[ -z "$c" ]] && continue
IMAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \
"docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
if [[ -z "$IMAGE" ]]; then
log "$c — not found on remote, skipping update"
continue
fi
info "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker pull $IMAGE" >/dev/null 2>&1; then
success "$c — remote image updated"
else
warn "$c — remote pull failed, will start on existing image"
fi
done
fi
else
info "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SYNC Critical Shares Sync ━━━
# -----------------------------------------------------------------------------------------------
PASS=()
FAIL=()
TOTAL_START=$(date +%s)
SYNC_JOBS=("${WEEKLY_SYNC_SHARES[@]}")
SHARE_COUNT=${#SYNC_JOBS[@]}
echo ""
echo "━━━ $ICON_SYNC Critical Shares Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_SUMMARY Jobs: $SHARE_COUNT"
echo ""
JOB_NUM=0
# Tier 1 + Tier 2 rsync gate check
if ! check_rsync_enabled "WEEKLY"; then
warn "Rsync disabled — skipping all $SHARE_COUNT weekly sync jobs"
warn "Proceeding to container updates and maintenance scripts..."
else
for JOB in "${SYNC_JOBS[@]}"; do
((JOB_NUM++))
JOB_NAME=$(basename "$JOB")
echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"
JOB_START=$(date +%s)
# Containers already stopped — rsync profile won't try to stop them again
# Pass --no-container-stop flag would be ideal but profiles handle this naturally
# since containers are already stopped, stop_containers finds nothing running
if [[ "$DRY_RUN" == true ]]; then
bash "$RSYNC_SCRIPT" "$JOB" --dry-run
else
bash "$RSYNC_SCRIPT" "$JOB"
fi
EXIT_CODE=$?
JOB_END=$(date +%s)
JOB_DURATION=$(format_duration $(( JOB_END - JOB_START )))
if [[ "$EXIT_CODE" -eq 0 ]]; then
PASS+=("$JOB_NAME")
success "$JOB_NAME — $ICON_SUCCESS done in $JOB_DURATION"
else
FAIL+=("$JOB_NAME")
error "$JOB_NAME — $ICON_ERROR failed after $JOB_DURATION"
fi
echo ""
done
fi # end check_rsync_enabled "WEEKLY"
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — containers will not be started"
else
start_containers
start_local_containers
fi
TOTAL_END=$(date +%s)
TOTAL_DURATION=$(format_duration $(( TOTAL_END - TOTAL_START )))
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_GEAR Post-sync Jobs ━━━
# docker_weekly_restart.sh and any other WEEKLY_MAINTENANCE_SCRIPTS run after sync
# -----------------------------------------------------------------------------------------------
JOB_PASS=()
JOB_FAIL=()
SCRIPTS_ROOT="$SCRIPT_DIR/.."
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
echo ""
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
[[ -z "$script_entry" ]] && continue
script_args=($script_entry)
script_path="$SCRIPTS_ROOT/${script_args[0]}"
script_name=$(basename "${script_args[0]}")
extra_args=("${script_args[@]:1}")
echo ""
info "$ICON_START Running: $script_name"
if [[ ! -f "$script_path" ]]; then
error "$script_name — not found at $script_path"
JOB_FAIL+=("$script_name")
continue
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would run: $script_name"
JOB_PASS+=("$script_name (dry run)")
elif bash "$script_path" "${extra_args[@]}"; then
success "$script_name — done"
JOB_PASS+=("$script_name")
else
error "$script_name — failed"
JOB_FAIL+=("$script_name")
fi
done
fi
WINDOW_END=$(date +%s)
# -----------------------------------------------------------------------------------------------
# ━━━ $ICON_SUMMARY Summary ━━━
# -----------------------------------------------------------------------------------------------
echo ""
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
echo "$ICON_TIME Duration: $TOTAL_DURATION"
echo "$ICON_GEAR Updates: local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE"
echo ""
echo "$ICON_SYNC Sync jobs:"
if [[ ${#PASS[@]} -gt 0 ]]; then
for job in "${PASS[@]}"; do echo " $ICON_SUCCESS $job"; done
fi
if [[ ${#FAIL[@]} -gt 0 ]]; then
for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done
fi
echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
echo ""
echo "$ICON_GEAR Post-sync jobs:"
for job in "${JOB_PASS[@]}"; do echo " $ICON_SUCCESS $job"; done
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
fi
echo ""
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
if [[ "$DRY_RUN" == true ]]; then
echo "$ICON_WARN Status: DRY RUN — no changes made"
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE"
notify "Weekly sync maintenance complete on $(hostname) — synced + updated (local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE)" "Weekly Maintenance" "normal"
else
echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs"
notify "Weekly sync maintenance failed on $(hostname) — sync: ${#FAIL[@]} failed, jobs: ${#JOB_FAIL[@]} failed" "Weekly Maintenance" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
Reference in New Issue View Git Blame Copy Permalink