#!/bin/bash # ----------------------------------------------------------------------------------------------- # ----------------------------- Weekly Sync Maintenance ---------------------------------------- # ----------------------------------------------------------------------------------------------- # Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts. # Schedule: 30 2 * * 0 (Sunday 2:30am — fits before 3am network reboot) # # Execution order: # 1. Stop local containers — Emby + auth stack stopped locally # 2. Stop remote containers — Emby + auth stack stopped remotely via SSH # 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true # 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true # 5. rsync Emby — full clean mirror, both instances stopped # 6. rsync Critical-Data — auth stack clean sync, databases flushed # 7. Start remote containers — correct order, delayed start respected # 8. Start local containers — correct order, delayed start respected # 9. docker_weekly_restart.sh — weekly container restarts # # Synced shares (WEEKLY_SYNC_SHARES in Master.conf): # /mnt/user/Media_Server/Emby — emby profile — full mirror, cache resets weekly # /mnt/user/appdata-Failover/Critical-Data — critical-data — auth stack clean state # # Why weekly instead of nightly for Emby: # Emby builds a warm image cache on HOST2 throughout the week # Syncing nightly resets cache — cold loads every morning for users # Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep # emby-failover dirty sync covers watch states + library every 30-60min between syncs # # Container updates during the window: # Containers already stopped for sync — updates pull at zero extra downtime # Both servers start on identical image versions after the window completes # Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in Master.conf # # What triggers weekly_health_digest.sh: # NOT this script — weekly_health_digest.sh runs on its own Saturday schedule # # Configuration in Master.conf: # WEEKLY_SYNC_SHARES — shares synced during the maintenance window # WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync (docker_weekly_restart) # WEEKLY_SYNC_UPDATES — toggle container updates on/off # WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates on/off # ----------------------------------------------------------------------------------------------- # All configuration in Master.conf. # Supports --dry-run to walk through without stopping containers, syncing, or updating. # ----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" parse_args "$@" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_GEAR Setup ━━━" if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi success "Running as root" acquire_lock detect_hosts resolve_remote_ip # Load container list from emby profile — used for update pulls read -r -a MAINTENANCE_CONTAINERS <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SHIELD Pre-flight Checks ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━" check_connectivity check_remote_rootfs # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — containers will not be stopped" else # Load critical-data + emby container list for stops read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}" read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}" CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}" stop_local_containers stop_containers fi # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Container Updates ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_GEAR Container Updates ━━━" if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would pull updates for local containers" for c in "${MAINTENANCE_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue warn "DRY RUN — would docker pull: $c" done else info "Pulling local container updates..." for c in "${MAINTENANCE_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue # Get image name from running or stopped container IMAGE=$(docker inspect "$c" --format '{{.Config.Image}}' 2>/dev/null) if [[ -z "$IMAGE" ]]; then log "$c — not found locally, skipping update" continue fi info "Pulling $IMAGE for $c..." if docker pull "$IMAGE" >/dev/null 2>&1; then success "$c — image updated" else warn "$c — pull failed, will start on existing image" fi done fi else info "WEEKLY_SYNC_UPDATES=false — skipping local updates" fi if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME" else info "Pulling remote container updates on $REMOTE_SERVER_NAME..." for c in "${MAINTENANCE_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue IMAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ "docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null) if [[ -z "$IMAGE" ]]; then log "$c — not found on remote, skipping update" continue fi info "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..." if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker pull $IMAGE" >/dev/null 2>&1; then success "$c — remote image updated" else warn "$c — remote pull failed, will start on existing image" fi done fi else info "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates" fi # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SYNC Critical Shares Sync ━━━ # ----------------------------------------------------------------------------------------------- PASS=() FAIL=() TOTAL_START=$(date +%s) SYNC_JOBS=("${WEEKLY_SYNC_SHARES[@]}") SHARE_COUNT=${#SYNC_JOBS[@]} echo "" echo "━━━ $ICON_SYNC Critical Shares Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_SUMMARY Jobs: $SHARE_COUNT" echo "" JOB_NUM=0 # Tier 1 + Tier 2 rsync gate check if ! check_rsync_enabled "WEEKLY"; then warn "Rsync disabled — skipping all $SHARE_COUNT weekly sync jobs" warn "Proceeding to container updates and maintenance scripts..." else for JOB in "${SYNC_JOBS[@]}"; do ((JOB_NUM++)) JOB_NAME=$(basename "$JOB") echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━" JOB_START=$(date +%s) # Containers already stopped — rsync profile won't try to stop them again # Pass --no-container-stop flag would be ideal but profiles handle this naturally # since containers are already stopped, stop_containers finds nothing running if [[ "$DRY_RUN" == true ]]; then bash "$RSYNC_SCRIPT" "$JOB" --dry-run else bash "$RSYNC_SCRIPT" "$JOB" fi EXIT_CODE=$? JOB_END=$(date +%s) JOB_DURATION=$(format_duration $(( JOB_END - JOB_START ))) if [[ "$EXIT_CODE" -eq 0 ]]; then PASS+=("$JOB_NAME") success "$JOB_NAME — $ICON_SUCCESS done in $JOB_DURATION" else FAIL+=("$JOB_NAME") error "$JOB_NAME — $ICON_ERROR failed after $JOB_DURATION" fi echo "" done fi # end check_rsync_enabled "WEEKLY" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — containers will not be started" else start_containers start_local_containers fi TOTAL_END=$(date +%s) TOTAL_DURATION=$(format_duration $(( TOTAL_END - TOTAL_START ))) # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Post-sync Jobs ━━━ # docker_weekly_restart.sh and any other WEEKLY_MAINTENANCE_SCRIPTS run after sync # ----------------------------------------------------------------------------------------------- JOB_PASS=() JOB_FAIL=() SCRIPTS_ROOT="$SCRIPT_DIR/.." if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_GEAR Post-sync Jobs ━━━" for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do [[ -z "$script_entry" ]] && continue script_args=($script_entry) script_path="$SCRIPTS_ROOT/${script_args[0]}" script_name=$(basename "${script_args[0]}") extra_args=("${script_args[@]:1}") echo "" info "$ICON_START Running: $script_name" if [[ ! -f "$script_path" ]]; then error "$script_name — not found at $script_path" JOB_FAIL+=("$script_name") continue fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would run: $script_name" JOB_PASS+=("$script_name (dry run)") elif bash "$script_path" "${extra_args[@]}"; then success "$script_name — done" JOB_PASS+=("$script_name") else error "$script_name — failed" JOB_FAIL+=("$script_name") fi done fi WINDOW_END=$(date +%s) # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Summary ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━" echo "$ICON_TIME Duration: $TOTAL_DURATION" echo "$ICON_GEAR Updates: local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE" echo "" echo "$ICON_SYNC Sync jobs:" if [[ ${#PASS[@]} -gt 0 ]]; then for job in "${PASS[@]}"; do echo " $ICON_SUCCESS $job"; done fi if [[ ${#FAIL[@]} -gt 0 ]]; then for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done fi echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}" if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then echo "" echo "$ICON_GEAR Post-sync jobs:" for job in "${JOB_PASS[@]}"; do echo " $ICON_SUCCESS $job"; done for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done fi echo "" TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} )) if [[ "$DRY_RUN" == true ]]; then echo "$ICON_WARN Status: DRY RUN — no changes made" elif [[ "$TOTAL_FAIL" -eq 0 ]]; then echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE" notify "Weekly sync maintenance complete on $(hostname) — synced + updated (local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE)" "Weekly Maintenance" "normal" else echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs" notify "Weekly sync maintenance failed on $(hostname) — sync: ${#FAIL[@]} failed, jobs: ${#JOB_FAIL[@]} failed" "Weekly Maintenance" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"