#!/bin/bash # ============================================================================================== # ============================= Weekly Sync Maintenance ======================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Weekly maintenance window orchestrator — clean Emby sync, container updates, # and weekly restarts. Schedule: 30 2 * * 0 (Sunday 2:30am) # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # 1. Stop local containers — Emby + auth stack stopped locally # 2. Stop remote containers — Emby + auth stack stopped remotely via SSH # 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (zero extra downtime) # 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true # 5. rsync WEEKLY_SYNC_SHARES — full clean mirror, containers stopped both sides # 6. Start remote containers — correct order, delayed start respected # 7. Start local containers — rebuild if new image pulled, docker start otherwise # 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh) # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Weekly Cadence Preserves Cache # Emby builds a warm image cache on HOST2 throughout the week. Syncing nightly # resets that cache — cold loads every morning for users. Weekly sync keeps # the cache warm for 6 days, resets Sunday night while users sleep. # play_state_sync covers watch/resume state every 30 min between weekly syncs. # # Zero-Downtime Updates # Containers are already stopped for the sync window — image pulls happen at # zero extra downtime. Both servers start on identical image versions after # the window completes. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Root check — stop/start containers and rsync require root # acquire_lock — prevents concurrent weekly windows # detect_hosts() — MY_ID in banner, summary, and notifications # check_connectivity — verifies remote before any remote operations # check_remote_rootfs — aborts rsync if remote rootfs nearly full # DOCKER_TIMEOUT — all docker calls protected # SSH_TIMEOUT — all SSH calls protected # platform_require_cmd — notify validated before use # Silent on success — runs weekly; only failures warrant notification # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # master.conf # # WEEKLY_SYNC_SHARES — shares synced during window # WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync # WEEKLY_SYNC_UPDATES — toggle local container updates # WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates # WEEKLY_RSYNC_ENABLED — enable/disable rsync section # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # weekly_sync_maintenance.sh # Normal run. # # weekly_sync_maintenance.sh --dry-run # Preview without stopping containers or syncing. # # weekly_sync_maintenance.sh --log # Verbose per-share/per-job output. # # weekly_sync_maintenance.sh --status # Show configuration and exit. # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh" parse_args "$@" DOCKER_TIMEOUT=30 # container stop/start needs longer than normal SSH_TIMEOUT=30 # remote pulls can be slow # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi acquire_lock if ! command -v docker &>/dev/null; then error "Docker command not found" exit 1 fi detect_hosts resolve_remote_ip WINDOW_START=$(date +%s) PASS=() FAIL=() JOB_PASS=() JOB_FAIL=() # Load container lists from profile config read -r -a MAINTENANCE_CONTAINERS <<< \ "${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates" # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)" echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}" echo "$ICON_SYNC Weekly enabled: ${WEEKLY_RSYNC_ENABLED:-false}" echo "$ICON_GEAR Local updates: ${WEEKLY_SYNC_UPDATES:-false}" echo "$ICON_GEAR Remote updates: ${WEEKLY_SYNC_UPDATES_REMOTE:-false}" echo "" echo "━━━ Weekly Sync Shares ━━━" if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then warn " No WEEKLY_SYNC_SHARES configured" else for share in "${WEEKLY_SYNC_SHARES[@]}"; do [[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)" done fi echo "" echo "━━━ Weekly Maintenance Scripts ━━━" if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then echo " None configured" else for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do [[ -n "$entry" ]] && echo " $ICON_GEAR ${entry##*/}" done fi echo "" echo "━━━ Containers (from profile config) ━━━" for c in "${MAINTENANCE_CONTAINERS[@]}"; do [[ -n "$c" ]] && echo " $ICON_CONTAINERS $c" done echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ━━━ Pre-flight Checks ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "" echo "━━━ $ICON_SHIELD Pre-flight ━━━" check_connectivity check_remote_rootfs # ============================================================================================== # ━━━ Stop Containers ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — containers will not be stopped" else # Load container lists for stop functions read -r -a CRITICAL_CONTAINER_NAMES <<< \ "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}" read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}" CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}" stop_local_containers stop_containers fi # ============================================================================================== # ━━━ Container Updates ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_GEAR Container Updates ━━━" declare -A _weekly_needs_rebuild=() if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then if [[ "$DRY_RUN" == true ]]; then for c in "${MAINTENANCE_CONTAINERS[@]}"; do [[ -n "$c" ]] && warn "DRY RUN — would pull: $c" done else log "Pulling local container updates..." for c in "${MAINTENANCE_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \ "$c" --format '{{.Config.Image}}' 2>/dev/null) if [[ -z "$IMAGE" ]]; then log "$c — not found locally, skipping update" continue fi _old_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "") log "Pulling $IMAGE for $c..." if docker pull "$IMAGE" >/dev/null 2>&1; then _new_id=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "") if [[ -n "$_old_id" && "$_old_id" != "$_new_id" ]]; then log "$c — new image (${_old_id:7:12} → ${_new_id:7:12}) — will rebuild after sync" _weekly_needs_rebuild["$c"]=1 else log "$c — already current" fi else warn "$c — pull failed, will start on existing image" fi done unset _old_id _new_id fi else echo "WEEKLY_SYNC_UPDATES=false — skipping local updates" fi if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME" else log "Pulling remote container updates on $REMOTE_SERVER_NAME..." for c in "${MAINTENANCE_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" \ root@"$REMOTE_SERVER" \ "docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null) if [[ -z "$IMAGE" ]]; then log "$c — not found on remote, skipping update" continue fi log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..." if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" \ root@"$REMOTE_SERVER" \ "docker pull $IMAGE" >/dev/null 2>&1; then echo "$c — remote image updated ✅" else warn "$c — remote pull failed, will start on existing image" fi done fi else echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates" fi # ============================================================================================== # ━━━ Critical Shares Sync ━━━ # ============================================================================================== SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]} echo "" echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━" SYNC_START=$(date +%s) JOB_NUM=0 if ! check_rsync_enabled "WEEKLY"; then warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)" warn "Proceeding to container start and maintenance scripts..." elif [[ "$SHARE_COUNT" -eq 0 ]]; then warn "No WEEKLY_SYNC_SHARES configured — skipping sync" warn "Check WEEKLY_SYNC_SHARES in master.conf" else RSYNC_DRY="" [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run" for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do (( JOB_NUM++ )) JOB_NAME=$(basename "$JOB") echo "" echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━" JOB_START=$(date +%s) bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY EXIT_CODE=$? JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START ))) if [[ "$EXIT_CODE" -eq 0 ]]; then PASS+=("$JOB_NAME") echo "$JOB_NAME — done in $JOB_DUR ✅" else FAIL+=("$JOB_NAME") error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)" fi echo "" done fi SYNC_END=$(date +%s) # ============================================================================================== # ━━━ Start Containers ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — containers will not be started" else start_containers # Local start — rebuild containers that received a new image, docker start the rest if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then log "No local containers to restart." else for _c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do [[ -z "$_c" ]] && continue _needs_delay=false for _d in "${DELAYED_CONTAINERS[@]}"; do [[ "$_c" == "$_d" ]] && _needs_delay=true && break done [[ "$_needs_delay" == true ]] && { info "Waiting ${CONTAINER_DELAY}s before starting $_c..." sleep "$CONTAINER_DELAY" } if [[ -n "${_weekly_needs_rebuild[$_c]:-}" ]]; then log "Rebuilding $_c on new image..." if platform_rebuild_container "$_c"; then echo "$_c rebuilt on new image ✅" else warn "$_c rebuild failed — falling back to docker start" docker start "$_c" >/dev/null 2>&1 || error "Failed to start $_c" fi else docker start "$_c" >/dev/null 2>&1 && echo "$_c started" || error "Failed to start $_c" fi done unset _c _d _needs_delay fi fi # ============================================================================================== # ━━━ Post-sync Jobs ━━━ # ============================================================================================== if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_GEAR Post-sync Jobs ━━━" for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do [[ -z "$script_entry" ]] && continue echo "" run_orch_child "$script_entry" done fi WINDOW_END=$(date +%s) # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')" echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))" echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}" echo "" echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):" for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}" if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then echo "" echo "$ICON_GEAR Post-sync jobs:" for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done fi TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} )) echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ "$TOTAL_FAIL" -eq 0 ]]; then echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run" else warn "Status: $TOTAL_FAIL failure(s)" notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \ "Weekly Maintenance" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" [[ "$TOTAL_FAIL" -gt 0 ]] && exit 1 exit 0