#!/bin/bash # ============================================================================================== # ============================= Weekly Sync Maintenance ======================================== # ============================================================================================== # Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts. # Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report) # # ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── # 1. Stop local containers — Emby + auth stack stopped locally # 2. Stop remote containers — Emby + auth stack stopped remotely via SSH # 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true (zero extra downtime) # 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true # 5. rsync WEEKLY_SYNC_SHARES — full clean mirror, containers stopped both sides # 6. Start remote containers — correct order, delayed start respected # 7. Start local containers — correct order, delayed start respected # 8. WEEKLY_MAINTENANCE_SCRIPTS — weekly restarts etc. (docker_weekly_restart.sh) # 9. docker_update.sh --remainder — update all containers not in daily or weekly sync window # # ── WHY WEEKLY NOT NIGHTLY FOR EMBY ────────────────────────────────────────────────────────── # Emby builds a warm image cache on HOST2 throughout the week. # Syncing nightly resets cache — cold loads every morning for users. # Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep. # emby-fallback dirty sync covers watch states + library every 15min between weekly syncs. # # ── CONTAINER UPDATES ───────────────────────────────────────────────────────────────────────── # Containers already stopped for sync — updates pull at zero extra downtime. # Both servers start on identical image versions after the window completes. # Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf # # ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── # detect_hosts() sets MY_ID — used in banner, summary, and notifications. # WEEKLY_SYNC_SHARES and WEEKLY_MAINTENANCE_SCRIPTS configured in master.conf. # Same script runs correctly on both servers. # # ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # Root check — stop/start containers, rsync require root # acquire_lock — prevents concurrent weekly windows # check_connectivity — verifies remote before any remote operations # check_remote_rootfs — aborts if remote rootfs nearly full # DOCKER_TIMEOUT — all docker calls protected # SSH_TIMEOUT — all SSH calls protected # validate_unraid_cmd — notify validated before use # Silent on success — runs weekly, only failures warrant notification # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # WEEKLY_SYNC_SHARES — shares synced during window # WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync # WEEKLY_SYNC_UPDATES — toggle local container updates # WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates # WEEKLY_RSYNC_ENABLED — enable/disable rsync section # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # weekly_sync_maintenance.sh — normal run # weekly_sync_maintenance.sh --dry-run — preview without stopping containers or syncing # weekly_sync_maintenance.sh --log — verbose per-share/per-job output # weekly_sync_maintenance.sh --status — show configuration and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" SCRIPTS_ROOT="$SCRIPT_DIR/.." parse_args "$@" DOCKER_TIMEOUT=30 # container stop/start needs longer than normal SSH_TIMEOUT=30 # remote pulls can be slow # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi validate_unraid_cmd \ "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ "" "" \ "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" acquire_lock if ! command -v docker &>/dev/null; then error "Docker command not found" exit 1 fi detect_hosts resolve_remote_ip WINDOW_START=$(date +%s) PASS=() FAIL=() JOB_PASS=() JOB_FAIL=() # Load container lists from profile config read -r -a MAINTENANCE_CONTAINERS <<< \ "${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates" # ── Helper — run a post-sync maintenance script ──────────────────────────────────────────────── run_job() { local script_entry="$1" local extra_dry="" [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run" read -r -a script_args <<< "$script_entry" local script_path="$SCRIPTS_ROOT/${script_args[0]}" local script_name script_name=$(basename "${script_args[0]}") local extra_args=("${script_args[@]:1}") if [[ ! -f "$script_path" ]]; then error "$script_name — not found at $script_path" JOB_FAIL+=("$script_name") return 1 fi log "Running: $script_name ${extra_args[*]}" # shellcheck disable=SC2086 if bash "$script_path" "${extra_args[@]}" $extra_dry; then log "$script_name — done ✅" JOB_PASS+=("$script_name ${extra_args[*]}") else error "$script_name — failed (exit $?)" JOB_FAIL+=("$script_name ${extra_args[*]}") fi } # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)" echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}" echo "$ICON_SYNC Weekly enabled: ${WEEKLY_RSYNC_ENABLED:-false}" echo "$ICON_GEAR Local updates: ${WEEKLY_SYNC_UPDATES:-false}" echo "$ICON_GEAR Remote updates: ${WEEKLY_SYNC_UPDATES_REMOTE:-false}" echo "" echo "━━━ Weekly Sync Shares ━━━" if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then warn " No WEEKLY_SYNC_SHARES configured" else for share in "${WEEKLY_SYNC_SHARES[@]}"; do [[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)" done fi echo "" echo "━━━ Weekly Maintenance Scripts ━━━" if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then echo " None configured" else for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do [[ -n "$entry" ]] && echo " $ICON_GEAR ${entry##*/}" done fi echo "" echo "━━━ Containers (from profile config) ━━━" for c in "${MAINTENANCE_CONTAINERS[@]}"; do [[ -n "$c" ]] && echo " $ICON_CONTAINERS $c" done echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ━━━ Pre-flight Checks ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "" echo "━━━ $ICON_SHIELD Pre-flight ━━━" check_connectivity check_remote_rootfs # ============================================================================================== # ━━━ Stop Containers ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — containers will not be stopped" else # Load container lists for stop functions read -r -a CRITICAL_CONTAINER_NAMES <<< \ "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}" read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}" CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}" stop_local_containers stop_containers fi # ============================================================================================== # ━━━ Container Updates ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_GEAR Container Updates ━━━" if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then if [[ "$DRY_RUN" == true ]]; then for c in "${MAINTENANCE_CONTAINERS[@]}"; do [[ -n "$c" ]] && warn "DRY RUN — would pull: $c" done else log "Pulling local container updates..." for c in "${MAINTENANCE_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \ "$c" --format '{{.Config.Image}}' 2>/dev/null) if [[ -z "$IMAGE" ]]; then log "$c — not found locally, skipping update" continue fi log "Pulling $IMAGE for $c..." if docker pull "$IMAGE" >/dev/null 2>&1; then log "$c — image updated ✅" else warn "$c — pull failed, will start on existing image" fi done fi else echo "WEEKLY_SYNC_UPDATES=false — skipping local updates" fi if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME" else log "Pulling remote container updates on $REMOTE_SERVER_NAME..." for c in "${MAINTENANCE_CONTAINERS[@]}"; do [[ -z "$c" ]] && continue IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" \ root@"$REMOTE_SERVER" \ "docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null) if [[ -z "$IMAGE" ]]; then log "$c — not found on remote, skipping update" continue fi log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..." if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \ -o ConnectTimeout="$SSH_TIMEOUT" \ root@"$REMOTE_SERVER" \ "docker pull $IMAGE" >/dev/null 2>&1; then log "$c — remote image updated ✅" else warn "$c — remote pull failed, will start on existing image" fi done fi else echo "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates" fi # ============================================================================================== # ━━━ Critical Shares Sync ━━━ # ============================================================================================== SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]} echo "" echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━" SYNC_START=$(date +%s) JOB_NUM=0 if ! check_rsync_enabled "WEEKLY"; then warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)" warn "Proceeding to container start and maintenance scripts..." elif [[ "$SHARE_COUNT" -eq 0 ]]; then warn "No WEEKLY_SYNC_SHARES configured — skipping sync" warn "Check WEEKLY_SYNC_SHARES in master.conf" else RSYNC_DRY="" [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run" for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do (( JOB_NUM++ )) JOB_NAME=$(basename "$JOB") echo "" echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━" JOB_START=$(date +%s) bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY EXIT_CODE=$? JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START ))) if [[ "$EXIT_CODE" -eq 0 ]]; then PASS+=("$JOB_NAME") log "$JOB_NAME — done in $JOB_DUR ✅" else FAIL+=("$JOB_NAME") error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)" fi echo "" done fi SYNC_END=$(date +%s) # ============================================================================================== # ━━━ Start Containers ━━━ # ============================================================================================== echo "" echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — containers will not be started" else start_containers start_local_containers fi # ============================================================================================== # ━━━ Post-sync Jobs ━━━ # ============================================================================================== if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_GEAR Post-sync Jobs ━━━" for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do [[ -z "$script_entry" ]] && continue echo "" run_job "$script_entry" done fi # ============================================================================================== # ━━━ Remainder Container Updates ━━━ # ============================================================================================== # Updates all running containers not already covered by daily or the weekly sync window. # Runs last — weekly sync-window containers are back up before this pulls their peers. echo "" echo "━━━ $ICON_CONTAINERS Remainder Container Updates ━━━" DOCKER_UPDATE_SCRIPT="$SCRIPTS_ROOT/Docker_Essentials/docker_update.sh" if [[ ! -f "$DOCKER_UPDATE_SCRIPT" ]]; then warn "docker_update.sh not found — skipping remainder updates" JOB_FAIL+=("docker_update.sh --remainder") else _remainder_args=("--remainder") [[ "$DRY_RUN" == true ]] && _remainder_args+=("--dry-run") if bash "$DOCKER_UPDATE_SCRIPT" "${_remainder_args[@]}"; then echo "Remainder updates complete ✅" JOB_PASS+=("docker_update.sh --remainder") else warn "Remainder updates completed with errors" JOB_FAIL+=("docker_update.sh --remainder") fi unset _remainder_args fi WINDOW_END=$(date +%s) # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')" echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))" echo "$ICON_GEAR Updates: local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}" echo "" echo "$ICON_SYNC Sync jobs ($SHARE_COUNT):" for job in "${PASS[@]}"; do echo " $ICON_DONE $job"; done for job in "${FAIL[@]}"; do echo " $ICON_ERROR $job"; done echo " Passed: ${#PASS[@]} Failed: ${#FAIL[@]}" if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then echo "" echo "$ICON_GEAR Post-sync jobs:" for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done fi TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} )) echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ "$TOTAL_FAIL" -eq 0 ]]; then echo "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run" else warn "Status: $TOTAL_FAIL failure(s)" notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \ "Weekly Maintenance" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" [[ "$TOTAL_FAIL" -gt 0 ]] && exit 1 exit 0