Varaverk/Orchestrators/weekly_sync_maintenance.sh

#!/bin/bash
# ==============================================================================================
# ============================= Weekly Sync Maintenance ========================================
# ==============================================================================================
# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts.
# Schedule: 30 2 * * 0 (Sunday 2:30am — before Sunday 7am coffee report)
#
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
#   1. Stop local containers         — Emby + auth stack stopped locally
#   2. Stop remote containers        — Emby + auth stack stopped remotely via SSH
#   3. Pull updates locally          — if WEEKLY_SYNC_UPDATES=true (zero extra downtime)
#   4. Pull updates remotely         — if WEEKLY_SYNC_UPDATES_REMOTE=true
#   5. rsync WEEKLY_SYNC_SHARES      — full clean mirror, containers stopped both sides
#   6. Start remote containers       — correct order, delayed start respected
#   7. Start local containers        — correct order, delayed start respected
#   8. WEEKLY_MAINTENANCE_SCRIPTS    — weekly restarts etc. (docker_weekly_restart.sh)
#
# ── WHY WEEKLY NOT NIGHTLY FOR EMBY ──────────────────────────────────────────────────────────
# Emby builds a warm image cache on HOST2 throughout the week.
# Syncing nightly resets cache — cold loads every morning for users.
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep.
# emby-failover dirty sync covers watch states + library every 15min between weekly syncs.
#
# ── CONTAINER UPDATES ─────────────────────────────────────────────────────────────────────────
# Containers already stopped for sync — updates pull at zero extra downtime.
# Both servers start on identical image versions after the window completes.
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in master.conf
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# detect_hosts() sets MY_ID — used in banner, summary, and notifications.
# WEEKLY_SYNC_SHARES and WEEKLY_MAINTENANCE_SCRIPTS configured in master.conf.
# Same script runs correctly on both servers.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
#   Root check           — stop/start containers, rsync require root
#   acquire_lock         — prevents concurrent weekly windows
#   check_connectivity   — verifies remote before any remote operations
#   check_remote_rootfs  — aborts if remote rootfs nearly full
#   DOCKER_TIMEOUT       — all docker calls protected
#   SSH_TIMEOUT          — all SSH calls protected
#   validate_unraid_cmd  — notify validated before use
#   Silent on success    — runs weekly, only failures warrant notification
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   WEEKLY_SYNC_SHARES             — shares synced during window
#   WEEKLY_MAINTENANCE_SCRIPTS     — scripts run after sync
#   WEEKLY_SYNC_UPDATES            — toggle local container updates
#   WEEKLY_SYNC_UPDATES_REMOTE     — toggle remote container updates
#   WEEKLY_RSYNC_ENABLED           — enable/disable rsync section
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   weekly_sync_maintenance.sh              — normal run
#   weekly_sync_maintenance.sh --dry-run    — preview without stopping containers or syncing
#   weekly_sync_maintenance.sh --log        — verbose per-share/per-job output
#   weekly_sync_maintenance.sh --status     — show configuration and exit
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
SCRIPTS_ROOT="$SCRIPT_DIR/.."

parse_args "$@"

DOCKER_TIMEOUT=30   # container stop/start needs longer than normal
SSH_TIMEOUT=30      # remote pulls can be slow

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

validate_unraid_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock

detect_hosts
resolve_remote_ip

WINDOW_START=$(date +%s)
PASS=()
FAIL=()
JOB_PASS=()
JOB_FAIL=()

# Load container lists from profile config
read -r -a MAINTENANCE_CONTAINERS <<< \
    "${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"

# ── Helper — run a post-sync maintenance script ────────────────────────────────────────────────
run_job() {
    local script_entry="$1"
    local extra_dry=""
    [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"

    read -r -a script_args <<< "$script_entry"
    local script_path="$SCRIPTS_ROOT/${script_args[0]}"
    local script_name
    script_name=$(basename "${script_args[0]}")
    local extra_args=("${script_args[@]:1}")

    if [[ ! -f "$script_path" ]]; then
        error "$script_name — not found at $script_path"
        JOB_FAIL+=("$script_name")
        return 1
    fi

    log "Running: $script_name ${extra_args[*]}"
    # shellcheck disable=SC2086
    if bash "$script_path" "${extra_args[@]}" $extra_dry; then
        log "$script_name — done ✅"
        JOB_PASS+=("$script_name ${extra_args[*]}")
    else
        error "$script_name — failed (exit $?)"
        JOB_FAIL+=("$script_name ${extra_args[*]}")
    fi
}

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC STATUS ━━━━━"
    echo "$ICON_HOST   Identity:        $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_NET    Remote:          $REMOTE_ID ($REMOTE_SERVER_NAME)"
    echo "$ICON_SYNC   Rsync enabled:   ${RSYNC_ENABLED:-true}"
    echo "$ICON_SYNC   Weekly enabled:  ${WEEKLY_RSYNC_ENABLED:-false}"
    echo "$ICON_GEAR   Local updates:   ${WEEKLY_SYNC_UPDATES:-false}"
    echo "$ICON_GEAR   Remote updates:  ${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
    echo ""
    echo "━━━ Weekly Sync Shares ━━━"
    if [[ ${#WEEKLY_SYNC_SHARES[@]} -eq 0 ]]; then
        warn "  No WEEKLY_SYNC_SHARES configured"
    else
        for share in "${WEEKLY_SYNC_SHARES[@]}"; do
            [[ -n "$share" ]] && echo "  $ICON_SYNC $(basename "$share") ($share)"
        done
    fi
    echo ""
    echo "━━━ Weekly Maintenance Scripts ━━━"
    if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
        echo "  None configured"
    else
        for entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
            [[ -n "$entry" ]] && echo "  $ICON_GEAR ${entry##*/}"
        done
    fi
    echo ""
    echo "━━━ Containers (from profile config) ━━━"
    for c in "${MAINTENANCE_CONTAINERS[@]}"; do
        [[ -n "$c" ]] && echo "  $ICON_CONTAINERS $c"
    done
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Pre-flight Checks ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Weekly Sync Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo ""
echo "━━━ $ICON_SHIELD Pre-flight ━━━"

check_connectivity
check_remote_rootfs

# ==============================================================================================
# ━━━ Stop Containers ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — containers will not be stopped"
else
    # Load container lists for stop functions
    read -r -a CRITICAL_CONTAINER_NAMES <<< \
        "${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-}"
    read -r -a DELAYED_CONTAINERS       <<< "${PROFILE_DELAYED_CONTAINERS[critical-data]:-}"
    CONTAINER_DELAY="${PROFILE_CONTAINER_DELAY[critical-data]:-15}"

    stop_local_containers
    stop_containers
fi

# ==============================================================================================
# ━━━ Container Updates ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Container Updates ━━━"

if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
    if [[ "$DRY_RUN" == true ]]; then
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -n "$c" ]] && warn "DRY RUN — would pull: $c"
        done
    else
        log "Pulling local container updates..."
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -z "$c" ]] && continue
            IMAGE=$(timeout "$DOCKER_TIMEOUT" docker inspect \
                "$c" --format '{{.Config.Image}}' 2>/dev/null)
            if [[ -z "$IMAGE" ]]; then
                log "$c — not found locally, skipping update"
                continue
            fi
            log "Pulling $IMAGE for $c..."
            if docker pull "$IMAGE" >/dev/null 2>&1; then
                log "$c — image updated ✅"
            else
                warn "$c — pull failed, will start on existing image"
            fi
        done
    fi
else
    log "WEEKLY_SYNC_UPDATES=false — skipping local updates"
fi

if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
    else
        log "Pulling remote container updates on $REMOTE_SERVER_NAME..."
        for c in "${MAINTENANCE_CONTAINERS[@]}"; do
            [[ -z "$c" ]] && continue
            IMAGE=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
                -o ConnectTimeout="$SSH_TIMEOUT" \
                root@"$REMOTE_SERVER" \
                "docker inspect $c --format '{{.Config.Image}}' 2>/dev/null" 2>/dev/null)
            if [[ -z "$IMAGE" ]]; then
                log "$c — not found on remote, skipping update"
                continue
            fi
            log "Pulling $IMAGE for $c on $REMOTE_SERVER_NAME..."
            if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
                -o ConnectTimeout="$SSH_TIMEOUT" \
                root@"$REMOTE_SERVER" \
                "docker pull $IMAGE" >/dev/null 2>&1; then
                log "$c — remote image updated ✅"
            else
                warn "$c — remote pull failed, will start on existing image"
            fi
        done
    fi
else
    log "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
fi

# ==============================================================================================
# ━━━ Critical Shares Sync ━━━
# ==============================================================================================
SHARE_COUNT=${#WEEKLY_SYNC_SHARES[@]}

echo ""
echo "━━━ $ICON_SYNC Critical Shares Sync — $SHARE_COUNT share(s) ━━━"

SYNC_START=$(date +%s)
JOB_NUM=0

if ! check_rsync_enabled "WEEKLY"; then
    warn "Weekly rsync disabled — skipping all $SHARE_COUNT sync job(s)"
    warn "Proceeding to container start and maintenance scripts..."
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
    warn "No WEEKLY_SYNC_SHARES configured — skipping sync"
    warn "Check WEEKLY_SYNC_SHARES in master.conf"
else
    RSYNC_DRY=""
    [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"

    for JOB in "${WEEKLY_SYNC_SHARES[@]}"; do
        (( JOB_NUM++ ))
        JOB_NAME=$(basename "$JOB")

        echo ""
        echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━"

        JOB_START=$(date +%s)
        bash "$RSYNC_SCRIPT" "$JOB" $RSYNC_DRY
        EXIT_CODE=$?
        JOB_DUR=$(format_duration $(( $(date +%s) - JOB_START )))

        if [[ "$EXIT_CODE" -eq 0 ]]; then
            PASS+=("$JOB_NAME")
            log "$JOB_NAME — done in $JOB_DUR ✅"
        else
            FAIL+=("$JOB_NAME")
            error "$JOB_NAME — failed after $JOB_DUR (exit $EXIT_CODE)"
        fi
        echo ""
    done
fi

SYNC_END=$(date +%s)

# ==============================================================================================
# ━━━ Start Containers ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — containers will not be started"
else
    start_containers
    start_local_containers
fi

# ==============================================================================================
# ━━━ Post-sync Jobs ━━━
# ==============================================================================================
if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
    for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
        [[ -z "$script_entry" ]] && continue
        echo ""
        run_job "$script_entry"
    done
fi

WINDOW_END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
echo "$ICON_HOST   Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME   Window:    $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
echo "$ICON_TIME   Duration:  $(format_duration $(( WINDOW_END - WINDOW_START )))"
echo "$ICON_GEAR   Updates:   local=${WEEKLY_SYNC_UPDATES:-false} remote=${WEEKLY_SYNC_UPDATES_REMOTE:-false}"
echo ""

echo "$ICON_SYNC  Sync jobs ($SHARE_COUNT):"
for job in "${PASS[@]}"; do echo "  $ICON_DONE  $job"; done
for job in "${FAIL[@]}"; do echo "  $ICON_ERROR $job"; done
echo "  Passed: ${#PASS[@]}  Failed: ${#FAIL[@]}"

if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
    echo ""
    echo "$ICON_GEAR  Post-sync jobs:"
    for job in "${JOB_PASS[@]}"; do echo "  $ICON_DONE  $job"; done
    for job in "${JOB_FAIL[@]}"; do echo "  $ICON_ERROR $job"; done
fi

TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))

echo ""
if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
    log "$ICON_DONE Status: all complete ✅ — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
else
    warn "Status: $TOTAL_FAIL failure(s)"
    notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
        "Weekly Maintenance" "warning"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
exit 0