Varaverk/Orchestrators/daily_sync_maintenance.sh

#!/bin/bash
# ==============================================================================================
# ============================= Daily Sync Maintenance =========================================
# ==============================================================================================
# Daily orchestrator — runs the full daily maintenance window in the correct order.
# Schedule: 0 1 * * * (1am daily via User Scripts plugin)
#
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
#   Pre-sync:
#     git_pull_execute.sh              — pull latest scripts first, always
#
#   Arr Sync (arr_sync.sh):
#     Syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
#     All nodes agree on tracked library before any files are transferred.
#     Remote nodes that don't have an arr running are skipped gracefully.
#
#   Rsync window (DAILY_SYNC_SHARES per host):
#     HOST*_DAILY_SYNC_SHARES          — media shares spread to all nodes (no --delete)
#     HOST*_PERSONAL_SHARES            — encrypted personal shares
#
#   Post-sync maintenance (DAILY_MAINTENANCE_SCRIPTS):
#     media_shares_permissions.sh      — fix ownership before arr cleanup
#     media_cleaner.sh anime           — remove junk from anime shares
#     media_cleaner.sh media           — remove junk from media shares
#     lidarr_cleanup.sh                — remove orphaned music files (local arr = truth)
#     sonarr_cleanup.sh                — remove orphaned TV files (local arr = truth)
#     radarr_cleanup.sh                — remove orphaned movie files (local arr = truth)
#     docker_daily_restart.sh          — restart containers needing daily restart
#
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
#   git pull first        — maintenance runs on latest code, not yesterday's
#   arr sync before rsync — all nodes track the same library before files are spread;
#                           prevents remote arrs from searching for content already owned
#   rsync before cleanup  — cleanup sees fully spread state, rsync has no --delete
#   permissions before arr cleanup — arrs need correct ownership to delete/rename
#   arr cleanup after permissions  — clean ownership = successful orphan deletion
#   docker restart last   — containers already processed by cleanup
#
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
# Bidirectional — same script runs on both servers, correct direction automatic.
# detect_hosts() aliases DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars.
# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server.
#
# ── DRIVE TEMP HANDLING ───────────────────────────────────────────────────────────────────────
# rsync.sh returns exit codes for temperature issues:
#   exit 1 = temp WARN — skip this share, continue to next
#   exit 2 = temp CRITICAL — abort ALL remaining syncs in this window
#   All other failures — skip share, continue to next
#
# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
# Runs daily at 1am — clean run should produce minimal output.
# Each job reports log() on success (silent), warn()/error() on failure (visible).
# Summary always shown — gives window timing and share/job counts.
# Notify only on failure — successful daily maintenance doesn't need notification.
#
# ── CONFIGURATION (master.conf + master_host*.conf) ───────────────────────────────────────────
#   HOST*_DAILY_SYNC_SHARES      — shares pushed to mirror each day
#   HOST*_PERSONAL_SHARES        — encrypted personal shares
#   DAILY_MAINTENANCE_SCRIPTS    — maintenance jobs (permissions, cleanup, restart)
#   DAILY_RSYNC_ENABLED          — enable/disable rsync section
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   daily_sync_maintenance.sh              — normal run
#   daily_sync_maintenance.sh --dry-run    — preview without syncing or changing
#   daily_sync_maintenance.sh --log        — verbose per-share/per-job output
#   daily_sync_maintenance.sh --status     — show configured shares and jobs
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
SCRIPTS_ROOT="$SCRIPT_DIR/.."

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

validate_unraid_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

detect_hosts
resolve_remote_ip

acquire_lock

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY DAILY SYNC STATUS ━━━━━"
    echo "$ICON_HOST   Identity:        $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_NET    Remote:          $REMOTE_ID ($REMOTE_SERVER_NAME)"
    echo "$ICON_SYNC   Rsync enabled:   ${RSYNC_ENABLED:-true}"
    echo "$ICON_SYNC   Daily enabled:   ${DAILY_RSYNC_ENABLED:-false}"
    echo ""
    echo "━━━ Daily Sync Shares ━━━"
    for share in "${DAILY_SYNC_SHARES[@]}"; do
        [[ -n "$share" ]] && echo "  $ICON_SYNC $share"
    done
    for share in "${PERSONAL_SHARES[@]}"; do
        [[ -n "$share" ]] && echo "  $ICON_SYNC $share (personal)"
    done
    echo ""
    echo "━━━ Daily Maintenance Scripts ━━━"
    for entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
        [[ -z "$entry" ]] && continue
        echo "  $ICON_GEAR ${entry##*/}"
    done
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ── BUILD SHARE LIST via detect_hosts aliases ─────────────────────────────────────────────────
# detect_hosts() sets DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars
# No manual HOST1/HOST2 comparison needed — aliased automatically per server
# ==============================================================================================
ALL_SHARES=()
for share in "${DAILY_SYNC_SHARES[@]}"; do
    [[ -n "$share" ]] && ALL_SHARES+=("$share")
done
for share in "${PERSONAL_SHARES[@]}"; do
    [[ -n "$share" ]] && ALL_SHARES+=("$share")
done
SHARE_COUNT=${#ALL_SHARES[@]}

# ── Split maintenance scripts: git pull runs pre-sync, rest run post-sync ─────────────────────
PRE_SYNC_SCRIPTS=()
POST_SYNC_SCRIPTS=()
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
    [[ -z "$script_entry" ]] && continue
    script_name=$(basename "${script_entry%% *}")
    if [[ "$script_name" == "git_pull_execute.sh" ]]; then
        PRE_SYNC_SCRIPTS+=("$script_entry")
    else
        POST_SYNC_SCRIPTS+=("$script_entry")
    fi
done

# Helper — run a maintenance script, track pass/fail
run_job() {
    local script_entry="$1"
    local extra_dry=""
    [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"

    read -r -a script_args <<< "$script_entry"
    local script_path="$SCRIPTS_ROOT/${script_args[0]}"
    local script_name
    script_name=$(basename "${script_args[0]}")
    local extra_args=("${script_args[@]:1}")

    if [[ ! -f "$script_path" ]]; then
        error "$script_name — not found at $script_path"
        JOB_FAIL+=("$script_name")
        return 1
    fi

    log "Running: $script_name ${extra_args[*]}"
    # shellcheck disable=SC2086
    if bash "$script_path" "${extra_args[@]}" $extra_dry; then
        log "$script_name — done ✅"
        JOB_PASS+=("$script_name ${extra_args[*]}")
    else
        error "$script_name — failed (exit $?)"
        JOB_FAIL+=("$script_name ${extra_args[*]}")
    fi
}

WINDOW_START=$(date +%s)
JOB_PASS=()
JOB_FAIL=()
PASS=()
FAIL=()
SHARE_TIMES=()

echo ""
echo "━━━ $ICON_GEAR Daily Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

# ==============================================================================================
# ━━━ Pre-sync — git pull ━━━
# ==============================================================================================
if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_GIT Pre-sync ━━━"
    for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
        run_job "$script_entry"
    done
fi

# ==============================================================================================
# ━━━ Arr Sync — library reconciliation before file spreading ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Arr Sync ━━━"

ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
    log "ARR_SYNC_ENABLED=false — skipping"
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
    warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
    JOB_FAIL+=("arr_sync.sh")
else
    _arr_sync_args=()
    [[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
    if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
        log "Arr sync complete ✅"
        JOB_PASS+=("arr_sync.sh")
    else
        warn "Arr sync completed with errors — continuing to rsync"
        JOB_FAIL+=("arr_sync.sh")
    fi
    unset _arr_sync_args
fi

# ==============================================================================================
# ━━━ Media Share Sync ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Media Share Sync — $SHARE_COUNT share(s) ━━━"

TOTAL_START=$(date +%s)
SHARE_INDEX=0
ABORT_ALL_SYNCS=false

if ! check_rsync_enabled "DAILY"; then
    warn "Daily rsync disabled — skipping all $SHARE_COUNT share syncs"
    warn "Proceeding to maintenance jobs..."
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
    warn "No shares configured for $MY_ID — check HOST*_DAILY_SYNC_SHARES in master_host*.conf"
else
    # Pre-flight — connectivity then remote rootfs
    check_connectivity
    check_remote_rootfs

    RSYNC_DRY=""
    [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"

    for SHARE in "${ALL_SHARES[@]}"; do
        (( SHARE_INDEX++ ))
        SHARE_NAME=$(basename "$SHARE")
        SHARE_START=$(date +%s)

        echo ""
        echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"

        if [[ "$ABORT_ALL_SYNCS" == true ]]; then
            warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
            FAIL+=("$SHARE_NAME:temp-critical")
            continue
        fi

        bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
        RSYNC_EXIT=$?

        SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")

        case "$RSYNC_EXIT" in
            0)
                PASS+=("$SHARE_NAME")
                log "$SHARE_NAME — done ✅"
                ;;
            1)
                FAIL+=("$SHARE_NAME:temp-warn")
                warn "$SHARE_NAME skipped — drive temps too high"
                ;;
            2)
                FAIL+=("$SHARE_NAME:temp-critical")
                ABORT_ALL_SYNCS=true
                error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
                notify "Daily sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
                    "Daily Sync" "warning"
                ;;
            *)
                FAIL+=("$SHARE_NAME")
                error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
                ;;
        esac

    done
fi

TOTAL_END=$(date +%s)

# ==============================================================================================
# ━━━ Post-sync Maintenance Jobs ━━━
# ==============================================================================================
if [[ ${#POST_SYNC_SCRIPTS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━"
    for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
        echo ""
        run_job "$script_entry"
    done
fi

WINDOW_END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━"
echo "$ICON_HOST   Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME   Window:   $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
echo "$ICON_TIME   Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
echo ""

echo "$ICON_SYNC  Shares ($SHARE_COUNT):"
for entry in "${SHARE_TIMES[@]}"; do
    sname="${entry%%:*}"
    sdur="${entry##*:}"
    if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
        echo "  $ICON_ERROR $sname — $(format_duration "$sdur")"
    else
        echo "  $ICON_DONE  $sname — $(format_duration "$sdur")"
    fi
done
[[ "$SHARE_COUNT" -eq 0 || "${DAILY_RSYNC_ENABLED:-false}" == "false" ]] && \
    echo "  (rsync disabled)"
echo "  Passed: ${#PASS[@]}/$SHARE_COUNT  Failed: ${#FAIL[@]}/$SHARE_COUNT"
echo ""

if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
    echo "$ICON_GEAR  Jobs:"
    for job in "${JOB_PASS[@]}"; do echo "  $ICON_DONE  $job"; done
    for job in "${JOB_FAIL[@]}"; do echo "  $ICON_ERROR $job"; done
    echo ""
fi

TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))

if [[ "$TOTAL_FAIL" -gt 0 ]]; then
    warn "Status: $TOTAL_FAIL failure(s)"
    notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
        "Daily Maintenance" "warning"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    exit 1
else
    log "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi