Varaverk/Orchestrators/daily_sync_maintenance.sh

#!/bin/bash
# ==============================================================================================
# ============================= Daily Sync Maintenance =========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Daily maintenance window orchestrator — runs the full daily sequence in the
# correct order. Schedule: 0 1 * * * (1am daily via User Scripts plugin)
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Pre-sync:
#   git_pull_execute.sh              — pull latest scripts first, always
#
# Arr Sync (arr_sync.sh):
#   Syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
#   All nodes agree on tracked library before any files are transferred.
#   Remote nodes that don't have an arr running are skipped gracefully.
#
# Rsync window (DAILY_SYNC_SHARES per host):
#   HOST*_DAILY_SYNC_SHARES          — media shares spread to all nodes (no --delete)
#   HOST*_PERSONAL_SHARES            — encrypted personal shares
#
# Post-sync maintenance (DAILY_MAINTENANCE_SCRIPTS):
#   media_shares_permissions.sh      — fix ownership before arr cleanup
#   media_cleaner.sh anime/media     — remove junk from media and anime shares
#   lidarr/sonarr/radarr_cleanup.sh  — remove orphaned files (local arr = truth)
#   docker_daily_restart.sh          — restart containers needing daily restart
#
# DRIVE TEMP HANDLING (rsync.sh exit codes):
#   exit 1 = temp WARN    → skip this share, continue to next
#   exit 2 = temp CRITICAL → abort ALL remaining syncs in this window
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Order Is Load-Bearing
#   git pull first — maintenance runs on latest code, not yesterday's. Arr sync
#   before rsync — all nodes track the same library before files are spread,
#   preventing remote arrs from searching for content already owned. Rsync before
#   cleanup — cleanup sees fully spread state. Permissions before arr cleanup —
#   arrs need correct ownership to delete/rename. Docker restart last — containers
#   already processed by cleanup.
#
# Host-Aware Routing
#   Bidirectional — same script runs on both servers, correct direction automatic.
#   detect_hosts() aliases DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars.
#   No manual HOST1/HOST2 comparisons needed.
#
# Silent When Healthy
#   Runs daily at 1am — clean runs produce minimal output. Each job logs silently
#   on success; failures surface to warn()/error(). Notify only on failure.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Enforcement
#   rsync and docker operations require root.
#
# Lock Acquisition
#   acquire_lock prevents two daily windows overlapping — the window is long and a
#   second pass would contend for the same shares and containers.
#
# Host Detection
#   detect_hosts() aliases the correct per-host share and script lists.
#
# Empty Job List Guard
#   Exits with an error and a notification if DAILY_MAINTENANCE_SCRIPTS is empty. This
#   is the largest tier in the ecosystem — an empty list would silently skip git pull,
#   permissions, cleaners, arr cleanup and docker updates while reporting a clean run.
#
# Connectivity Check
#   check_connectivity is verified before any rsync is attempted.
#
# Remote Rootfs Check
#   check_remote_rootfs aborts rsync if the remote rootfs is nearly full, rather than
#   pushing data to a partner that cannot hold it.
#
# Drive Temperature Escalation
#   rsync.sh's exit code is honoured per share: exit 1 (temp WARN) skips that share and
#   continues; exit 2 (temp CRITICAL) sets ABORT_ALL_SYNCS so every remaining share in
#   the window is skipped and a notification is raised. Continuing to hammer drives that
#   are already too hot is how a thermal warning becomes a dead disk.
#
# Non-Fatal Jobs
#   A failed job is logged and the remaining jobs still run. Partial completion of a
#   maintenance window beats abandoning it at the first error.
#
# Quiet on Success
#   A successful daily run produces no notification — only failures surface.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# host*.conf
#
#   HOST*_DAILY_SYNC_SHARES   — shares pushed to mirror each day
#   HOST*_PERSONAL_SHARES     — encrypted personal shares
#
# master.conf
#
#   DAILY_MAINTENANCE_SCRIPTS — maintenance jobs (permissions, cleanup, restart)
#   DAILY_RSYNC_ENABLED       — enable/disable rsync section
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# daily_sync_maintenance.sh
#     Normal run.
#
# daily_sync_maintenance.sh --dry-run
#     Preview without syncing or changing.
#
# daily_sync_maintenance.sh --log
#     Verbose per-share/per-job output.
#
# daily_sync_maintenance.sh --status
#     Show configured shares and jobs.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi


if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts

# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
# from a healthy run. Fail loudly instead of silently doing no work.
if [[ ${#DAILY_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
    error "DAILY_MAINTENANCE_SCRIPTS is empty — no daily maintenance scripts will run"
    error "Check DAILY_MAINTENANCE_SCRIPTS in master.conf"
    notify "daily maintenance scripts skipped on $(hostname) ($MY_ID) — DAILY_MAINTENANCE_SCRIPTS is empty" \
        "$(basename "$0" .sh)" "warning"
    exit 1
fi
resolve_remote_ip

acquire_lock

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY DAILY SYNC STATUS ━━━━━"
    echo "$ICON_HOST   Identity:        $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_NET    Remote:          $REMOTE_ID ($REMOTE_SERVER_NAME)"
    echo "$ICON_SYNC   Rsync enabled:   ${RSYNC_ENABLED:-true}"
    echo "$ICON_SYNC   Daily enabled:   ${DAILY_RSYNC_ENABLED:-false}"
    echo ""
    echo "━━━ Daily Sync Shares ━━━"
    for share in "${DAILY_SYNC_SHARES[@]}"; do
        [[ -n "$share" ]] && echo "  $ICON_SYNC $share"
    done
    for share in "${PERSONAL_SHARES[@]}"; do
        [[ -n "$share" ]] && echo "  $ICON_SYNC $share (personal)"
    done
    echo ""
    echo "━━━ Daily Maintenance Scripts ━━━"
    for entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
        [[ -z "$entry" ]] && continue
        echo "  $ICON_GEAR ${entry##*/}"
    done
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ── BUILD SHARE LIST via detect_hosts aliases ─────────────────────────────────────────────────
# detect_hosts() sets DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars
# No manual HOST1/HOST2 comparison needed — aliased automatically per server
# ==============================================================================================
ALL_SHARES=()
for share in "${DAILY_SYNC_SHARES[@]}"; do
    [[ -n "$share" ]] && ALL_SHARES+=("$share")
done
for share in "${PERSONAL_SHARES[@]}"; do
    [[ -n "$share" ]] && ALL_SHARES+=("$share")
done
SHARE_COUNT=${#ALL_SHARES[@]}

# ── Split maintenance scripts: git pull runs pre-sync, rest run post-sync ─────────────────────
PRE_SYNC_SCRIPTS=()
POST_SYNC_SCRIPTS=()
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
    [[ -z "$script_entry" ]] && continue
    script_name=$(basename "${script_entry%% *}")
    if [[ "$script_name" == "git_pull_execute.sh" ]]; then
        PRE_SYNC_SCRIPTS+=("$script_entry")
    else
        POST_SYNC_SCRIPTS+=("$script_entry")
    fi
done

WINDOW_START=$(date +%s)
JOB_PASS=()
JOB_FAIL=()
PASS=()
FAIL=()
SHARE_TIMES=()

echo ""
echo "━━━ $ICON_GEAR Daily Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

# ==============================================================================================
# ━━━ Pre-sync — git pull ━━━
# ==============================================================================================
if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_GIT Pre-sync ━━━"
    for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
        run_orch_child "$script_entry"
    done
fi

# ==============================================================================================
# ━━━ Arr Sync — library reconciliation before file spreading ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Arr Sync ━━━"

ARR_SYNC_SCRIPT="$ECOSYSTEM_ROOT/Arrs_Stack/arr_sync.sh"
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
    echo "ARR_SYNC_ENABLED=false — skipping"
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
    warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
    JOB_FAIL+=("arr_sync.sh")
else
    _arr_sync_args=()
    [[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
    if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
        echo "Arr sync complete ✅"
        JOB_PASS+=("arr_sync.sh")
    else
        warn "Arr sync completed with errors — continuing to rsync"
        JOB_FAIL+=("arr_sync.sh")
    fi
    unset _arr_sync_args
fi

# ==============================================================================================
# ━━━ Media Share Sync ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Media Share Sync — $SHARE_COUNT share(s) ━━━"

TOTAL_START=$(date +%s)
SHARE_INDEX=0
ABORT_ALL_SYNCS=false

if ! check_rsync_enabled "DAILY"; then
    warn "Daily rsync disabled — skipping all $SHARE_COUNT share syncs"
    warn "Proceeding to maintenance jobs..."
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
    warn "No shares configured for $MY_ID — check HOST*_DAILY_SYNC_SHARES in host*.conf"
else
    # Pre-flight — connectivity then remote rootfs
    check_connectivity
    check_remote_rootfs

    RSYNC_DRY=""
    [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"

    for SHARE in "${ALL_SHARES[@]}"; do
        (( SHARE_INDEX++ ))
        SHARE_NAME=$(basename "$SHARE")
        SHARE_START=$(date +%s)

        echo ""
        echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"

        if [[ "$ABORT_ALL_SYNCS" == true ]]; then
            warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
            FAIL+=("$SHARE_NAME:temp-critical")
            continue
        fi

        bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
        RSYNC_EXIT=$?

        SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")

        case "$RSYNC_EXIT" in
            0)
                PASS+=("$SHARE_NAME")
                echo "$SHARE_NAME — done ✅"
                ;;
            1)
                FAIL+=("$SHARE_NAME:temp-warn")
                warn "$SHARE_NAME skipped — drive temps too high"
                ;;
            2)
                FAIL+=("$SHARE_NAME:temp-critical")
                ABORT_ALL_SYNCS=true
                error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
                notify "Daily sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
                    "Daily Sync" "warning"
                ;;
            *)
                FAIL+=("$SHARE_NAME")
                error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
                ;;
        esac

    done
fi

TOTAL_END=$(date +%s)

# ==============================================================================================
# ━━━ Post-sync Maintenance Jobs ━━━
# ==============================================================================================
if [[ ${#POST_SYNC_SCRIPTS[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━"
    for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
        echo ""
        run_orch_child "$script_entry"
    done
fi

WINDOW_END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━"
echo "$ICON_HOST   Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME   Window:   $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
echo "$ICON_TIME   Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
echo ""

echo "$ICON_SYNC  Shares ($SHARE_COUNT):"
for entry in "${SHARE_TIMES[@]}"; do
    sname="${entry%%:*}"
    sdur="${entry##*:}"
    if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
        echo "  $ICON_ERROR $sname — $(format_duration "$sdur")"
    else
        echo "  $ICON_DONE  $sname — $(format_duration "$sdur")"
    fi
done
[[ "$SHARE_COUNT" -eq 0 || "${DAILY_RSYNC_ENABLED:-false}" == "false" ]] && \
    echo "  (rsync disabled)"
echo "  Passed: ${#PASS[@]}/$SHARE_COUNT  Failed: ${#FAIL[@]}/$SHARE_COUNT"
echo ""

if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
    echo "$ICON_GEAR  Jobs:"
    for job in "${JOB_PASS[@]}"; do echo "  $ICON_DONE  $job"; done
    for job in "${JOB_FAIL[@]}"; do echo "  $ICON_ERROR $job"; done
    echo ""
fi

TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))

if [[ "$TOTAL_FAIL" -gt 0 ]]; then
    warn "Status: $TOTAL_FAIL failure(s)"
    notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
        "Daily Maintenance" "warning"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    exit 1
else
    echo "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi