diff --git a/Failover/failover.sh b/Failover/failover.sh index ab03978..85be47e 100644 --- a/Failover/failover.sh +++ b/Failover/failover.sh @@ -471,6 +471,13 @@ run_handback() { info "Outage duration: ${outage_minutes}min" + # Check rsync gate before running any writeback + if ! check_rsync_enabled "FAILOVER"; then + warn "FAILOVER_RSYNC_ENABLED=false — skipping all writeback jobs" + warn "Handback will complete without syncing state back to primary" + return 0 + fi + run_writeback_tier() { local tier="$1" local threshold="$2" diff --git a/Master.conf b/Master.conf index 56e00b0..f29d780 100644 --- a/Master.conf +++ b/Master.conf @@ -265,6 +265,30 @@ MEDIA_MANAGEMENT_JOBS=( # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== +# ━━━ Rsync Enable/Disable ━━━ +# Two-tier toggle system — Tier 1 overrides Tier 2. +# +# Tier 1 — Global gate: +# RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions +# Use when: remote server completely offline, major maintenance, disaster recovery +# +# Tier 2 — Per-orchestrator (only applies when Tier 1 is true): +# Fine grained control — disable specific orchestrators while keeping others +# Use when: rebuilding secondary, testing, per-window bandwidth management +# +# Real world example (HOST2 data rebuild — your current situation): +# RSYNC_ENABLED=true ← rsync works, individual scripts run fine +# DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild +# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe, separate BW) +# FAILOVER_RSYNC_ENABLED=true ← handback writeback still works when needed +# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies (test each share manually) +# → When ready: DAILY_RSYNC_ENABLED=true + + RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below + DAILY_RSYNC_ENABLED=false # Tier 2 — daily_sync_maintenance.sh rsync section + WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section + FAILOVER_RSYNC_ENABLED=true # Tier 2 — failover.sh writeback jobs on handback + # ━━━ Rsync Defaults ━━━ # Global fallback values used when no profile match is found. # Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed. diff --git a/Orchestrators/daily_sync_maintenance.sh b/Orchestrators/daily_sync_maintenance.sh index 864f070..e4f2688 100644 --- a/Orchestrators/daily_sync_maintenance.sh +++ b/Orchestrators/daily_sync_maintenance.sh @@ -147,10 +147,14 @@ echo "$ICON_SUMMARY Shares: $SHARE_COUNT" echo "" SHARE_INDEX=0 - ABORT_ALL_SYNCS=false -for SHARE in "${ALL_SHARES[@]}"; do +# Tier 1 + Tier 2 rsync gate check +if ! check_rsync_enabled "DAILY"; then + warn "Rsync disabled — skipping all $SHARE_COUNT share syncs" + warn "Proceeding to media management jobs..." +else + for SHARE in "${ALL_SHARES[@]}"; do SHARE_INDEX=$((SHARE_INDEX + 1)) SHARE_NAME=$(basename "$SHARE") SHARE_START=$(date +%s) @@ -190,6 +194,7 @@ for SHARE in "${ALL_SHARES[@]}"; do echo "" done +fi # end check_rsync_enabled "DAILY" TOTAL_END=$(date +%s) TOTAL_DURATION=$((TOTAL_END - TOTAL_START)) diff --git a/Orchestrators/weekly_sync_maintenance.sh b/Orchestrators/weekly_sync_maintenance.sh index 3074ce6..5560e71 100644 --- a/Orchestrators/weekly_sync_maintenance.sh +++ b/Orchestrators/weekly_sync_maintenance.sh @@ -177,10 +177,16 @@ echo "$ICON_SUMMARY Jobs: $SHARE_COUNT" echo "" JOB_NUM=0 -for JOB in "${SYNC_JOBS[@]}"; do - ((JOB_NUM++)) - JOB_NAME=$(basename "$JOB") - echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━" + +# Tier 1 + Tier 2 rsync gate check +if ! check_rsync_enabled "WEEKLY"; then + warn "Rsync disabled — skipping all $SHARE_COUNT weekly sync jobs" + warn "Proceeding to container updates and maintenance scripts..." +else + for JOB in "${SYNC_JOBS[@]}"; do + ((JOB_NUM++)) + JOB_NAME=$(basename "$JOB") + echo "━━━ [$JOB_NUM/$SHARE_COUNT] $JOB_NAME ━━━" JOB_START=$(date +%s) @@ -207,6 +213,7 @@ for JOB in "${SYNC_JOBS[@]}"; do echo "" done +fi # end check_rsync_enabled "WEEKLY" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━ diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index b27ad07..3894c4a 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -48,6 +48,14 @@ echo "" echo "━━━ $ICON_GEAR Setup ━━━" detect_hosts + +# Tier 1 global gate — check before doing anything +# Tier 2 (per-orchestrator) is handled by the calling orchestrator +# Direct calls to rsync.sh only check Tier 1 +if ! check_rsync_enabled; then + warn "RSYNC_ENABLED=false — exiting cleanly" + exit 0 +fi resolve_remote_ip # ----------------------------------------------------------------------------------------------- diff --git a/common.sh b/common.sh index 976adc0..2865239 100644 --- a/common.sh +++ b/common.sh @@ -339,6 +339,45 @@ resolve_remote_ip() { # Pings remote and exits if unreachable. # For failover use ping_remote() which returns status without exiting. # ----------------------------------------------------------------------------------------------- + +# ----------------------------------------------------------------------------------------------- +# check_rsync_enabled — two-tier rsync gate check +# Tier 1: RSYNC_ENABLED — global, overrides everything +# Tier 2: orchestrator-specific flag passed as argument +# +# Usage: +# check_rsync_enabled "DAILY" ← checks RSYNC_ENABLED + DAILY_RSYNC_ENABLED +# check_rsync_enabled "WEEKLY" ← checks RSYNC_ENABLED + WEEKLY_RSYNC_ENABLED +# check_rsync_enabled "FAILOVER" ← checks RSYNC_ENABLED + FAILOVER_RSYNC_ENABLED +# check_rsync_enabled ← checks RSYNC_ENABLED only (rsync.sh direct call) +# +# Returns: +# 0 = rsync enabled, proceed +# 1 = rsync disabled, skip cleanly +# ----------------------------------------------------------------------------------------------- +check_rsync_enabled() { + local orchestrator="${1:-}" + + # Tier 1 — global gate + if [[ "${RSYNC_ENABLED:-true}" == false ]]; then + warn "RSYNC_ENABLED=false — rsync globally disabled, skipping all syncs" + return 1 + fi + + # Tier 2 — per-orchestrator + if [[ -n "$orchestrator" ]]; then + local var_name="${orchestrator}_RSYNC_ENABLED" + local var_value="${!var_name:-true}" + if [[ "$var_value" == false ]]; then + info "${var_name}=false — rsync disabled for $orchestrator orchestrator" + info "All other $orchestrator jobs will still run normally" + return 1 + fi + fi + + return 0 +} + check_connectivity() { log "Checking connectivity to $REMOTE_SERVER..." if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then