diff --git a/Configurations/master.conf b/Configurations/master.conf index 0a23966..4aca331 100644 --- a/Configurations/master.conf +++ b/Configurations/master.conf @@ -687,8 +687,6 @@ FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db" FALLBACK_ENABLED=true # HOST2 back online # false = suppresses "not running" warnings in status scripts - FALLBACK_PARTNERSHIP_REQUIRED=true # gate fallback on an active partnership - # false = standalone mode, no partnership dependency FALLBACK_PARTNERSHIP_SUSPEND_AFTER=120 # minutes without active partnership before suspending # 0 = suspend immediately when partnership goes inactive diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 0e7239d..9c62272 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -682,8 +682,6 @@ FALLBACK_STATE_FILE="$STATE_DIR/fallback_state.db" FALLBACK_ENABLED=false # set true once both servers are configured and paired # false = suppresses "not running" warnings in status scripts - FALLBACK_PARTNERSHIP_REQUIRED=true # gate fallback on an active partnership - # false = standalone mode, no partnership dependency FALLBACK_PARTNERSHIP_SUSPEND_AFTER=120 # minutes without active partnership before suspending # 0 = suspend immediately when partnership goes inactive diff --git a/Fallback/fallback.sh b/Fallback/fallback.sh index ce30e1c..9b90231 100755 --- a/Fallback/fallback.sh +++ b/Fallback/fallback.sh @@ -116,12 +116,11 @@ # Exits if the host cannot be identified — prevents running on an unknown machine. # # Partnership Gate -# When FALLBACK_PARTNERSHIP_REQUIRED=true, fallback is only permitted when the -# local partnership DB reports an active partnership. If partnership goes inactive, -# a grace timer starts. After FALLBACK_PARTNERSHIP_SUSPEND_AFTER minutes the loop -# suspends: no new FALLBACK is entered, and any live fallback containers are -# stopped cleanly. Resumes automatically when partnership becomes active again. -# Set FALLBACK_PARTNERSHIP_REQUIRED=false to run fallback without a partnership. +# Fallback is only permitted when the local partnership DB reports an active +# partnership. If partnership goes inactive, a grace timer starts. After +# FALLBACK_PARTNERSHIP_SUSPEND_AFTER minutes the loop suspends: no new FALLBACK +# is entered, and any live fallback containers are stopped cleanly. Resumes +# automatically when partnership becomes active again. # # Silent by Default # State transitions: warn() — always visible. Healthy routine cycles: log() @@ -174,11 +173,6 @@ # FALLBACK_ENABLED # false = exit cleanly (e.g. remote server being rebuilt). (default: false) # -# FALLBACK_PARTNERSHIP_REQUIRED -# true = fallback is gated on an active partnership — suspends when no active -# partner is found after FALLBACK_PARTNERSHIP_SUSPEND_AFTER minutes. -# false = fallback runs standalone regardless of partnership state. (default: true) -# # FALLBACK_PARTNERSHIP_SUSPEND_AFTER # Minutes without an active partnership before fallback suspends itself. # Grace period: fallback continues normally during this window. @@ -290,6 +284,7 @@ if ! command -v docker &>/dev/null; then fi detect_hosts +require_partnership resolve_remote_ip # Validate unRAID notify script — used throughout for state change notifications @@ -301,7 +296,7 @@ DOCKER_TIMEOUT=15 SSH_TIMEOUT=10 CONTAINER_VERIFY_WAIT=5 # seconds after start before verifying container is up -log "$ICON_GEAR Config: check-interval=${FALLBACK_CHECK_INTERVAL}s handback-strikes=${FALLBACK_HANDBACK_STRIKES} partnership-required=${FALLBACK_PARTNERSHIP_REQUIRED:-true} rsync-enabled=${FALLBACK_RSYNC_ENABLED:-true}" +log "$ICON_GEAR Config: check-interval=${FALLBACK_CHECK_INTERVAL}s handback-strikes=${FALLBACK_HANDBACK_STRIKES} rsync-enabled=${FALLBACK_RSYNC_ENABLED:-true}" log "$ICON_GEAR Timeouts: docker=${DOCKER_TIMEOUT}s ssh=${SSH_TIMEOUT}s verify-wait=${CONTAINER_VERIFY_WAIT}s" # ============================================================================================== @@ -627,19 +622,15 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_GEAR Enabled: $FALLBACK_ENABLED" echo "$ICON_GEAR Dry Run: $DRY_RUN" - if [[ "${FALLBACK_PARTNERSHIP_REQUIRED:-true}" == "true" ]]; then - SUSPENDED=$(state_get partnership_suspended) - PLOST=$(state_get partner_lost_at) - if [[ "$SUSPENDED" == "true" ]]; then - echo "$ICON_GEAR Partnership: SUSPENDED (no active partnership)" - elif [[ -n "$PLOST" && "$PLOST" != "0" ]]; then - PLOST_MIN=$(( ($(date +%s) - PLOST) / 60 )) - echo "$ICON_GEAR Partnership: GRACE (${PLOST_MIN}min / ${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}min until suspend)" - else - echo "$ICON_GEAR Partnership: ACTIVE" - fi + SUSPENDED=$(state_get partnership_suspended) + PLOST=$(state_get partner_lost_at) + if [[ "$SUSPENDED" == "true" ]]; then + echo "$ICON_GEAR Partnership: SUSPENDED (no active partnership)" + elif [[ -n "$PLOST" && "$PLOST" != "0" ]]; then + PLOST_MIN=$(( ($(date +%s) - PLOST) / 60 )) + echo "$ICON_GEAR Partnership: GRACE (${PLOST_MIN}min / ${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}min until suspend)" else - echo "$ICON_GEAR Partnership: REQUIRED=false (standalone mode)" + echo "$ICON_GEAR Partnership: ACTIVE" fi echo "━━━━━━━━━━━━━━━━━━━━━━━" @@ -862,57 +853,55 @@ while [[ "$FALLBACK_RUNNING" == true ]]; do log "Remote: $REMOTE_UP | Internet: $INTERNET_UP | State: $CURRENT_STATE" # ── Partnership gate ────────────────────────────────────────────────────────────────────── - # When FALLBACK_PARTNERSHIP_REQUIRED=true, a fallback without an active partner is - # meaningless — who are we covering for? The gate suspends the loop after a grace - # period and aborts any live fallback containers. Clears automatically when partnership - # is restored. - if [[ "${FALLBACK_PARTNERSHIP_REQUIRED:-true}" == "true" ]]; then - if check_partnership_active; then - # Partnership healthy — clear any accumulated grace state - if [[ "$(state_get partnership_suspended)" == "true" ]]; then - warn "Active partnership restored — re-enabling fallback monitoring" - state_set partnership_suspended "false" - state_set partner_lost_at "0" - notify "Fallback re-enabled on $(hostname) — active partnership restored" \ - "Fallback" "normal" - elif [[ "$(state_get partner_lost_at)" != "0" ]]; then - log "Partnership recovered within grace window — resetting timer" - state_set partner_lost_at "0" + # ── Partnership Gate ────────────────────────────────────────────────────────────────────── + # A fallback without an active partner is meaningless — who are we covering for? + # The gate suspends the loop after a grace period and aborts any live fallback + # containers. Clears automatically when partnership becomes active again. + if check_partnership_active; then + # Partnership healthy — clear any accumulated grace state + if [[ "$(state_get partnership_suspended)" == "true" ]]; then + warn "Active partnership restored — re-enabling fallback monitoring" + state_set partnership_suspended "false" + state_set partner_lost_at "0" + notify "Fallback re-enabled on $(hostname) — active partnership restored" \ + "Fallback" "normal" + elif [[ "$(state_get partner_lost_at)" != "0" ]]; then + log "Partnership recovered within grace window — resetting timer" + state_set partner_lost_at "0" + fi + else + PARTNER_LOST_AT=$(state_get partner_lost_at) + if [[ -z "$PARTNER_LOST_AT" || "$PARTNER_LOST_AT" == "0" ]]; then + warn "No active partnership detected — grace timer started (${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}min before suspend)" + state_set partner_lost_at "$NOW" + PARTNER_LOST_AT=$NOW + fi + + PARTNER_LOST_MIN=$(( (NOW - PARTNER_LOST_AT) / 60 )) + SUSPEND_AFTER=${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120} + + if [[ "$(state_get partnership_suspended)" == "true" || "$PARTNER_LOST_MIN" -ge "$SUSPEND_AFTER" ]]; then + # Suspend — abort any live fallback containers, then skip this cycle entirely + if [[ "$(state_get partnership_suspended)" != "true" ]]; then + warn "No active partnership for ${PARTNER_LOST_MIN}min — suspending fallback" + state_set partnership_suspended "true" + notify "Fallback suspended on $(hostname) — no active partnership for ${PARTNER_LOST_MIN}min" \ + "Fallback" "warning" fi + if [[ "$CURRENT_STATE" == "FALLBACK" ]]; then + warn "Aborting in-progress fallback — partnership suspended" + _abort_fallback_containers + state_set state "NORMAL" + state_set fallback_start "0" + state_set handback_strikes "0" + notify "Fallback aborted on $(hostname) — no active partnership, containers stopped" \ + "Fallback" "warning" + fi + log "Partnership suspended — skipping fallback cycle" + sleep "$FALLBACK_CHECK_INTERVAL" & wait $! + continue else - PARTNER_LOST_AT=$(state_get partner_lost_at) - if [[ -z "$PARTNER_LOST_AT" || "$PARTNER_LOST_AT" == "0" ]]; then - warn "No active partnership detected — grace timer started (${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}min before suspend)" - state_set partner_lost_at "$NOW" - PARTNER_LOST_AT=$NOW - fi - - PARTNER_LOST_MIN=$(( (NOW - PARTNER_LOST_AT) / 60 )) - SUSPEND_AFTER=${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120} - - if [[ "$(state_get partnership_suspended)" == "true" || "$PARTNER_LOST_MIN" -ge "$SUSPEND_AFTER" ]]; then - # Suspend — abort any live fallback containers, then skip this cycle entirely - if [[ "$(state_get partnership_suspended)" != "true" ]]; then - warn "No active partnership for ${PARTNER_LOST_MIN}min — suspending fallback" - state_set partnership_suspended "true" - notify "Fallback suspended on $(hostname) — no active partnership for ${PARTNER_LOST_MIN}min" \ - "Fallback" "warning" - fi - if [[ "$CURRENT_STATE" == "FALLBACK" ]]; then - warn "Aborting in-progress fallback — partnership suspended" - _abort_fallback_containers - state_set state "NORMAL" - state_set fallback_start "0" - state_set handback_strikes "0" - notify "Fallback aborted on $(hostname) — no active partnership, containers stopped" \ - "Fallback" "warning" - fi - log "Partnership suspended — skipping fallback cycle" - sleep "$FALLBACK_CHECK_INTERVAL" & wait $! - continue - else - log "No active partnership — ${PARTNER_LOST_MIN}min elapsed / ${SUSPEND_AFTER}min until suspend" - fi + log "No active partnership — ${PARTNER_LOST_MIN}min elapsed / ${SUSPEND_AFTER}min until suspend" fi fi diff --git a/Media/arr_sync.sh b/Media/arr_sync.sh index 2658707..59c02d3 100755 --- a/Media/arr_sync.sh +++ b/Media/arr_sync.sh @@ -174,6 +174,7 @@ done unset _tool detect_hosts +require_partnership # ── Runtime config with defaults ────────────────────────────────────────────────────────────── ARR_SYNC_ENABLED="${ARR_SYNC_ENABLED:-true}" diff --git a/Media/play_state_sync.sh b/Media/play_state_sync.sh index 77ef7a2..4aaf32b 100755 --- a/Media/play_state_sync.sh +++ b/Media/play_state_sync.sh @@ -80,6 +80,7 @@ for _a in "$@"; do [[ "$_a" == "--full" ]] && FULL_SYNC=true || _FILTERED+=("$_a") done parse_args "${_FILTERED[@]}" +require_partnership # ============================================================================================== # ━━━ Setup ━━━ diff --git a/Monitors/backup_verify.sh b/Monitors/backup_verify.sh index 8329040..6915dee 100755 --- a/Monitors/backup_verify.sh +++ b/Monitors/backup_verify.sh @@ -124,6 +124,7 @@ acquire_lock # detect_hosts() sets MY_ID and aliases BACKUP_VERIFY_SHARES + DAILY_SYNC_SHARES detect_hosts +require_partnership # Share selection — configured list or fallback to daily sync shares if [[ ${#BACKUP_VERIFY_SHARES[@]} -gt 0 ]]; then diff --git a/Plugin/unraid/include/monitor.php b/Plugin/unraid/include/monitor.php index 4dda969..079f094 100644 --- a/Plugin/unraid/include/monitor.php +++ b/Plugin/unraid/include/monitor.php @@ -56,14 +56,13 @@ function vv_fallback_state(): array { $enabled = ($vars['FALLBACK_ENABLED'] ?? 'false') === 'true'; $interval = (int)($vars['FALLBACK_CHECK_INTERVAL'] ?? 30); $reqStrikes = (int)($vars['FALLBACK_HANDBACK_STRIKES'] ?? 3); - $ptRequired = ($vars['FALLBACK_PARTNERSHIP_REQUIRED'] ?? 'true') === 'true'; $suspendAfter = (int)($vars['FALLBACK_PARTNERSHIP_SUSPEND_AFTER'] ?? 120); $stateFile = '/tmp/fallback_state.db'; if (!file_exists($stateFile)) { return ['state' => 'UNKNOWN', 'enabled' => $enabled, 'check_interval' => $interval, 'handback_strikes' => 0, 'handback_strikes_required' => $reqStrikes, - 'partnership_required' => $ptRequired, 'partnership_suspended' => false, + 'partnership_suspended' => false, 'partner_lost_at' => 0, 'partnership_suspend_after' => $suspendAfter]; } $raw = []; @@ -83,7 +82,6 @@ function vv_fallback_state(): array { 'enabled' => $enabled, 'check_interval' => $interval, 'handback_strikes_required' => $reqStrikes, - 'partnership_required' => $ptRequired, 'partnership_suspend_after' => $suspendAfter, ]; } diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 1b6a54a..a6505df 100755 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -186,6 +186,7 @@ fi detect_hosts +require_partnership # Tier 1 global gate — Tier 2 (per-orchestrator) checked by caller if ! check_rsync_enabled; then diff --git a/System_Essentials/conf_cache_restore.sh b/System_Essentials/conf_cache_restore.sh index f973389..3c2d48a 100755 --- a/System_Essentials/conf_cache_restore.sh +++ b/System_Essentials/conf_cache_restore.sh @@ -27,6 +27,7 @@ source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" detect_hosts +require_partnership RAM_CACHE="/tmp/.cache/vv/d" SAVE_DIR="/boot/config/.cache/vv/d" diff --git a/System_Essentials/conf_cache_save.sh b/System_Essentials/conf_cache_save.sh index c0adcd4..8f0d3d7 100755 --- a/System_Essentials/conf_cache_save.sh +++ b/System_Essentials/conf_cache_save.sh @@ -23,6 +23,7 @@ source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" detect_hosts +require_partnership RAM_CACHE="/tmp/.cache/vv/d" SAVE_DIR="/boot/config/.cache/vv/d" diff --git a/System_Essentials/conf_sync.sh b/System_Essentials/conf_sync.sh index 0debb59..2e2ce38 100755 --- a/System_Essentials/conf_sync.sh +++ b/System_Essentials/conf_sync.sh @@ -48,6 +48,7 @@ for arg in "$@"; do done parse_args "${FILTERED_ARGS[@]}" detect_hosts +require_partnership if [[ "${CONF_SYNC_ENABLED:-true}" == false ]]; then log "CONF_SYNC_ENABLED=false — skipping" diff --git a/Watchdogs/System/conf_cache_watchdog.sh b/Watchdogs/System/conf_cache_watchdog.sh old mode 100644 new mode 100755 index d722f51..5d3df2a --- a/Watchdogs/System/conf_cache_watchdog.sh +++ b/Watchdogs/System/conf_cache_watchdog.sh @@ -26,6 +26,7 @@ source "$SCRIPT_DIR/../../load_config.sh" parse_args "$@" detect_hosts +require_partnership [[ "${FALLBACK_ENABLED:-false}" != "true" ]] && exit 0 [[ "${CONF_SYNC_ENABLED:-true}" != "true" ]] && exit 0 diff --git a/common.sh b/common.sh index 2f776d4..7fcccaf 100755 --- a/common.sh +++ b/common.sh @@ -1923,4 +1923,10 @@ show_status() { echo "Delayed: ${DELAYED_CONTAINERS[*]:-n/a}" echo "Excludes: ${EXCLUDE_DIRS[*]:-n/a}" echo "━━━━━━━━━━━━━━━━━━━━━━━" +} + +require_partnership() { + [[ "${PARTNERSHIP_ENABLED:-false}" == "true" ]] && return 0 + log "PARTNERSHIP_ENABLED=false — skipping cross-server operation" + exit 0 } \ No newline at end of file