feat: arr-native distributed media sync + config architecture fixes
Arr sync (new): - Media/arr_sync.sh — full mesh bidirectional sync across all HOST* nodes - Lidarr (MusicBrainz), Sonarr (TVDB), Radarr (TMDB) all handled in one script - Remote API keys read live from config.xml via SSH — never stored in conf files - Shared blocklist (DATA_DIR/arr_sync_blocklist.tsv) merged from all nodes at runtime - Graceful skip if arr not configured locally or not reachable on a remote node - --blocklist-add / --blocklist-remove / --blocklist-list management flags - daily_sync_maintenance.sh — arr sync runs as explicit phase before rsync - partnership_onboard.sh — Step 3 bootstraps merged library on both sides at onboard - master.conf — ARR_SYNC_* config block, DOCKER_APPDATA_BASE Rsync / cleanup: - DEFAULT_RSYNC_OPTS — removed --delete; arr_cleanup.sh owns orphan enforcement - lidarr_cleanup.sh — removed HOST1-only guard; runs on any node with Lidarr configured Config architecture: - HOST1/HOST2 hostnames moved from master_host*.conf → master.conf (not credentials) - Sparse checkout now works correctly: each server only needs its own host conf - detect_hosts() still resolves MY_ID + REMOTE_ID via master.conf hostname values Bug fix: - common.sh line 493 — watchdog toggle eval had broken quoting; all SYS_WATCHDOG_CHECK_* globals were silently set to empty instead of their configured values Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Gmer4Lfe
co-authored by
Claude Sonnet 4.6
Claude Sonnet 4.6
parent
009820e981
commit
b65f572367
+122
-135
@@ -21,7 +21,7 @@
|
||||
# NORMAL — remote up, internet up
|
||||
# Own containers only. DDNS ON. Silent operation.
|
||||
#
|
||||
# FAILOVER — remote down, internet up
|
||||
# FALLBACK — remote down, internet up
|
||||
# Start remote containers locally — tiered by outage duration.
|
||||
# Remote DDNS started immediately (Tier 1).
|
||||
# Own containers keep running — failover is additive.
|
||||
@@ -40,20 +40,21 @@
|
||||
# Script controls DDNS exclusively — network state NEVER auto-starts DDNS.
|
||||
# DDNS only starts after full handback sequence confirms containers are up.
|
||||
# One DDNS per domain active at all times — never two, never zero for long.
|
||||
# 1 minute TTL + 1 minute check interval = minimal user impact on failover.
|
||||
# 1 minute TTL + 90s strike window = minimal user impact on failover.
|
||||
#
|
||||
# ── HANDBACK SEQUENCE ─────────────────────────────────────────────────────────────────────────
|
||||
# When remote returns after FAILOVER:
|
||||
# When remote returns after FALLBACK:
|
||||
# 1. Strike confirmation — FALLBACK_HANDBACK_STRIKES consecutive remote-up checks
|
||||
# 2. Pre-flight checks — version parity, remote array, remote Docker daemon
|
||||
# 3. Stop remote DDNS first — prevents split brain DNS during rsync
|
||||
# 4. Stop remote containers — clean state before rsync
|
||||
# 5. Tiered rsync writeback — skip if outage under threshold
|
||||
# 6. Start remote containers — in dependency order with verification
|
||||
# 7. Start remote DDNS last — DNS cuts back ONLY after containers confirmed up
|
||||
# 8. Return to NORMAL
|
||||
# 3. Staged reverse-tier handback — Tier 4 → Tier 3 → Tier 2 → Tier 1
|
||||
# Each stage: stop local tier containers → rsync writeback → start on remote
|
||||
# Emby (Tier 1) stays on fallback serving users until all higher tiers are done
|
||||
# 4. DDNS handoff — immediately before Tier 1 goes down (not at start)
|
||||
# 5. Tier 1 handback — stop local → rsync → start remote
|
||||
# 6. Start remote DDNS last — DNS cuts back ONLY after containers confirmed up
|
||||
# 7. Return to NORMAL
|
||||
#
|
||||
# ── TIERED FAILOVER ───────────────────────────────────────────────────────────────────────────
|
||||
# ── TIERED FALLBACK ───────────────────────────────────────────────────────────────────────────
|
||||
# Tier 1 — Immediate — vital services + Live TV — cannot wait
|
||||
# Tier 2 — HOST*_TIER2_DELAY — shared productivity services (default 4hr)
|
||||
# Tier 3 — HOST*_TIER3_DELAY — secondary services (default 12hr)
|
||||
@@ -72,16 +73,16 @@
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_DDNS_CONTAINERS — DDNS containers this host manages
|
||||
# FAILOVER_HOST*_STOP_ON_NO_NET — containers stopped on internet loss
|
||||
# FAILOVER_HOST*_RUNS_FOR_HOST*_TIER1-4 — what this host runs for the other
|
||||
# FALLBACK_HOST*_STOP_ON_NO_NET — containers stopped on internet loss
|
||||
# FALLBACK_HOST*_RUNS_FOR_HOST*_TIER1-4 — what this host runs for the other
|
||||
# HOST*_TIER2_DELAY / TIER3_DELAY / TIER4_DELAY — tier activation delays in minutes
|
||||
# HOST*_TIER1_WRITEBACK_DELAY — skip Tier 1 writeback if outage under this
|
||||
# FAILOVER_HOST*_WRITEBACK_TIER1-4 — paths synced back on handback per tier
|
||||
# FALLBACK_HOST*_WRITEBACK_TIER1-4 — paths synced back on handback per tier
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# FALLBACK_ENABLED — false = exit cleanly (HOST2 being rebuilt etc.)
|
||||
# FALLBACK_CHECK_INTERVAL — seconds between checks
|
||||
# FALLBACK_HANDBACK_STRIKES — consecutive remote-up checks before handback
|
||||
# FALLBACK_CHECK_INTERVAL — seconds between checks (default 30)
|
||||
# FALLBACK_HANDBACK_STRIKES — consecutive remote-up checks before handback (default 3 = 90s)
|
||||
# FALLBACK_STATE_FILE — /boot/config path — survives reboots
|
||||
# FALLBACK_RSYNC_ENABLED — gate for writeback rsync jobs
|
||||
# EXTERNAL_IP — IP to ping for internet check (default 8.8.8.8)
|
||||
@@ -332,7 +333,7 @@ remote_ddns_stop() {
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── TIERED FAILOVER HELPERS ───────────────────────────────────────────────────────────────────
|
||||
# ── TIERED FALLBACK HELPERS ───────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# All routing uses MY_ID — not hostname string comparison.
|
||||
# MY_ID set by detect_hosts() — "HOST1" or "HOST2" etc.
|
||||
@@ -436,150 +437,152 @@ fi
|
||||
# ==============================================================================================
|
||||
# ── HANDBACK SEQUENCE ─────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Called when remote returns after FAILOVER state.
|
||||
# Called when remote returns after FALLBACK state.
|
||||
# CRITICAL sequencing — do not reorder without understanding the consequences.
|
||||
# Each step must succeed before proceeding — aborts and retries next cycle on failure.
|
||||
|
||||
run_handback() {
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Handback Sequence — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
warn "$REMOTE_SERVER_NAME has returned — beginning handback"
|
||||
warn "$REMOTE_SERVER_NAME has returned — beginning staged handback"
|
||||
|
||||
# ── Step 1: Pre-flight checks ────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
# Version parity — refuse handback if unRAID versions mismatch
|
||||
if ! check_unraid_version_parity; then
|
||||
warn "Version parity check failed — aborting handback, will retry next cycle"
|
||||
state_set handback_strikes 0
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Remote array must be mounted before rsync
|
||||
if ! check_remote_array; then
|
||||
warn "Remote array not ready — aborting handback, will retry next cycle"
|
||||
state_set handback_strikes 0
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Remote Docker daemon must be responsive before issuing container commands
|
||||
if ! check_remote_docker_daemon; then
|
||||
warn "Remote Docker daemon not ready — aborting handback, will retry next cycle"
|
||||
state_set handback_strikes 0
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Pre-flight checks passed ✅"
|
||||
echo " Pre-flight checks passed ✅"
|
||||
|
||||
# ── Step 2: Stop remote DDNS FIRST ──────────────────────────────────────────────────────
|
||||
# CRITICAL — prevents split-brain DNS while data is being synced
|
||||
local outage_minutes
|
||||
outage_minutes=$(( ($(date +%s) - $(state_get fallback_start)) / 60 ))
|
||||
local tier1_wb_delay t2_delay t3_delay t4_delay
|
||||
tier1_wb_delay=$(get_tier1_writeback_delay)
|
||||
t2_delay=$(get_tier2_delay)
|
||||
t3_delay=$(get_tier3_delay)
|
||||
t4_delay=$(get_tier4_delay)
|
||||
|
||||
warn "Outage duration: ${outage_minutes}min — staged handback Tier 4→3→2→1"
|
||||
|
||||
# ── Staged handback helper ────────────────────────────────────────────────────────────────
|
||||
# Each tier: stop local containers → rsync writeback → start on remote.
|
||||
# Skips tiers that were never activated. Emby (Tier 1) stays on fallback
|
||||
# serving users until all higher tiers complete.
|
||||
handback_tier() {
|
||||
local tier="$1" threshold="$2" label="$3"
|
||||
|
||||
[[ "$(state_get "tier${tier}_started")" != "true" ]] && return 0
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Handback Tier $tier — $label ━━━"
|
||||
|
||||
local containers
|
||||
read -r -a containers <<< "$(get_tier_containers "$tier")"
|
||||
|
||||
echo " Stopping local Tier $tier containers..."
|
||||
for container in "${containers[@]}"; do
|
||||
[[ -n "$container" ]] && local_stop "$container"
|
||||
done
|
||||
|
||||
if ! check_rsync_enabled "FALLBACK"; then
|
||||
warn "FALLBACK_RSYNC_ENABLED=false — skipping Tier $tier writeback"
|
||||
else
|
||||
local jobs
|
||||
read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")"
|
||||
if [[ ${#jobs[@]} -gt 0 && "$outage_minutes" -ge "$threshold" ]]; then
|
||||
echo " Rsync writeback — outage ${outage_minutes}min >= ${threshold}min"
|
||||
for job in "${jobs[@]}"; do
|
||||
[[ -z "$job" ]] && continue
|
||||
log "Syncing: $job"
|
||||
[[ "$DRY_RUN" == false ]] && bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
|
||||
|| warn "DRY RUN — would rsync: $job"
|
||||
done
|
||||
else
|
||||
log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo " Starting Tier $tier on $REMOTE_SERVER_NAME..."
|
||||
for container in "${containers[@]}"; do
|
||||
[[ -n "$container" ]] && remote_start "$container"
|
||||
done
|
||||
|
||||
state_set "tier${tier}_started" "false"
|
||||
warn "$ICON_DONE Tier $tier handed back to $REMOTE_SERVER_NAME"
|
||||
}
|
||||
|
||||
# ── Steps 2–4: Tiers 4→3→2 — Emby stays up throughout ──────────────────────────────────
|
||||
handback_tier 4 "$t4_delay" "media shares + edge cases"
|
||||
handback_tier 3 "$t3_delay" "secondary services"
|
||||
handback_tier 2 "$t2_delay" "productivity services"
|
||||
|
||||
# ── Step 5: DDNS handoff — immediately before Tier 1 goes down ──────────────────────────
|
||||
# Moved here (not at start) so DNS keeps pointing at fallback during higher-tier handbacks.
|
||||
# Prevents split-brain DNS during Tier 1 rsync window.
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET DDNS Handoff ━━━"
|
||||
remote_ddns_stop
|
||||
sleep 5 # brief pause to ensure DDNS stop propagates before rsync
|
||||
sleep 5
|
||||
|
||||
# ── Step 3: Stop remote containers ──────────────────────────────────────────────────────
|
||||
# Clean state before rsync — no dirty writes during transfer window
|
||||
# ── Step 6: Tier 1 handback — vital services + Emby ─────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Stop Remote Containers ━━━"
|
||||
echo "━━━ $ICON_FALLBACK Handback Tier 1 — vital services ━━━"
|
||||
|
||||
local ALL_FALLBACK_CONTAINERS=()
|
||||
|
||||
# Collect all started tiers in reverse order (highest tier stops first)
|
||||
if [[ "$(state_get tier4_started)" == "true" ]]; then
|
||||
read -r -a t4 <<< "$(get_tier_containers 4)"
|
||||
ALL_FALLBACK_CONTAINERS+=("${t4[@]}")
|
||||
fi
|
||||
if [[ "$(state_get tier3_started)" == "true" ]]; then
|
||||
read -r -a t3 <<< "$(get_tier_containers 3)"
|
||||
ALL_FALLBACK_CONTAINERS+=("${t3[@]}")
|
||||
fi
|
||||
if [[ "$(state_get tier2_started)" == "true" ]]; then
|
||||
read -r -a t2 <<< "$(get_tier_containers 2)"
|
||||
ALL_FALLBACK_CONTAINERS+=("${t2[@]}")
|
||||
fi
|
||||
|
||||
# Tier 1 always started — stop last (DDNS already handled above)
|
||||
read -r -a t1 <<< "$(get_tier_containers 1)"
|
||||
|
||||
echo " Stopping local Tier 1 containers..."
|
||||
for container in "${t1[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local is_ddns=false
|
||||
for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||||
[[ "$container" == "$ddns" ]] && is_ddns=true && break
|
||||
done
|
||||
[[ "$is_ddns" == false ]] && ALL_FALLBACK_CONTAINERS+=("$container")
|
||||
[[ "$is_ddns" == false ]] && local_stop "$container"
|
||||
done
|
||||
|
||||
for container in "${ALL_FALLBACK_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local_stop "$container"
|
||||
done
|
||||
|
||||
log "All fallback containers stopped locally"
|
||||
|
||||
# ── Step 4: Rsync writeback ──────────────────────────────────────────────────────────────
|
||||
# Tiered writeback with skip window — short outages do not benefit from writeback.
|
||||
# After a short outage primary's clean nightly state is more reliable than dirty
|
||||
# accumulation — skip writeback entirely for outages under the threshold.
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Rsync Writeback ━━━"
|
||||
|
||||
local outage_minutes
|
||||
outage_minutes=$(( ($(date +%s) - $(state_get fallback_start)) / 60 ))
|
||||
local tier1_wb_delay
|
||||
tier1_wb_delay=$(get_tier1_writeback_delay)
|
||||
|
||||
warn "Outage duration: ${outage_minutes}min"
|
||||
|
||||
if ! check_rsync_enabled "FALLBACK"; then
|
||||
warn "FALLBACK_RSYNC_ENABLED=false — skipping all writeback jobs"
|
||||
warn "Handback will complete without syncing state back to primary"
|
||||
warn "FALLBACK_RSYNC_ENABLED=false — skipping Tier 1 writeback"
|
||||
else
|
||||
run_writeback_tier() {
|
||||
local tier="$1" threshold="$2" label="$3"
|
||||
local jobs
|
||||
read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")"
|
||||
[[ ${#jobs[@]} -eq 0 ]] && return
|
||||
|
||||
if [[ "$outage_minutes" -ge "$threshold" ]]; then
|
||||
warn "Tier $tier writeback ($label) — outage ${outage_minutes}min >= ${threshold}min"
|
||||
for job in "${jobs[@]}"; do
|
||||
[[ -z "$job" ]] && continue
|
||||
log "Syncing: $job"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if [[ "$(basename "$job")" == "Emby" ]]; then
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" --profile=emby-fallback
|
||||
else
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job"
|
||||
fi
|
||||
local t1_jobs
|
||||
read -r -a t1_jobs <<< "$(get_writeback_jobs_for_tier 1)"
|
||||
if [[ ${#t1_jobs[@]} -gt 0 && "$outage_minutes" -ge "$tier1_wb_delay" ]]; then
|
||||
echo " Rsync writeback — outage ${outage_minutes}min >= ${tier1_wb_delay}min"
|
||||
for job in "${t1_jobs[@]}"; do
|
||||
[[ -z "$job" ]] && continue
|
||||
log "Syncing: $job"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if [[ "$(basename "$job")" == "Emby" ]]; then
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" --profile=emby-fallback
|
||||
else
|
||||
warn "DRY RUN — would rsync: $job"
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job"
|
||||
fi
|
||||
done
|
||||
else
|
||||
log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min — primary state is cleaner"
|
||||
fi
|
||||
}
|
||||
|
||||
local t2_delay t3_delay t4_delay
|
||||
t2_delay=$(get_tier2_delay)
|
||||
t3_delay=$(get_tier3_delay)
|
||||
t4_delay=$(get_tier4_delay)
|
||||
|
||||
run_writeback_tier 1 "$tier1_wb_delay" "Emby + auth stack"
|
||||
run_writeback_tier 2 "$t2_delay" "productivity services"
|
||||
run_writeback_tier 3 "$t3_delay" "secondary services"
|
||||
run_writeback_tier 4 "$t4_delay" "media shares + edge cases"
|
||||
else
|
||||
warn "DRY RUN — would rsync: $job"
|
||||
fi
|
||||
done
|
||||
else
|
||||
log "Tier 1 writeback skipped — outage ${outage_minutes}min < ${tier1_wb_delay}min"
|
||||
fi
|
||||
fi
|
||||
|
||||
log "Writeback complete"
|
||||
|
||||
# ── Step 5: Start remote containers ─────────────────────────────────────────────────────
|
||||
# Start in tier order with dependency awareness — databases before apps
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Start Remote Containers ━━━"
|
||||
|
||||
# Tier 1 — excluding DDNS (handled separately as final step)
|
||||
echo " Starting Tier 1 on $REMOTE_SERVER_NAME..."
|
||||
sleep 10 # give databases time to initialise before apps
|
||||
for container in "${t1[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local is_ddns=false
|
||||
@@ -589,36 +592,17 @@ run_handback() {
|
||||
[[ "$is_ddns" == false ]] && remote_start "$container"
|
||||
done
|
||||
|
||||
sleep 10 # give databases time to initialise before apps
|
||||
|
||||
[[ "$(state_get tier2_started)" == "true" ]] && \
|
||||
for container in "${t2[@]}"; do
|
||||
[[ -n "$container" ]] && remote_start "$container"
|
||||
done
|
||||
|
||||
[[ "$(state_get tier3_started)" == "true" ]] && \
|
||||
for container in "${t3[@]}"; do
|
||||
[[ -n "$container" ]] && remote_start "$container"
|
||||
done
|
||||
|
||||
[[ "$(state_get tier4_started)" == "true" ]] && \
|
||||
for container in "${t4[@]}"; do
|
||||
[[ -n "$container" ]] && remote_start "$container"
|
||||
done
|
||||
|
||||
log "Remote containers started"
|
||||
|
||||
# ── Step 6: Start remote DDNS LAST ──────────────────────────────────────────────────────
|
||||
# DNS cuts over ONLY after containers confirmed up — this is the final step
|
||||
# ── Step 7: DNS Cutover — final step ────────────────────────────────────────────────────
|
||||
# DNS cuts over ONLY after Tier 1 containers confirmed up
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET DNS Cutover ━━━"
|
||||
sleep 5 # brief pause to ensure containers are accepting connections
|
||||
sleep 5
|
||||
for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
|
||||
remote_start "$container"
|
||||
done
|
||||
warn "$ICON_NET Remote DDNS started — DNS now points at $REMOTE_SERVER_NAME"
|
||||
|
||||
# ── Step 7: Return to NORMAL ─────────────────────────────────────────────────────────────
|
||||
# ── Step 8: Return to NORMAL ─────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
state_set state "NORMAL"
|
||||
state_set fallback_start "0"
|
||||
@@ -627,6 +611,9 @@ run_handback() {
|
||||
state_set tier3_started "false"
|
||||
state_set tier4_started "false"
|
||||
|
||||
# Restore local DDNS — may have been stopped during NO_INTERNET or DARK state before handback
|
||||
local_ddns_start
|
||||
|
||||
warn "$ICON_DONE Handback complete — returned to NORMAL"
|
||||
notify "Fallback handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" \
|
||||
"Fallback" "normal"
|
||||
@@ -671,7 +658,7 @@ while [[ "$FALLBACK_RUNNING" == true ]]; do
|
||||
log "$ICON_SUCCESS NORMAL — all systems up"
|
||||
|
||||
elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then
|
||||
# Remote is down — enter FAILOVER
|
||||
# Remote is down — enter FALLBACK immediately
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Entering FALLBACK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
warn "$REMOTE_SERVER_NAME ($REMOTE_ID) is unreachable — internet is up — starting fallback"
|
||||
@@ -713,7 +700,7 @@ while [[ "$FALLBACK_RUNNING" == true ]]; do
|
||||
fi
|
||||
|
||||
# ════════════════════════════════════════════════════════════════
|
||||
# FAILOVER STATE
|
||||
# FALLBACK STATE
|
||||
# ════════════════════════════════════════════════════════════════
|
||||
elif [[ "$CURRENT_STATE" == "FALLBACK" ]]; then
|
||||
|
||||
|
||||
Reference in New Issue
Block a user