added write back timing checks correlastes with tiers timing
This commit is contained in:
+81
-21
@@ -49,7 +49,11 @@
|
||||
# 2. Pre-flight checks — remote array started, Docker healthy, rootfs not full
|
||||
# 3. Stop remote DDNS first — prevents split brain DNS during rsync
|
||||
# 4. Stop remote containers — clean state, no dirty writes during rsync
|
||||
# 5. Rsync writeback — full bandwidth, clean source, minimal data
|
||||
# 5. Tiered rsync writeback — skip if outage under threshold (short outages = cleaner to skip)
|
||||
# Tier 1: skip if under HOST1_TIER1_WRITEBACK_DELAY (60min default)
|
||||
# Tier 2: skip if under HOST1_TIER2_DELAY
|
||||
# Tier 3: skip if under HOST1_TIER3_DELAY
|
||||
# Tier 4: skip if under HOST*_TIER4_DELAY — opposing daily sync shares + edge cases
|
||||
# 6. Start local containers — confirmed up before DNS cuts over
|
||||
# 7. Start local DDNS — DNS cuts back ONLY after containers confirmed up
|
||||
# 8. Return to NORMAL
|
||||
@@ -283,11 +287,41 @@ get_tier4_delay() {
|
||||
echo "$HOST1_TIER4_DELAY" || echo "$HOST2_TIER4_DELAY"
|
||||
}
|
||||
|
||||
get_writeback_jobs() {
|
||||
get_tier1_writeback_delay() {
|
||||
[[ "$LOCAL_SERVER_NAME" == "$HOST2" ]] && \
|
||||
echo "$HOST1_TIER1_WRITEBACK_DELAY" || echo "$HOST2_TIER1_WRITEBACK_DELAY"
|
||||
}
|
||||
|
||||
get_writeback_jobs_for_tier() {
|
||||
local tier="$1"
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then
|
||||
echo "${FAILOVER_HOST1_WRITEBACK[@]}"
|
||||
# HOST2 is covering HOST1 — write back HOST1's data
|
||||
case "$tier" in
|
||||
1) echo "${FAILOVER_HOST1_WRITEBACK_TIER1[@]}" ;;
|
||||
2) echo "${FAILOVER_HOST1_WRITEBACK_TIER2[@]}" ;;
|
||||
3) echo "${FAILOVER_HOST1_WRITEBACK_TIER3[@]}" ;;
|
||||
4)
|
||||
# Tier 4 — push HOST1's daily sync shares back (opposing orch list)
|
||||
# These are HOST1's source-of-truth shares that HOST2's arrs managed during outage
|
||||
# Plus any edge case paths defined in FAILOVER_HOST1_WRITEBACK_TIER4
|
||||
echo "${HOST1_DAILY_SYNC_SHARES[@]}"
|
||||
echo "${FAILOVER_HOST1_WRITEBACK_TIER4[@]}"
|
||||
;;
|
||||
esac
|
||||
else
|
||||
echo "${FAILOVER_HOST2_WRITEBACK[@]}"
|
||||
# HOST1 is covering HOST2 — write back HOST2's data
|
||||
case "$tier" in
|
||||
1) echo "${FAILOVER_HOST2_WRITEBACK_TIER1[@]}" ;;
|
||||
2) echo "${FAILOVER_HOST2_WRITEBACK_TIER2[@]}" ;;
|
||||
3) echo "${FAILOVER_HOST2_WRITEBACK_TIER3[@]}" ;;
|
||||
4)
|
||||
# Tier 4 — push HOST2's daily sync shares back (opposing orch list)
|
||||
# These are HOST2's source-of-truth shares that HOST1's arrs managed during outage
|
||||
# Plus any edge case paths defined in FAILOVER_HOST2_WRITEBACK_TIER4
|
||||
echo "${HOST2_DAILY_SYNC_SHARES[@]}"
|
||||
echo "${FAILOVER_HOST2_WRITEBACK_TIER4[@]}"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -416,27 +450,53 @@ run_handback() {
|
||||
success "All failover containers stopped locally"
|
||||
|
||||
# ── Step 4: Rsync writeback ──────────────────────────────────────────────────────────────
|
||||
# Full bandwidth, clean source, minimal data
|
||||
# Only critical appdata synced back — media files and downloads skipped
|
||||
# Tiered writeback with skip window — short outages do not benefit from writeback.
|
||||
# Emby syncs every 30min dirty (live container). Clean sync runs nightly at 2:30am.
|
||||
# After a short outage HOST1's clean nightly state is more reliable than HOST2's
|
||||
# dirty sync accumulation — skip writeback entirely for short outages.
|
||||
#
|
||||
# Tier 1 — skip if under HOST1_TIER1_WRITEBACK_DELAY (default 60min)
|
||||
# Tier 2 — skip if under HOST1_TIER2_DELAY (reused — if Tier 2 never started, skip)
|
||||
# Tier 3 — skip if under HOST1_TIER3_DELAY (reused — same logic)
|
||||
# Tier 4 — always writeback — 18hr+ means meaningful delta accumulated
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Rsync Writeback ━━━"
|
||||
|
||||
local writeback_jobs
|
||||
read -r -a writeback_jobs <<< "$(get_writeback_jobs)"
|
||||
local outage_minutes=$(( ($(date +%s) - $(state_get failover_start)) / 60 ))
|
||||
local tier1_wb_delay
|
||||
tier1_wb_delay=$(get_tier1_writeback_delay)
|
||||
|
||||
if [[ ${#writeback_jobs[@]} -eq 0 ]]; then
|
||||
info "No writeback jobs configured — skipping rsync"
|
||||
else
|
||||
for job in "${writeback_jobs[@]}"; do
|
||||
[[ -z "$job" ]] && continue
|
||||
info "Syncing: $job"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job"
|
||||
else
|
||||
warn "DRY RUN — would rsync: $job"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
info "Outage duration: ${outage_minutes}min"
|
||||
|
||||
run_writeback_tier() {
|
||||
local tier="$1"
|
||||
local threshold="$2"
|
||||
local label="$3"
|
||||
local jobs
|
||||
read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")"
|
||||
[[ ${#jobs[@]} -eq 0 ]] && return
|
||||
|
||||
if [[ "$outage_minutes" -ge "$threshold" ]]; then
|
||||
info "Tier $tier writeback ($label) — outage ${outage_minutes}min >= ${threshold}min"
|
||||
for job in "${jobs[@]}"; do
|
||||
[[ -z "$job" ]] && continue
|
||||
info "Syncing: $job"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job"
|
||||
else
|
||||
warn "DRY RUN — would rsync: $job"
|
||||
fi
|
||||
done
|
||||
else
|
||||
info "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min threshold"
|
||||
info "Primary has cleaner state — no writeback needed"
|
||||
fi
|
||||
}
|
||||
|
||||
run_writeback_tier 1 "$tier1_wb_delay" "Emby + auth stack"
|
||||
run_writeback_tier 2 "$HOST1_TIER2_DELAY" "NextCloud + Immich"
|
||||
run_writeback_tier 3 "$HOST1_TIER3_DELAY" "secondary services"
|
||||
run_writeback_tier 4 "$(get_tier4_delay)" "media shares + edge cases"
|
||||
|
||||
success "Writeback complete"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user