1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Fallback ===================================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Mutual container fallback between two unRAID servers. Runs continuously as a
9	# background process — started at array start by array_started.sh. Each server
10	# runs this script independently with no coordination between servers.
11	#
12	# All decisions are based solely on two pings per cycle: remote Tailscale IP
13	# reachable + internet reachable. No SSH signaling, no shared state, no election
14	# algorithm. Each server acts entirely from its own network perspective.
15	#
16	# Never requires human intervention during normal fallback and handback.
17	# Stop only via `fallback.sh --stop` — do NOT kill directly (state file may corrupt).
18	#
19	# ==============================================================================================
20	# OPERATIONAL MODEL
21	# ==============================================================================================
22	#
23	# Each cycle (every FALLBACK_CHECK_INTERVAL seconds):
24	#   1. Ping remote Tailscale IP — is the other server reachable?
25	#   2. Ping internet (EXTERNAL_IP) — do I have internet?
26	#   3. Determine state from the two results
27	#   4. Take the action for that state
28	#
29	# States:
30	#   NORMAL       — remote up, internet up — own containers, own DDNS on, silent
31	#   FALLBACK     — remote down, internet up — start tier containers, DDNS over
32	#   NO_INTERNET  — internet down — stop own DDNS immediately, wait
33	#   DARK         — remote down AND internet down — same actions as NO_INTERNET
34	#
35	# FALLBACK escalation (tiered by outage duration):
36	#   Tier 1 — immediate — vital services + Live TV (cannot wait)
37	#   Tier 2 — after HOST*_TIER2_DELAY (default 4hr) — shared productivity
38	#   Tier 3 — after HOST*_TIER3_DELAY (default 12hr) — secondary services
39	#   Tier 4 — after HOST*_TIER4_DELAY (default 24hr) — arrs + downloaders
40	#
41	# Handback (when remote returns after FALLBACK):
42	#   1. Strike confirmation  — FALLBACK_HANDBACK_STRIKES consecutive remote-up checks
43	#   2. Pre-flight checks    — version parity, remote array, remote Docker daemon
44	#   3. Staged reverse: Tier 4→3→2 — Emby stays on covering server throughout
45	#      Each tier: stop local → rsync writeback → start on remote
46	#   4. DDNS handoff         — stop remote DDNS before Tier 1 goes down
47	#   5. Tier 1 handback      — stop local vital services, rsync writeback, start remote
48	#   6. Start remote DDNS    — DNS cuts back ONLY after containers confirmed running
49	#   7. Return to NORMAL
50	#
51	# ==============================================================================================
52	# DESIGN PRINCIPLES
53	# ==============================================================================================
54	#
55	# Pure Ping-Only Detection
56	#   No SSH signaling, no election algorithm, no shared state between servers.
57	#   Each server makes all decisions from its own vantage point. A server that
58	#   cannot reach the remote and has internet acts — no coordination needed.
59	#
60	# DDNS Absolute Control
61	#   This script is the sole authority over DDNS containers. Network state
62	#   returning is not permission to start DDNS — only the completion of the
63	#   full handback sequence grants that. Eliminates the split-brain DNS window
64	#   where two servers update the same domain with different IPs simultaneously.
65	#
66	# Staged Reverse-Tier Handback
67	#   Emby stays on the covering server serving users while Tiers 4→3→2 hand
68	#   back. The user impact window (Emby down) is only the Tier 1 rsync
69	#   duration — typically minutes. Users are served continuously until the
70	#   final step.
71	#
72	# Strike Confirmation
73	#   Handback requires FALLBACK_HANDBACK_STRIKES consecutive remote-up checks.
74	#   Brief network recovery during an ongoing outage would otherwise trigger a
75	#   failed handback: containers stop on the covering server, remote goes down
76	#   again mid-rsync. Strikes prevent this.
77	#
78	# MY_ID-Based Routing
79	#   All tier arrays, DDNS lists, and writeback paths are selected via MY_ID
80	#   set by detect_hosts() — not hostname string comparison. The same script
81	#   and same config file handle both directions symmetrically.
82	#
83	# ==============================================================================================
84	# OPERATIONAL SAFEGUARDS
85	# ==============================================================================================
86	#
87	# Root Enforcement
88	#   Docker and SSH operations require root.
89	#
90	# FALLBACK_ENABLED Gate
91	#   Exits cleanly when disabled — safe to run on servers being rebuilt without
92	#   triggering spurious fallback actions.
93	#
94	# Version Parity Check
95	#   Refuses handback if remote unRAID version doesn't match. A mismatch may
96	#   mean the remote is not fully ready or on an incompatible version.
97	#
98	# Remote Docker Daemon Check
99	#   SSH-checks the remote Docker daemon is responding before any remote commands.
100	#   Reachable on Tailscale does not mean Docker is ready.
101	#
102	# Container Verification
103	#   After every local_start(), verifies the container came up and stayed running.
104	#   Catches crashes-on-start before declaring fallback successful.
105	#
106	# Timeout Protection
107	#   All docker commands (DOCKER_TIMEOUT=15s) and SSH commands (SSH_TIMEOUT=10s)
108	#   wrapped in timeouts. A hung daemon does not block the state machine.
109	#
110	# Single Instance Lock
111	#   acquire_lock() prevents two instances running simultaneously. Both would
112	#   modify containers and DDNS independently — conflict is certain.
113	#
114	# Host Detection
115	#   detect_hosts() resolves MY_ID / REMOTE_ID from master.conf at startup.
116	#   Exits if the host cannot be identified — prevents running on an unknown machine.
117	#
118	# Partnership Gate
119	#   When FALLBACK_PARTNERSHIP_REQUIRED=true, fallback is only permitted when the
120	#   local partnership DB reports an active partnership. If partnership goes inactive,
121	#   a grace timer starts. After FALLBACK_PARTNERSHIP_SUSPEND_AFTER minutes the loop
122	#   suspends: no new FALLBACK is entered, and any live fallback containers are
123	#   stopped cleanly. Resumes automatically when partnership becomes active again.
124	#   Set FALLBACK_PARTNERSHIP_REQUIRED=false to run fallback without a partnership.
125	#
126	# Silent by Default
127	#   State transitions: warn() — always visible. Healthy routine cycles: log()
128	#   — suppressed unless --log. Produces no output on clean cycles.
129	#
130	# ==============================================================================================
131	# STATE FILES
132	# ==============================================================================================
133	#
134	#   FALLBACK_STATE_FILE   — /boot/config/fallback_state.db (survives reboots)
135	#       Keys: state, fallback_start, handback_strikes, tier2_started,
136	#       tier3_started, tier4_started, partnership_suspended, partner_lost_at.
137	#       Lives on /boot/ intentionally — if the server was in FALLBACK when it
138	#       rebooted, it resumes FALLBACK on restart.
139	#
140	# ==============================================================================================
141	# CONFIGURATION
142	# ==============================================================================================
143	#
144	# host*.conf
145	#
146	#   HOST*_DDNS_CONTAINERS
147	#       DDNS containers this host manages — stopped on internet loss, started
148	#       as the last step of handback
149	#
150	#   FALLBACK_HOST*_STOP_ON_NO_NET
151	#       Containers stopped when this host loses internet
152	#
153	#   FALLBACK_HOST*_COVERS_HOST*_TIER1–4
154	#       Containers this host starts for the remote when remote is down, by tier.
155	#       Variable pattern: FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${N}
156	#
157	#   HOST*_TIER2_DELAY / TIER3_DELAY / TIER4_DELAY
158	#       Minutes after FALLBACK entry before activating each tier.
159	#       References the remote host's ID: HOST1_TIER2_DELAY for HOST1 outage.
160	#       (defaults: 240 / 720 / 1440)
161	#
162	#   HOST*_TIER1_WRITEBACK_DELAY
163	#       Minimum outage minutes before Tier 1 writeback runs. Skip for short
164	#       outages where primary state is cleaner. (default: 60)
165	#
166	#   FALLBACK_HOST*_WRITEBACK_TIER1–4
167	#       Paths rsynced back to remote on handback, per tier.
168	#       Tier 4 auto-uses HOST*_DAILY_SYNC_SHARES — no separate list needed.
169	#       Variable pattern: FALLBACK_${REMOTE_ID}_WRITEBACK_TIER${N}
170	#
171	# master.conf
172	#
173	#   FALLBACK_ENABLED
174	#       false = exit cleanly (e.g. remote server being rebuilt). (default: false)
175	#
176	#   FALLBACK_PARTNERSHIP_REQUIRED
177	#       true = fallback is gated on an active partnership — suspends when no active
178	#       partner is found after FALLBACK_PARTNERSHIP_SUSPEND_AFTER minutes.
179	#       false = fallback runs standalone regardless of partnership state. (default: true)
180	#
181	#   FALLBACK_PARTNERSHIP_SUSPEND_AFTER
182	#       Minutes without an active partnership before fallback suspends itself.
183	#       Grace period: fallback continues normally during this window.
184	#       0 = suspend immediately when partnership goes inactive. (default: 120)
185	#
186	#   FALLBACK_CHECK_INTERVAL
187	#       Seconds between connectivity checks. (default: 30)
188	#
189	#   FALLBACK_HANDBACK_STRIKES
190	#       Consecutive remote-up checks required before handback begins. (default: 3)
191	#
192	#   FALLBACK_STATE_FILE
193	#       State file path — /boot/config/fallback_state.db — survives reboots.
194	#
195	#   FALLBACK_RSYNC_ENABLED
196	#       Gate for writeback rsync jobs during handback. (default: true)
197	#
198	#   EXTERNAL_IP
199	#       IP pinged for internet check. (default: 8.8.8.8)
200	#
201	# ==============================================================================================
202	# RUNTIME MODES
203	# ==============================================================================================
204	#
205	# fallback.sh
206	#     Normal start — continuous loop. Started by array_started.sh at array start.
207	#
208	# fallback.sh --stop
209	#     Gracefully stop the running instance (SIGTERM → wait 10s → SIGKILL).
210	#     Used by array_stopping.sh during array shutdown.
211	#
212	# fallback.sh --dry-run
213	#     Preview state changes without starting or stopping containers or DDNS.
214	#
215	# fallback.sh --status
216	#     Show current state, identity, DDNS containers, outage duration if in
217	#     FALLBACK, tier flags, strike counter. Then exit.
218	#
219	# fallback.sh --log
220	#     Verbose output on every decision in every cycle.
221	#
222	# To stop: use `fallback.sh --stop`. Do NOT kill -9 — state file may corrupt if mid-write.
223	#
224	# ==============================================================================================
225
226	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
227
228	source "$SCRIPT_DIR/../load_config.sh"
229
230	parse_args "$@"
231
232	# ── Stop mode — runs before acquire_lock so we can target the holding instance ────────────────
233	if [[ " ${PARSED_ARGS[*]:-} " == *" --stop "* ]]; then
234	    LOCKFILE="${LOCK_DIR}/fallback.lock"
235	    if [[ ! -f "$LOCKFILE" ]]; then
236	        log "No fallback.sh lock found — not running"
237	        exit 0
238	    fi
239	    lock_content=$(cat "$LOCKFILE" 2>/dev/null)
240	    target_pid="${lock_content%%:*}"
241	    if [[ -z "$target_pid" ]] || ! kill -0 "$target_pid" 2>/dev/null; then
242	        warn "Stale lock — fallback.sh not running (PID $target_pid gone) — clearing"
243	        rm -f "$LOCKFILE"
244	        exit 0
245	    fi
246	    warn "Stopping fallback.sh (PID $target_pid)..."
247	    kill -TERM "$target_pid" 2>/dev/null || true
248	    waited=0
249	    while kill -0 "$target_pid" 2>/dev/null && [[ "$waited" -lt 10 ]]; do
250	        sleep 1
251	        (( waited++ ))
252	    done
253	    if kill -0 "$target_pid" 2>/dev/null; then
254	        warn "SIGTERM ignored — sending SIGKILL to fallback.sh (PID $target_pid)"
255	        kill -KILL "$target_pid" 2>/dev/null || true
256	        sleep 2
257	        if kill -0 "$target_pid" 2>/dev/null; then
258	            error "Failed to kill fallback.sh (PID $target_pid)"
259	            exit 1
260	        fi
261	        warn "Force-stopped: fallback.sh (PID $target_pid) ✅"
262	    else
263	        warn "Stopped: fallback.sh (PID $target_pid) ✅"
264	    fi
265	    exit 0
266	fi
267
268	# ==============================================================================================
269	# ━━━ Setup ━━━
270	# ==============================================================================================
271
272	if [[ "$EUID" -ne 0 ]]; then
273	    error "Must be run as root"
274	    exit 1
275	fi
276
277	# FALLBACK_ENABLED gate — exits cleanly when disabled (e.g. HOST2 being rebuilt)
278	if [[ "${FALLBACK_ENABLED:-false}" == false ]]; then
279	    warn "FALLBACK_ENABLED=false — fallback monitoring disabled"
280	    warn "Set FALLBACK_ENABLED=true in master.conf when both servers are ready"
281	    exit 0
282	fi
283
284	acquire_lock   # strict single instance — notifies on collision
285
286	if ! command -v docker &>/dev/null; then
287	    error "Docker command not found"
288	    exit 1
289	fi
290
291	detect_hosts
292	resolve_remote_ip
293
294	# Validate unRAID notify script — used throughout for state change notifications
295
296	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no container or DDNS changes will be made"
297
298	# Timeout for all docker and SSH docker commands
299	DOCKER_TIMEOUT=15
300	SSH_TIMEOUT=10
301	CONTAINER_VERIFY_WAIT=5   # seconds after start before verifying container is up
302
303	log "$ICON_GEAR Config: check-interval=${FALLBACK_CHECK_INTERVAL}s handback-strikes=${FALLBACK_HANDBACK_STRIKES} partnership-required=${FALLBACK_PARTNERSHIP_REQUIRED:-true} rsync-enabled=${FALLBACK_RSYNC_ENABLED:-true}"
304	log "$ICON_GEAR Timeouts: docker=${DOCKER_TIMEOUT}s ssh=${SSH_TIMEOUT}s verify-wait=${CONTAINER_VERIFY_WAIT}s"
305
306	# ==============================================================================================
307	# ── STATE FILE HELPERS ────────────────────────────────────────────────────────────────────────
308	# ==============================================================================================
309	# State file on /boot/config — survives reboots.
310	# Format: key=value one per line.
311	# Keys: state, fallback_start, handback_strikes, tier2_started, tier3_started, tier4_started
312
313	state_get() {
314	    grep "^${1}=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2
315	}
316
317	state_set() {
318	    local key="$1" value="$2"
319	    if grep -q "^${key}=" "$FALLBACK_STATE_FILE" 2>/dev/null; then
320	        # Escape sed replacement metacharacters: \ first, then & and |
321	        local safe="${value//\\/\\\\}"; safe="${safe//&/\\&}"; safe="${safe//|/\\|}"
322	        sed -i "s|^${key}=.*|${key}=${safe}|" "$FALLBACK_STATE_FILE"
323	    else
324	        echo "${key}=${value}" >> "$FALLBACK_STATE_FILE"
325	    fi
326	}
327
328	state_init() {
329	    mkdir -p "$(dirname "$FALLBACK_STATE_FILE")"
330	    [[ ! -f "$FALLBACK_STATE_FILE" ]] && touch "$FALLBACK_STATE_FILE"
331	    [[ -z "$(state_get state)" ]]                  && state_set state "NORMAL"
332	    [[ -z "$(state_get fallback_start)" ]]         && state_set fallback_start "0"
333	    [[ -z "$(state_get handback_strikes)" ]]       && state_set handback_strikes "0"
334	    [[ -z "$(state_get tier2_started)" ]]          && state_set tier2_started "false"
335	    [[ -z "$(state_get tier3_started)" ]]          && state_set tier3_started "false"
336	    [[ -z "$(state_get tier4_started)" ]]          && state_set tier4_started "false"
337	    [[ -z "$(state_get partnership_suspended)" ]]  && state_set partnership_suspended "false"
338	    [[ -z "$(state_get partner_lost_at)" ]]        && state_set partner_lost_at "0"
339	}
340
341	# ==============================================================================================
342	# ── PARTNERSHIP GATE HELPERS ──────────────────────────────────────────────────────────────────
343	# ==============================================================================================
344
345	# Returns 0 (true) if the local partnership DB reports an ACTIVE partnership.
346	# Reads /boot/config/partnership_<hostname>.db — no subprocess, no SSH.
347	check_partnership_active() {
348	    local state_file="${STATE_DIR}/partnership_${LOCAL_SERVER_NAME}.db"
349	    local state
350	    state=$(grep "^state=" "$state_file" 2>/dev/null | cut -d= -f2)
351	    [[ "$state" == "ACTIVE" ]]
352	}
353
354	# Stop all tiers that were started during a fallback and clear their state flags.
355	# Called when the partnership gate suspends an in-progress fallback.
356	_abort_fallback_containers() {
357	    for tier in 4 3 2 1; do
358	        [[ "$(state_get "tier${tier}_started")" != "true" ]] && continue
359	        warn "Aborting Tier $tier containers — partnership suspended"
360	        local containers
361	        read -r -a containers <<< "$(get_tier_containers "$tier")"
362	        for container in "${containers[@]}"; do
363	            [[ -n "$container" ]] && local_stop "$container"
364	        done
365	        state_set "tier${tier}_started" "false"
366	    done
367	}
368
369	# ==============================================================================================
370	# ── CONTAINER HELPERS ─────────────────────────────────────────────────────────────────────────
371	# ==============================================================================================
372	# All docker commands wrapped in DOCKER_TIMEOUT.
373	# All SSH docker commands wrapped in SSH_TIMEOUT.
374	# Verification after start — fallback is critical, confirm containers came up.
375
376	# Start a container locally and verify it came up
377	local_start() {
378	    local container="$1"
379	    [[ -z "$container" ]] && return
380	    local status
381	    status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
382	        "$container" 2>/dev/null)
383	    if [[ "$status" == "true" ]]; then
384	        log "$container already running locally"
385	        return 0
386	    fi
387	    log "$ICON_START Starting $container locally..."
388	    if [[ "$DRY_RUN" == true ]]; then
389	        warn "DRY RUN — would start $container locally"
390	        return 0
391	    fi
392	    if timeout "$DOCKER_TIMEOUT" docker start "$container" >/dev/null 2>&1; then
393	        sleep "$CONTAINER_VERIFY_WAIT"
394	        local post_status
395	        post_status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
396	            "$container" 2>/dev/null)
397	        if [[ "$post_status" == "true" ]]; then
398	            log "$ICON_STARTED $container started and running ✅"
399	            return 0
400	        else
401	            error "$container started but crashed immediately"
402	            notify "$container failed to stay running during fallback on $(hostname)" \
403	                "Fallback" "warning"
404	            return 1
405	        fi
406	    else
407	        error "Failed to start $container locally"
408	        notify "Failed to start $container locally during fallback on $(hostname)" \
409	            "Fallback" "warning"
410	        return 1
411	    fi
412	}
413
414	# Stop a container locally
415	local_stop() {
416	    local container="$1"
417	    [[ -z "$container" ]] && return
418	    local status
419	    status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
420	        "$container" 2>/dev/null)
421	    if [[ "$status" != "true" ]]; then
422	        log "$container already stopped locally"
423	        return 0
424	    fi
425	    log "$ICON_STOP Stopping $container locally..."
426	    if [[ "$DRY_RUN" == true ]]; then
427	        warn "DRY RUN — would stop $container locally"
428	        return 0
429	    fi
430	    timeout "$DOCKER_TIMEOUT" docker stop "$container" >/dev/null 2>&1 && \
431	        log "$ICON_STOPPED $container stopped" || \
432	        error "Failed to stop $container locally"
433	}
434
435	# Start a container on remote via SSH and verify it came up
436	remote_start() {
437	    local container="$1"
438	    [[ -z "$container" ]] && return
439	    local status
440	    status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
441	        root@"$REMOTE_SERVER" \
442	        "timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
443	        $container 2>/dev/null" 2>/dev/null)
444	    if [[ "$status" == "true" ]]; then
445	        log "$container already running on $REMOTE_SERVER_NAME"
446	        return 0
447	    fi
448	    log "$ICON_START Starting $container on $REMOTE_SERVER_NAME..."
449	    if [[ "$DRY_RUN" == true ]]; then
450	        warn "DRY RUN — would start $container on $REMOTE_SERVER_NAME"
451	        return 0
452	    fi
453	    if timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
454	        root@"$REMOTE_SERVER" \
455	        "timeout $DOCKER_TIMEOUT docker start $container" >/dev/null 2>&1; then
456	        sleep "$CONTAINER_VERIFY_WAIT"
457	        local post_status
458	        post_status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
459	            -o ConnectTimeout="$SSH_TIMEOUT" root@"$REMOTE_SERVER" \
460	            "timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
461	            $container 2>/dev/null" 2>/dev/null)
462	        if [[ "$post_status" == "true" ]]; then
463	            log "$ICON_STARTED $container started on $REMOTE_SERVER_NAME ✅"
464	            return 0
465	        else
466	            error "$container started on $REMOTE_SERVER_NAME but crashed immediately"
467	            notify "$container failed after start on $REMOTE_SERVER_NAME during handback on $(hostname)" \
468	                "Fallback" "warning"
469	            return 1
470	        fi
471	    else
472	        error "Failed to start $container on $REMOTE_SERVER_NAME"
473	        notify "Failed to start $container on $REMOTE_SERVER_NAME during handback on $(hostname)" \
474	            "Fallback" "warning"
475	        return 1
476	    fi
477	}
478
479	# Stop a container on remote via SSH
480	remote_stop() {
481	    local container="$1"
482	    [[ -z "$container" ]] && return
483	    local status
484	    status=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
485	        root@"$REMOTE_SERVER" \
486	        "timeout $DOCKER_TIMEOUT docker inspect -f '{{.State.Running}}' \
487	        $container 2>/dev/null" 2>/dev/null)
488	    if [[ "$status" != "true" ]]; then
489	        log "$container already stopped on $REMOTE_SERVER_NAME"
490	        return 0
491	    fi
492	    log "$ICON_STOP Stopping $container on $REMOTE_SERVER_NAME..."
493	    if [[ "$DRY_RUN" == true ]]; then
494	        warn "DRY RUN — would stop $container on $REMOTE_SERVER_NAME"
495	        return 0
496	    fi
497	    timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" -o ConnectTimeout="$SSH_TIMEOUT" \
498	        root@"$REMOTE_SERVER" \
499	        "timeout $DOCKER_TIMEOUT docker stop $container" >/dev/null 2>&1 && \
500	        log "$ICON_STOPPED $container stopped on $REMOTE_SERVER_NAME" || \
501	        error "Failed to stop $container on $REMOTE_SERVER_NAME"
502	}
503
504	# ==============================================================================================
505	# ── DDNS HELPERS ──────────────────────────────────────────────────────────────────────────────
506	# ==============================================================================================
507	# DDNS managed exclusively by this script — never by network state alone.
508	# Stopping DDNS on internet loss prevents split-brain DNS updates.
509
510	local_ddns_stop() {
511	    warn "$ICON_NET Stopping local DDNS — internet lost"
512	    for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do
513	        local_stop "$container"
514	    done
515	}
516
517	local_ddns_start() {
518	    warn "$ICON_NET Starting local DDNS — handback complete, containers confirmed up"
519	    for container in "${LOCAL_DDNS_CONTAINERS[@]}"; do
520	        local_start "$container"
521	    done
522	}
523
524	remote_ddns_stop() {
525	    warn "$ICON_NET Stopping remote DDNS on $REMOTE_SERVER_NAME — prevents split-brain during rsync"
526	    for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
527	        remote_stop "$container"
528	    done
529	}
530
531	# ==============================================================================================
532	# ── TIERED FALLBACK HELPERS ───────────────────────────────────────────────────────────────────
533	# ==============================================================================================
534	# All routing uses MY_ID — not hostname string comparison.
535	# MY_ID set by detect_hosts() — "HOST1" or "HOST2" etc.
536	# "This server covers the other" — MY_ID selects which tier arrays to use.
537
538	get_tier_containers() {
539	    local tier="$1"
540	    local remote_id="${REMOTE_ID}"
541	    local var_name="FALLBACK_${MY_ID}_COVERS_${remote_id}_TIER${tier}"
542	    eval "echo \"\${${var_name}[@]:-}\""
543	}
544
545	get_tier2_delay() {
546	    local var_name="${REMOTE_ID}_TIER2_DELAY"
547	    echo "${!var_name:-240}"
548	}
549
550	get_tier3_delay() {
551	    local var_name="${REMOTE_ID}_TIER3_DELAY"
552	    echo "${!var_name:-720}"
553	}
554
555	get_tier4_delay() {
556	    local var_name="${REMOTE_ID}_TIER4_DELAY"
557	    echo "${!var_name:-1440}"
558	}
559
560	get_tier1_writeback_delay() {
561	    local var_name="${REMOTE_ID}_TIER1_WRITEBACK_DELAY"
562	    echo "${!var_name:-60}"
563	}
564
565	get_writeback_jobs_for_tier() {
566	    local tier="$1"
567	    local remote_id="${REMOTE_ID}"
568	    if [[ "$tier" -eq 4 ]]; then
569	        # Tier 4 — daily sync shares of the remote + any extra writeback paths
570	        local shares_var="${remote_id}_DAILY_SYNC_SHARES"
571	        local extra_var="FALLBACK_${remote_id}_WRITEBACK_TIER4"
572	        eval "echo \"\${${shares_var}[@]:-} \${${extra_var}[@]:-}\""
573	    else
574	        local var_name="FALLBACK_${remote_id}_WRITEBACK_TIER${tier}"
575	        eval "echo \"\${${var_name}[@]:-}\""
576	    fi
577	}
578
579	# Select DDNS arrays based on MY_ID
580	set_ddns_arrays() {
581	    local local_ddns_var="${MY_ID}_DDNS_CONTAINERS"
582	    local remote_ddns_var="${REMOTE_ID}_DDNS_CONTAINERS"
583	    eval "LOCAL_DDNS_CONTAINERS=(\"\${${local_ddns_var}[@]:-}\")"
584	    eval "REMOTE_DDNS_CONTAINERS=(\"\${${remote_ddns_var}[@]:-}\")"
585	}
586
587	# Select internet-loss stop list based on MY_ID
588	get_stop_on_no_net() {
589	    local var_name="FALLBACK_${MY_ID}_STOP_ON_NO_NET"
590	    eval "echo \"\${${var_name}[@]:-}\""
591	}
592
593	# ==============================================================================================
594	# ━━━ Status ━━━
595	# ==============================================================================================
596	if [[ "$SHOW_STATUS" == true ]]; then
597	    state_init
598	    set_ddns_arrays
599	    CURRENT_STATE=$(state_get state)
600	    FALLBACK_START_TS=$(state_get fallback_start)
601	    TIER2=$(state_get tier2_started)
602	    TIER3=$(state_get tier3_started)
603	    TIER4=$(state_get tier4_started)
604	    STRIKES=$(state_get handback_strikes)
605
606	    local_ver=$(platform_get_os_version 2>/dev/null || echo "unknown")
607
608	    echo ""
609	    echo "━━━━━ $ICON_SUMMARY FALLBACK STATUS ━━━━━"
610	    echo "$ICON_HOST    My ID:       $MY_ID ($LOCAL_SERVER_NAME)"
611	    echo "$ICON_HOST    Remote ID:   $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
612	    echo "$ICON_GEAR    OS ver:      $local_ver"
613	    echo "$ICON_FALLBACK State:      $CURRENT_STATE"
614	    echo "$ICON_NET     Local DDNS:  ${LOCAL_DDNS_CONTAINERS[*]:-none}"
615	    echo "$ICON_NET     Remote DDNS: ${REMOTE_DDNS_CONTAINERS[*]:-none}"
616	    echo "$ICON_TIME    Interval:    ${FALLBACK_CHECK_INTERVAL}s"
617	    echo "$ICON_FALLBACK Strikes:    $STRIKES / $FALLBACK_HANDBACK_STRIKES"
618
619	    if [[ "$CURRENT_STATE" == "FALLBACK" && "$FALLBACK_START_TS" -gt 0 ]]; then
620	        ELAPSED=$(( ($(date +%s) - FALLBACK_START_TS) / 60 ))
621	        echo "$ICON_TIME    Outage:      ${ELAPSED}min"
622	        echo "$ICON_FALLBACK Tier 2:     $TIER2 (delay: $(get_tier2_delay)min)"
623	        echo "$ICON_FALLBACK Tier 3:     $TIER3 (delay: $(get_tier3_delay)min)"
624	        echo "$ICON_FALLBACK Tier 4:     $TIER4 (delay: $(get_tier4_delay)min)"
625	    fi
626
627	    echo "$ICON_GEAR    Enabled:     $FALLBACK_ENABLED"
628	    echo "$ICON_GEAR    Dry Run:     $DRY_RUN"
629
630	    if [[ "${FALLBACK_PARTNERSHIP_REQUIRED:-true}" == "true" ]]; then
631	        SUSPENDED=$(state_get partnership_suspended)
632	        PLOST=$(state_get partner_lost_at)
633	        if [[ "$SUSPENDED" == "true" ]]; then
634	            echo "$ICON_GEAR    Partnership: SUSPENDED (no active partnership)"
635	        elif [[ -n "$PLOST" && "$PLOST" != "0" ]]; then
636	            PLOST_MIN=$(( ($(date +%s) - PLOST) / 60 ))
637	            echo "$ICON_GEAR    Partnership: GRACE (${PLOST_MIN}min / ${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}min until suspend)"
638	        else
639	            echo "$ICON_GEAR    Partnership: ACTIVE"
640	        fi
641	    else
642	        echo "$ICON_GEAR    Partnership: REQUIRED=false (standalone mode)"
643	    fi
644
645	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
646	    exit 0
647	fi
648
649	# ==============================================================================================
650	# ── HANDBACK SEQUENCE ─────────────────────────────────────────────────────────────────────────
651	# ==============================================================================================
652	# Called when remote returns after FALLBACK state.
653	# CRITICAL sequencing — do not reorder without understanding the consequences.
654	# Each step must succeed before proceeding — aborts and retries next cycle on failure.
655
656	run_handback() {
657	    echo ""
658	    echo "━━━ $ICON_FALLBACK Handback Sequence — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
659	    warn "$REMOTE_SERVER_NAME has returned — beginning staged handback"
660
661	    # ── Step 1: Pre-flight checks ────────────────────────────────────────────────────────────
662	    echo ""
663	    echo "━━━ $ICON_SHIELD Pre-flight ━━━"
664
665	    if ! check_os_version_parity; then
666	        warn "Version parity check failed — aborting handback, will retry next cycle"
667	        state_set handback_strikes 0
668	        return 1
669	    fi
670
671	    if ! check_remote_array; then
672	        warn "Remote array not ready — aborting handback, will retry next cycle"
673	        state_set handback_strikes 0
674	        return 1
675	    fi
676
677	    if ! check_remote_docker_daemon; then
678	        warn "Remote Docker daemon not ready — aborting handback, will retry next cycle"
679	        state_set handback_strikes 0
680	        return 1
681	    fi
682
683	    echo "  Pre-flight checks passed ✅"
684
685	    local outage_minutes
686	    outage_minutes=$(( ($(date +%s) - $(state_get fallback_start)) / 60 ))
687	    local tier1_wb_delay t2_delay t3_delay t4_delay
688	    tier1_wb_delay=$(get_tier1_writeback_delay)
689	    t2_delay=$(get_tier2_delay)
690	    t3_delay=$(get_tier3_delay)
691	    t4_delay=$(get_tier4_delay)
692
693	    warn "Outage duration: ${outage_minutes}min — staged handback Tier 4→3→2→1"
694
695	    # ── Staged handback helper ────────────────────────────────────────────────────────────────
696	    # Each tier: stop local containers → rsync writeback → start on remote.
697	    # Skips tiers that were never activated. Emby (Tier 1) stays on fallback
698	    # serving users until all higher tiers complete.
699	    handback_tier() {
700	        local tier="$1" threshold="$2" label="$3"
701
702	        [[ "$(state_get "tier${tier}_started")" != "true" ]] && return 0
703
704	        echo ""
705	        echo "━━━ $ICON_FALLBACK Handback Tier $tier — $label ━━━"
706
707	        local containers
708	        read -r -a containers <<< "$(get_tier_containers "$tier")"
709
710	        echo "  Stopping local Tier $tier containers..."
711	        for container in "${containers[@]}"; do
712	            [[ -n "$container" ]] && local_stop "$container"
713	        done
714
715	        if ! check_rsync_enabled "FALLBACK"; then
716	            warn "FALLBACK_RSYNC_ENABLED=false — skipping Tier $tier writeback"
717	        else
718	            local jobs
719	            read -r -a jobs <<< "$(get_writeback_jobs_for_tier "$tier")"
720	            if [[ ${#jobs[@]} -gt 0 && "$outage_minutes" -ge "$threshold" ]]; then
721	                echo "  Rsync writeback — outage ${outage_minutes}min >= ${threshold}min"
722	                for job in "${jobs[@]}"; do
723	                    [[ -z "$job" ]] && continue
724	                    log "Syncing: $job"
725	                    [[ "$DRY_RUN" == false ]] && bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" \
726	                        || warn "DRY RUN — would rsync: $job"
727	                done
728	            else
729	                log "Tier $tier writeback skipped — outage ${outage_minutes}min < ${threshold}min"
730	            fi
731	        fi
732
733	        echo "  Starting Tier $tier on $REMOTE_SERVER_NAME..."
734	        for container in "${containers[@]}"; do
735	            [[ -n "$container" ]] && remote_start "$container"
736	        done
737
738	        state_set "tier${tier}_started" "false"
739	        warn "$ICON_DONE Tier $tier handed back to $REMOTE_SERVER_NAME"
740	    }
741
742	    # ── Steps 2–4: Tiers 4→3→2 — Emby stays up throughout ──────────────────────────────────
743	    handback_tier 4 "$t4_delay" "media shares + edge cases"
744	    handback_tier 3 "$t3_delay" "secondary services"
745	    handback_tier 2 "$t2_delay" "productivity services"
746
747	    # ── Step 5: DDNS handoff — immediately before Tier 1 goes down ──────────────────────────
748	    # Moved here (not at start) so DNS keeps pointing at fallback during higher-tier handbacks.
749	    # Prevents split-brain DNS during Tier 1 rsync window.
750	    echo ""
751	    echo "━━━ $ICON_NET DDNS Handoff ━━━"
752	    remote_ddns_stop
753	    sleep 5
754
755	    # ── Step 6: Tier 1 handback — vital services + Emby ─────────────────────────────────────
756	    echo ""
757	    echo "━━━ $ICON_FALLBACK Handback Tier 1 — vital services ━━━"
758
759	    read -r -a t1 <<< "$(get_tier_containers 1)"
760
761	    echo "  Stopping local Tier 1 containers..."
762	    for container in "${t1[@]}"; do
763	        [[ -z "$container" ]] && continue
764	        local is_ddns=false
765	        for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
766	            [[ "$container" == "$ddns" ]] && is_ddns=true && break
767	        done
768	        [[ "$is_ddns" == false ]] && local_stop "$container"
769	    done
770
771	    if ! check_rsync_enabled "FALLBACK"; then
772	        warn "FALLBACK_RSYNC_ENABLED=false — skipping Tier 1 writeback"
773	    else
774	        local t1_jobs
775	        read -r -a t1_jobs <<< "$(get_writeback_jobs_for_tier 1)"
776	        if [[ ${#t1_jobs[@]} -gt 0 && "$outage_minutes" -ge "$tier1_wb_delay" ]]; then
777	            echo "  Rsync writeback — outage ${outage_minutes}min >= ${tier1_wb_delay}min"
778	            for job in "${t1_jobs[@]}"; do
779	                [[ -z "$job" ]] && continue
780	                log "Syncing: $job"
781	                if [[ "$DRY_RUN" == false ]]; then
782	                    if [[ "$(basename "$job")" == "Emby" ]]; then
783	                        bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job" --profile=emby-fallback
784	                    else
785	                        bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$job"
786	                    fi
787	                else
788	                    warn "DRY RUN — would rsync: $job"
789	                fi
790	            done
791	        else
792	            log "Tier 1 writeback skipped — outage ${outage_minutes}min < ${tier1_wb_delay}min"
793	        fi
794	    fi
795
796	    echo "  Starting Tier 1 on $REMOTE_SERVER_NAME..."
797	    sleep 10  # give databases time to initialise before apps
798	    for container in "${t1[@]}"; do
799	        [[ -z "$container" ]] && continue
800	        local is_ddns=false
801	        for ddns in "${REMOTE_DDNS_CONTAINERS[@]}"; do
802	            [[ "$container" == "$ddns" ]] && is_ddns=true && break
803	        done
804	        [[ "$is_ddns" == false ]] && remote_start "$container"
805	    done
806
807	    # ── Step 7: DNS Cutover — final step ────────────────────────────────────────────────────
808	    # DNS cuts over ONLY after Tier 1 containers confirmed up
809	    echo ""
810	    echo "━━━ $ICON_NET DNS Cutover ━━━"
811	    sleep 5
812	    for container in "${REMOTE_DDNS_CONTAINERS[@]}"; do
813	        remote_start "$container"
814	    done
815	    warn "$ICON_NET Remote DDNS started — DNS now points at $REMOTE_SERVER_NAME"
816
817	    # ── Step 8: Return to NORMAL ─────────────────────────────────────────────────────────────
818	    echo ""
819	    state_set state "NORMAL"
820	    state_set fallback_start "0"
821	    state_set handback_strikes "0"
822	    state_set tier2_started "false"
823	    state_set tier3_started "false"
824	    state_set tier4_started "false"
825
826	    # Restore local DDNS — may have been stopped during NO_INTERNET or DARK state before handback
827	    local_ddns_start
828
829	    warn "$ICON_DONE Handback complete — returned to NORMAL"
830	    notify "Fallback handback complete on $(hostname) — $REMOTE_SERVER_NAME is back, all containers returned" \
831	        "Fallback" "normal"
832	}
833
834	# ==============================================================================================
835	# ━━━ Main State Machine ━━━
836	# ==============================================================================================
837	state_init
838	set_ddns_arrays
839
840	echo ""
841	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
842	echo "  $ICON_FALLBACK FALLBACK — $(date '+%Y-%m-%d %H:%M:%S')"
843	echo "  $ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → monitoring $REMOTE_ID ($REMOTE_SERVER_NAME)"
844	echo "  $ICON_NET  Remote IP: $REMOTE_SERVER"
845	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
846
847	FALLBACK_RUNNING=true
848	trap 'FALLBACK_RUNNING=false; warn "Fallback received shutdown signal — stopping cleanly"; exit 0' \
849	    SIGTERM SIGINT
850
851	while [[ "$FALLBACK_RUNNING" == true ]]; do
852	    NOW=$(date +%s)
853	    CURRENT_STATE=$(state_get state)
854
855	    # ── Connectivity checks ──────────────────────────────────────────────────────────────────
856	    REMOTE_UP=false
857	    INTERNET_UP=false
858
859	    ping_remote   && REMOTE_UP=true
860	    ping_internet && INTERNET_UP=true
861
862	    log "Remote: $REMOTE_UP | Internet: $INTERNET_UP | State: $CURRENT_STATE"
863
864	    # ── Partnership gate ──────────────────────────────────────────────────────────────────────
865	    # When FALLBACK_PARTNERSHIP_REQUIRED=true, a fallback without an active partner is
866	    # meaningless — who are we covering for? The gate suspends the loop after a grace
867	    # period and aborts any live fallback containers. Clears automatically when partnership
868	    # is restored.
869	    if [[ "${FALLBACK_PARTNERSHIP_REQUIRED:-true}" == "true" ]]; then
870	        if check_partnership_active; then
871	            # Partnership healthy — clear any accumulated grace state
872	            if [[ "$(state_get partnership_suspended)" == "true" ]]; then
873	                warn "Active partnership restored — re-enabling fallback monitoring"
874	                state_set partnership_suspended "false"
875	                state_set partner_lost_at "0"
876	                notify "Fallback re-enabled on $(hostname) — active partnership restored" \
877	                    "Fallback" "normal"
878	            elif [[ "$(state_get partner_lost_at)" != "0" ]]; then
879	                log "Partnership recovered within grace window — resetting timer"
880	                state_set partner_lost_at "0"
881	            fi
882	        else
883	            PARTNER_LOST_AT=$(state_get partner_lost_at)
884	            if [[ -z "$PARTNER_LOST_AT" || "$PARTNER_LOST_AT" == "0" ]]; then
885	                warn "No active partnership detected — grace timer started (${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}min before suspend)"
886	                state_set partner_lost_at "$NOW"
887	                PARTNER_LOST_AT=$NOW
888	            fi
889
890	            PARTNER_LOST_MIN=$(( (NOW - PARTNER_LOST_AT) / 60 ))
891	            SUSPEND_AFTER=${FALLBACK_PARTNERSHIP_SUSPEND_AFTER:-120}
892
893	            if [[ "$(state_get partnership_suspended)" == "true" || "$PARTNER_LOST_MIN" -ge "$SUSPEND_AFTER" ]]; then
894	                # Suspend — abort any live fallback containers, then skip this cycle entirely
895	                if [[ "$(state_get partnership_suspended)" != "true" ]]; then
896	                    warn "No active partnership for ${PARTNER_LOST_MIN}min — suspending fallback"
897	                    state_set partnership_suspended "true"
898	                    notify "Fallback suspended on $(hostname) — no active partnership for ${PARTNER_LOST_MIN}min" \
899	                        "Fallback" "warning"
900	                fi
901	                if [[ "$CURRENT_STATE" == "FALLBACK" ]]; then
902	                    warn "Aborting in-progress fallback — partnership suspended"
903	                    _abort_fallback_containers
904	                    state_set state "NORMAL"
905	                    state_set fallback_start "0"
906	                    state_set handback_strikes "0"
907	                    notify "Fallback aborted on $(hostname) — no active partnership, containers stopped" \
908	                        "Fallback" "warning"
909	                fi
910	                log "Partnership suspended — skipping fallback cycle"
911	                sleep "$FALLBACK_CHECK_INTERVAL" & wait $!
912	                continue
913	            else
914	                log "No active partnership — ${PARTNER_LOST_MIN}min elapsed / ${SUSPEND_AFTER}min until suspend"
915	            fi
916	        fi
917	    fi
918
919	    # ════════════════════════════════════════════════════════════════
920	    # NORMAL STATE
921	    # ════════════════════════════════════════════════════════════════
922	    if [[ "$CURRENT_STATE" == "NORMAL" ]]; then
923
924	        if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
925	            log "$ICON_SUCCESS NORMAL — all systems up"
926
927	        elif [[ "$REMOTE_UP" == false && "$INTERNET_UP" == true ]]; then
928	            # Remote is down — enter FALLBACK immediately
929	            echo ""
930	            echo "━━━ $ICON_FALLBACK Entering FALLBACK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
931	            warn "$REMOTE_SERVER_NAME ($REMOTE_ID) is unreachable — internet is up — starting fallback"
932
933	            state_set state "FALLBACK"
934	            state_set fallback_start "$NOW"
935	            state_set handback_strikes "0"
936	            state_set tier2_started "false"
937	            state_set tier3_started "false"
938	            state_set tier4_started "false"
939
940	            echo ""
941	            echo "━━━ $ICON_START Tier 1 — Immediate ━━━"
942	            read -r -a tier1 <<< "$(get_tier_containers 1)"
943	            for container in "${tier1[@]}"; do
944	                [[ -n "$container" ]] && local_start "$container"
945	            done
946
947	            notify "FALLBACK started on $(hostname) — $REMOTE_SERVER_NAME is down — Tier 1 started" \
948	                "Fallback" "warning"
949
950	        elif [[ "$INTERNET_UP" == false ]]; then
951	            # Lost internet — enter NO_INTERNET
952	            echo ""
953	            echo "━━━ $ICON_NET Entering NO_INTERNET — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
954	            warn "Internet connectivity lost — stopping local DDNS"
955
956	            state_set state "NO_INTERNET"
957	            local_ddns_stop
958
959	            # Stop containers configured to stop on internet loss
960	            read -r -a stop_on_no_net <<< "$(get_stop_on_no_net)"
961	            for container in "${stop_on_no_net[@]}"; do
962	                [[ -n "$container" ]] && local_stop "$container"
963	            done
964
965	            notify "NO_INTERNET on $(hostname) — DDNS stopped, waiting for recovery" \
966	                "Fallback" "warning"
967	        fi
968
969	    # ════════════════════════════════════════════════════════════════
970	    # FALLBACK STATE
971	    # ════════════════════════════════════════════════════════════════
972	    elif [[ "$CURRENT_STATE" == "FALLBACK" ]]; then
973
974	        FALLBACK_START_TS=$(state_get fallback_start)
975	        ELAPSED_MIN=$(( (NOW - FALLBACK_START_TS) / 60 ))
976
977	        if [[ "$REMOTE_UP" == true && "$INTERNET_UP" == true ]]; then
978	            # Remote returned — increment handback strikes
979	            STRIKES=$(state_get handback_strikes)
980	            STRIKES=$(( STRIKES + 1 ))
981	            state_set handback_strikes "$STRIKES"
982
983	            warn "$ICON_FALLBACK $REMOTE_SERVER_NAME is back — handback strike $STRIKES/$FALLBACK_HANDBACK_STRIKES"
984
985	            if [[ "$STRIKES" -ge "$FALLBACK_HANDBACK_STRIKES" ]]; then
986	                run_handback
987	            fi
988
989	        elif [[ "$INTERNET_UP" == false ]]; then
990	            # Lost internet during fallback — enter DARK
991	            echo ""
992	            echo "━━━ $ICON_NET Entering DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
993	            warn "Lost internet during fallback — entering DARK state"
994	            state_set state "DARK"
995	            local_ddns_stop
996	            notify "DARK state on $(hostname) — lost internet during fallback" \
997	                "Fallback" "warning"
998
999	        else
1000	            # Still in FALLBACK state — check tier escalation
1001	            state_set handback_strikes "0"
1002
1003	            TIER2_DELAY=$(get_tier2_delay)
1004	            if [[ "$(state_get tier2_started)" == "false" && \
1005	                  "$ELAPSED_MIN" -ge "$TIER2_DELAY" ]]; then
1006	                echo ""
1007	                echo "━━━ $ICON_START Tier 2 — ${ELAPSED_MIN}min outage ━━━"
1008	                read -r -a tier2 <<< "$(get_tier_containers 2)"
1009	                for container in "${tier2[@]}"; do
1010	                    [[ -n "$container" ]] && local_start "$container"
1011	                done
1012	                state_set tier2_started "true"
1013	                notify "Fallback Tier 2 started on $(hostname) — ${ELAPSED_MIN}min outage" \
1014	                    "Fallback" "warning"
1015	            fi
1016
1017	            TIER3_DELAY=$(get_tier3_delay)
1018	            if [[ "$(state_get tier3_started)" == "false" && \
1019	                  "$ELAPSED_MIN" -ge "$TIER3_DELAY" ]]; then
1020	                echo ""
1021	                echo "━━━ $ICON_START Tier 3 — ${ELAPSED_MIN}min outage ━━━"
1022	                read -r -a tier3 <<< "$(get_tier_containers 3)"
1023	                for container in "${tier3[@]}"; do
1024	                    [[ -n "$container" ]] && local_start "$container"
1025	                done
1026	                state_set tier3_started "true"
1027	                notify "Fallback Tier 3 started on $(hostname) — ${ELAPSED_MIN}min outage" \
1028	                    "Fallback" "warning"
1029	            fi
1030
1031	            TIER4_DELAY=$(get_tier4_delay)
1032	            if [[ "$(state_get tier4_started)" == "false" && \
1033	                  "$ELAPSED_MIN" -ge "$TIER4_DELAY" ]]; then
1034	                echo ""
1035	                echo "━━━ $ICON_START Tier 4 — ${ELAPSED_MIN}min outage — full workflow ━━━"
1036	                read -r -a tier4 <<< "$(get_tier_containers 4)"
1037	                for container in "${tier4[@]}"; do
1038	                    [[ -n "$container" ]] && local_start "$container"
1039	                done
1040	                state_set tier4_started "true"
1041	                notify "Fallback Tier 4 started on $(hostname) — ${ELAPSED_MIN}min outage — full workflow active" \
1042	                    "Fallback" "warning"
1043	            fi
1044
1045	            log "$ICON_FALLBACK FALLBACK — ${ELAPSED_MIN}min — T2:$(state_get tier2_started) T3:$(state_get tier3_started) T4:$(state_get tier4_started)"
1046	        fi
1047
1048	    # ════════════════════════════════════════════════════════════════
1049	    # NO_INTERNET STATE
1050	    # ════════════════════════════════════════════════════════════════
1051	    elif [[ "$CURRENT_STATE" == "NO_INTERNET" ]]; then
1052
1053	        if [[ "$INTERNET_UP" == true ]]; then
1054	            echo ""
1055	            echo "━━━ $ICON_NET Internet Recovered — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
1056
1057	            if [[ "$REMOTE_UP" == true ]]; then
1058	                warn "Remote is up — returning to NORMAL"
1059	                state_set state "NORMAL"
1060	                local_ddns_start
1061	                notify "Internet recovered on $(hostname) — returning to NORMAL" \
1062	                    "Fallback" "normal"
1063	            else
1064	                warn "Internet back but $REMOTE_SERVER_NAME still down — entering FALLBACK"
1065	                state_set state "FALLBACK"
1066	                state_set fallback_start "$NOW"
1067	                state_set handback_strikes "0"
1068	                state_set tier2_started "false"
1069	                state_set tier3_started "false"
1070	                state_set tier4_started "false"
1071
1072	                echo ""
1073	                echo "━━━ $ICON_START Tier 1 — Immediate ━━━"
1074	                read -r -a tier1 <<< "$(get_tier_containers 1)"
1075	                for container in "${tier1[@]}"; do
1076	                    [[ -n "$container" ]] && local_start "$container"
1077	                done
1078	                notify "Internet recovered on $(hostname) but $REMOTE_SERVER_NAME still down — entering FALLBACK" \
1079	                    "Fallback" "warning"
1080	            fi
1081	        else
1082	            log "$ICON_NET NO_INTERNET — waiting for connectivity"
1083	        fi
1084
1085	    # ════════════════════════════════════════════════════════════════
1086	    # DARK STATE
1087	    # ════════════════════════════════════════════════════════════════
1088	    elif [[ "$CURRENT_STATE" == "DARK" ]]; then
1089
1090	        if [[ "$INTERNET_UP" == true ]]; then
1091	            echo ""
1092	            echo "━━━ $ICON_NET Emerging from DARK — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
1093
1094	            if [[ "$REMOTE_UP" == true ]]; then
1095	                warn "Remote up, internet up — transitioning through FALLBACK for handback"
1096	                # Was in FALLBACK state before DARK — route through FALLBACK state for handback
1097	                state_set state "FALLBACK"
1098	                state_set handback_strikes "0"
1099	            else
1100	                warn "Internet back but $REMOTE_SERVER_NAME still down — entering FALLBACK"
1101	                state_set state "FALLBACK"
1102	                state_set fallback_start "$NOW"
1103	                state_set handback_strikes "0"
1104	                state_set tier2_started "false"
1105	                state_set tier3_started "false"
1106	                state_set tier4_started "false"
1107
1108	                read -r -a tier1 <<< "$(get_tier_containers 1)"
1109	                for container in "${tier1[@]}"; do
1110	                    [[ -n "$container" ]] && local_start "$container"
1111	                done
1112	            fi
1113	            notify "Emerging from DARK state on $(hostname)" "Fallback" "warning"
1114	        else
1115	            log "$ICON_FALLBACK DARK — no internet, no remote — waiting"
1116	        fi
1117	    fi
1118
1119	    # ── Sleep until next check ───────────────────────────────────────────────────────────────
1120	    log "Next check in ${FALLBACK_CHECK_INTERVAL}s — $(date '+%H:%M:%S')"
1121	    sleep "$FALLBACK_CHECK_INTERVAL" &
1122	    wait $!
1123
1124	done


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= CONFIGURATION LOADER =======================================
4	# ==============================================================================================
5	# Single entry point for all configuration sourcing across the ecosystem.
6	# Every script sources this file instead of sourcing master.conf files directly.
7	#
8	# ── HOW IT WORKS ──────────────────────────────────────────────────────────────────────────────
9	# 1. Detects OS platform → PLATFORM=unraid|truenas|unknown; exports SCRIPTS_DIR
10	# 2. Sources master.conf (shared config — hostnames, thresholds, toggles, profiles, job lists)
11	# 3. Auto-discovers and sources all host*.conf files in the same directory
12	#    Each host conf extends the shared profile arrays and adds host-specific credentials
13	# 4. Sources common.sh (shared functions — detect_hosts, logging, notifications etc.)
14	# 5. Sources Plugin/<platform>/adapter.sh (platform_*() functions for OS-specific ops)
15	#
16	# ── WHY THIS EXISTS ───────────────────────────────────────────────────────────────────────────
17	# Without this loader every script had to explicitly source each conf file:
18	#   source master.conf
19	#   source host1.conf
20	#   source host2.conf
21	#   source common.sh
22	#
23	# Adding a new server meant updating every script.
24	# With this loader — add host3.conf to the git repo and every server
25	# auto-discovers it on next git pull. Zero script changes required. Ever.
26	#
27	# ── ADDING A NEW SERVER ───────────────────────────────────────────────────────────────────────
28	# 1. Create Configurations/host3.conf following the same structure as HOST1/HOST2
29	# 2. Commit and push to git repo
30	# 3. All servers pull it automatically — no other changes needed
31	#
32	# ── USAGE IN SCRIPTS ──────────────────────────────────────────────────────────────────────────
33	# Replace the three source lines at the top of every script with:
34	#
35	#   SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
36	#   source "$SCRIPT_DIR/../load_config.sh"
37	#
38	# Scripts in subdirectories (Rsync/, Docker_Essentials/ etc.) use ../ to reach root.
39	# Scripts in root directory use ./ instead:
40	#
41	#   SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
42	#   source "$SCRIPT_DIR/load_config.sh"
43	#
44	# ── SPARSE CHECKOUT NOTE ──────────────────────────────────────────────────────────────────────
45	# Sparse checkout controls which host*.conf files each server receives.
46	# HOST1 only pulls Configurations/host1.conf — never HOST2's credentials.
47	# HOST2 only pulls Configurations/host2.conf — never HOST1's credentials.
48	# This loader sources whatever conf files ARE present — sparse checkout handles the rest.
49	# Both servers pull all non-credential conf files (Configurations/master.conf, common.sh, load_config.sh).
50	#
51	# ==============================================================================================
52
53	# ━━━ Locate config root ━━━
54	# load_config.sh always lives in the repo root.
55	# Scripts call it from subdirectories using ../ — resolve to the actual root.
56	    LOAD_CONFIG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
57
58	# ━━━ Platform detection ━━━
59	# PLATFORM drives the adapter layer — Plugin/<platform>/adapter.sh
60	# Each platform adapter provides the same function API; scripts stay OS-agnostic.
61	    if   [[ -f /etc/unraid-version ]]; then  PLATFORM="unraid"
62	    elif [[ -f /etc/truenas ]];        then  PLATFORM="truenas"
63	    else                                     PLATFORM="unknown"
64	    fi
65	    export PLATFORM SCRIPTS_DIR="$LOAD_CONFIG_DIR"
66
67	# ━━━ Source shared config ━━━
68	# master.conf must be sourced first — it declares the shared PROFILE_* arrays
69	# that Host confs extend. Sourcing host confs before master.conf would fail.
70	    if [[ ! -f "$LOAD_CONFIG_DIR/Configurations/master.conf" ]]; then
71	        echo "[FATAL] master.conf not found at $LOAD_CONFIG_DIR/Configurations/master.conf" >&2
72	        echo "[FATAL] Check TARGET_DIR and git pull status" >&2
73	        exit 1
74	    fi
75	    source "$LOAD_CONFIG_DIR/Configurations/master.conf"
76
77	# ━━━ Auto-discover and source all host*.conf files ━━━
78	# Sorted for consistent load order — HOST1 before HOST2 before HOST3 etc.
79	# Each host conf extends the shared PROFILE_* arrays and adds host-specific vars.
80	# Missing files are silently skipped — sparse checkout intentionally withholds some.
81	# At least one host conf must be present or the ecosystem has no identity to work with.
82	    _host_confs_loaded=0
83	    declare -A _disk_conf_basenames=()
84
85	    # Use a sorted array glob — avoids word-splitting on paths with spaces
86	    while IFS= read -r _conf; do
87	        [[ -f "$_conf" ]] || continue
88	        source "$_conf"
89	        (( _host_confs_loaded++ ))
90	        _disk_conf_basenames["$(basename "$_conf")"]=1
91	        [[ "${ENABLE_LOGGING:-false}" == "true" ]] && \
92	            echo "[LOG] Loaded host config: $(basename "$_conf")" >&2
93	    done < <(printf '%s\n' "$LOAD_CONFIG_DIR/Configurations"/host*.conf 2>/dev/null | sort)
94
95	    if [[ "$_host_confs_loaded" -eq 0 ]]; then
96	        echo "[FATAL] No host*.conf files found in $LOAD_CONFIG_DIR" >&2
97	        echo "[FATAL] At least one host conf required — check git pull and sparse checkout" >&2
98	        exit 1
99	    fi
100
101	# ━━━ Source partner confs from /tmp cache ━━━
102	# conf_sync.sh pulls partner host*.conf files into /tmp/.vv/config/cached/.confs/
103	# on array start and after any conf save. Sourcing them here makes partner vars
104	# (HOST2_*, HOST3_*, …) available without committing credentials to the git repo
105	# or violating sparse checkout — partner confs live in RAM only, cleared on reboot.
106	# Confs already loaded from disk are skipped — disk copy is authoritative.
107	    _VV_CONF_CACHE="/tmp/.vv/config/cached/.confs"
108	    if [[ -d "$_VV_CONF_CACHE" ]]; then
109	        while IFS= read -r _conf; do
110	            [[ -f "$_conf" ]] || continue
111	            _conf_base="$(basename "$_conf")"
112	            # Skip if already sourced from disk
113	            [[ -n "${_disk_conf_basenames[$_conf_base]:-}" ]] && continue
114	            source "$_conf"
115	            [[ "${ENABLE_LOGGING:-false}" == "true" ]] && \
116	                echo "[LOG] Loaded cached partner config: $_conf_base" >&2
117	        done < <(printf '%s\n' "$_VV_CONF_CACHE"/host*.conf 2>/dev/null | sort)
118	    fi
119	    unset _VV_CONF_CACHE _conf_base _disk_conf_basenames
120
121	# ━━━ Source shared functions ━━━
122	# common.sh sourced last — it calls detect_hosts() which needs HOST* vars to be set.
123	    if [[ ! -f "$LOAD_CONFIG_DIR/common.sh" ]]; then
124	        echo "[FATAL] common.sh not found at $LOAD_CONFIG_DIR/common.sh" >&2
125	        exit 1
126	    fi
127	    source "$LOAD_CONFIG_DIR/common.sh"
128
129	# ━━━ Source platform adapter ━━━
130	# Provides platform_*() functions used by common.sh and scripts.
131	# Guard lets the ecosystem run before Plugin/<platform>/adapter.sh exists.
132	    _adapter="$LOAD_CONFIG_DIR/Plugin/$PLATFORM/adapter.sh"
133	    [[ -f "$_adapter" ]] && source "$_adapter"
134
135	# ━━━ Derived path constants ━━━
136	# Centralised here so every script that sources load_config.sh has them without
137	# re-deriving from SCRIPTS_DIR or hardcoding /var/log or /tmp paths inline.
138	    CONF_DIR="${SCRIPTS_DIR}/Configurations"
139	    LOG_DIR="/var/log/varaverk"
140	    VV_CACHE_DIR="/tmp/vv_cache"
141	    export CONF_DIR LOG_DIR VV_CACHE_DIR
142
143	# ━━━ Cleanup ━━━
144	    unset _conf _host_confs_loaded _adapter LOAD_CONFIG_DIR


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Git Pull & Execute =========================================
4	# ==============================================================================================
5	# Pulls the latest scripts from the Gitea repository via SSH.
6	# Lives at the repo root — sources load_config.sh from the same directory.
7	#
8	# ── WHAT THIS SCRIPT DOES ─────────────────────────────────────────────────────────────────────
9	# 1. Detects which server it's running on via detect_hosts() (MY_ID)
10	# 2. Configures sparse checkout to exclude other servers' credential files
11	#    Each server only pulls its own host*.conf — never sees peer credentials
12	# 3. Pulls or clones latest scripts from Gitea
13	# 4. Sets executable permissions on all .sh files
14	#
15	# ── SPARSE CHECKOUT ───────────────────────────────────────────────────────────────────────────
16	# Sparse checkout ensures each server only receives its own host conf:
17	#   HOST1 pulls:  master.conf + host1.conf + all scripts
18	#   HOST1 skips:  host2.conf, host3.conf etc.
19	#   HOST2 pulls:  master.conf + host2.conf + all scripts
20	#   HOST2 skips:  host1.conf, host3.conf etc.
21	#
22	# Adding a new server:
23	#   Create host3.conf in the repo
24	#   All existing servers automatically exclude it on next pull
25	#   New server gets only its own conf ✅
26	#
27	# ── GITEA LOCATION DETECTION ──────────────────────────────────────────────────────────────────
28	# Detects where Gitea is running at runtime — works through fallback:
29	#   Gitea local  → connects via local IP
30	#   Gitea remote → connects via Tailscale IP
31	#   Both fail    → falls back to GITEA_DOMAIN if configured
32	#
33	# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
34	#   GITEA_CONTAINER  — Docker container name for Gitea
35	#   GITEA_REPO_PATH  — repo path on Gitea (e.g. Varaverk/varaverk.git)
36	#   GITEA_DOMAIN     — public domain fallback (optional)
37	#   TARGET_DIR       — local path to clone/pull into
38	#   GITEA_SSH_KEY    — SSH key for Gitea authentication
39	#   SSH_PORT         — Gitea SSH port (often 221 or 222)
40	#
41	# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
42	#   git_pull_execute.sh              — normal pull
43	#   git_pull_execute.sh --dry-run    — preview without making changes
44	#   git_pull_execute.sh --log        — verbose output
45	#   git_pull_execute.sh --status     — show config and exit
46	# ==============================================================================================
47
48	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
49
50	# Root level script — load_config.sh is in the same directory
51	source "$SCRIPT_DIR/load_config.sh"
52
53	parse_args "$@"
54
55	# ==============================================================================================
56	# ━━━ Setup ━━━
57	# ==============================================================================================
58	if [[ "$EUID" -ne 0 ]]; then
59	    error "Must be run as root"
60	    exit 1
61	fi
62	acquire_lock
63
64	if ! command -v docker &>/dev/null; then
65	    error "Docker command not found"
66	    exit 1
67	fi
68
69	# detect_hosts() sets MY_ID — needed for sparse checkout configuration
70	detect_hosts
71
72	# ==============================================================================================
73	# ━━━ Locate Gitea ━━━
74	# ==============================================================================================
75	echo ""
76	echo "━━━ $ICON_CONTAINERS Locate Gitea ━━━"
77
78	if docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^${GITEA_CONTAINER}$"; then
79	    # Gitea is running on this server — use local IP
80	    GITEA_IP=$(hostname -I | awk '{print $1}')
81	    log "Gitea running locally — connecting via $GITEA_IP"
82	else
83	    # Gitea not running locally — find it on the remote server via Tailscale
84	    log "Gitea not running locally — checking remote server"
85	    GITEA_IP=$(resolve_tailscale_ip "${REMOTE_SERVER_NAME}")
86	    if [[ -n "$GITEA_IP" ]]; then
87	        echo "  Gitea on $REMOTE_SERVER_NAME — connecting via Tailscale $GITEA_IP"
88	    elif [[ -n "${GITEA_DOMAIN:-}" ]]; then
89	        warn "Tailscale resolution failed — falling back to $GITEA_DOMAIN"
90	        GITEA_IP="$GITEA_DOMAIN"
91	    else
92	        error "Cannot find Gitea — local: not running, Tailscale: failed, domain: not configured"
93	        notify "Git pull failed on $(hostname) — cannot locate Gitea container" "Git Sync" "alert"
94	        exit 1
95	    fi
96	fi
97
98	REPO_SSH="git@${GITEA_IP}:${GITEA_REPO_PATH}"
99
100	require_var REPO_SSH
101	require_var TARGET_DIR
102	require_var GITEA_SSH_KEY
103	require_var SSH_PORT
104
105	# ==============================================================================================
106	# ━━━ Status ━━━
107	# ==============================================================================================
108	if [[ "$SHOW_STATUS" == true ]]; then
109	    echo ""
110	    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
111	    echo "$ICON_NET  Repo:        $REPO_SSH"
112	    echo "$ICON_GEAR Target:      $TARGET_DIR"
113	    echo "$ICON_GEAR SSH Key:     $GITEA_SSH_KEY"
114	    echo "$ICON_GEAR SSH Port:    $SSH_PORT"
115	    echo "$ICON_HOST My ID:       $MY_ID ($LOCAL_SERVER_NAME)"
116	    echo "$ICON_HOST Remote ID:   $REMOTE_ID ($REMOTE_SERVER_NAME)"
117	    echo "$ICON_NOTIFY Notify:    unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
118	    echo "$ICON_GEAR Dry Run:     $DRY_RUN"
119	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
120	    exit 0
121	fi
122
123	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
124
125	# ==============================================================================================
126	# ━━━ Sparse Checkout Configuration ━━━
127	# ==============================================================================================
128	# Build the list of host*.conf files that belong to OTHER servers.
129	# This server pulls everything EXCEPT those files.
130	# MY_ID is set by detect_hosts() — e.g. "HOST1"
131
132	configure_sparse_checkout() {
133	    local repo_dir="$1"
134
135	    log "Configuring sparse checkout for $MY_ID..."
136
137	    # Enable sparse checkout
138	    git -C "$repo_dir" config core.sparseCheckout true 2>/dev/null
139
140	    # Build exclusion list — all host*.conf files except MY_ID's
141	    local sparse_file="$repo_dir/.git/info/sparse-checkout"
142	    mkdir -p "$(dirname "$sparse_file")"
143
144	    # Start with: pull everything
145	    echo "/*" > "$sparse_file"
146
147	    # Exclude each other server's conf file
148	    # Find all host*.conf files present in the repo
149	    local excluded=0
150	    for conf_file in "$repo_dir"/host*.conf; do
151	        [[ -f "$conf_file" ]] || continue
152	        local conf_name
153	        conf_name=$(basename "$conf_file")
154
155	        # Determine which HOST ID owns this conf by grepping its hostname var
156	        # Pattern: HOST1="unRAID-..." or HOST2="unRAID-..."
157	        local conf_host_id
158	        conf_host_id=$(grep -m1 -oP '^\s+HOST[0-9]+(?==)' "$conf_file" 2>/dev/null | tr -d ' ')
159
160	        if [[ -z "$conf_host_id" ]]; then
161	            log "Cannot determine HOST ID for $conf_name — including in pull (safe default)"
162	            continue
163	        fi
164
165	        if [[ "$conf_host_id" != "$MY_ID" ]]; then
166	            echo "!$conf_name" >> "$sparse_file"
167	            log "Sparse checkout: excluding $conf_name (belongs to $conf_host_id)"
168	            ((excluded++))
169	        else
170	            log "Sparse checkout: including $conf_name (belongs to $MY_ID — this server)"
171	        fi
172	    done
173
174	    if [[ "$excluded" -gt 0 ]]; then
175	        echo "  Sparse checkout: excluding $excluded peer conf file(s) — credentials protected"
176	    else
177	        log "Sparse checkout: no peer conf files to exclude (single server or first run)"
178	    fi
179	}
180
181	# ==============================================================================================
182	# ━━━ Git Sync ━━━
183	# ==============================================================================================
184	echo ""
185	echo "━━━ $ICON_SYNC Git Sync ━━━"
186	echo "$ICON_NET  Repo:    $REPO_SSH"
187	echo "$ICON_GEAR Target:  $TARGET_DIR"
188	echo ""
189
190	START=$(date +%s)
191	SYNC_SUCCESS=false
192
193	if [[ "$DRY_RUN" == true ]]; then
194	    warn "DRY RUN — would sync $REPO_SSH → $TARGET_DIR"
195	    warn "DRY RUN — would configure sparse checkout for $MY_ID"
196	    warn "DRY RUN — would exclude peer host*.conf files"
197	    SYNC_SUCCESS=true
198	else
199	    mkdir -p "$TARGET_DIR"
200	    git config --global --add safe.directory "$TARGET_DIR"
201	    cd "$TARGET_DIR" || { error "Cannot cd into $TARGET_DIR"; exit 1; }
202
203	    if [[ -d ".git" ]]; then
204	        # ── Existing repository ──────────────────────────────────────────────
205	        echo "  Existing repository — updating"
206
207	        # Configure sparse checkout BEFORE pull
208	        # Uses conf files already present from last pull to determine exclusions
209	        configure_sparse_checkout "$TARGET_DIR"
210
211	        echo "  Pulling latest changes..."
212	        if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git pull --ff-only; then
213	            echo "  Git pull successful"
214	            SYNC_SUCCESS=true
215	        else
216	            # ff-only fails when local commits or tracked changes exist that can't
217	            # fast-forward. Fail loudly — never silently destroy local work.
218	            error "Git pull failed — local changes conflict with remote (will not force-reset)"
219	            notify "Git pull failed on $(hostname) — local changes conflict, manual resolve needed" "Git Sync" "alert"
220	            exit 1
221	        fi
222
223	    else
224	        # ── Fresh clone ──────────────────────────────────────────────────────
225	        echo "  No repository found — cloning"
226
227	        # Clone first — need the repo to exist before configuring sparse checkout
228	        if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git clone "$REPO_SSH" .; then
229	            echo "  Clone successful"
230
231	            # Configure sparse checkout after clone
232	            # Now all host*.conf files are present — can detect exclusions
233	            configure_sparse_checkout "$TARGET_DIR"
234
235	            # Apply sparse checkout — removes excluded files from working tree
236	            echo "  Applying sparse checkout..."
237	            git read-tree -mu HEAD
238	            echo "  Sparse checkout applied — peer credentials removed from working tree"
239
240	            SYNC_SUCCESS=true
241	        else
242	            error "Clone failed"
243	            notify "Git clone failed on $(hostname) — check Gitea connectivity" "Git Sync" "alert"
244	            exit 1
245	        fi
246	    fi
247
248	    # ── Permissions ──────────────────────────────────────────────────────────
249	    echo ""
250	    echo "━━━ $ICON_GEAR Permissions ━━━"
251	    log "Setting executable permissions on all .sh files..."
252	    find "$TARGET_DIR" -type f -name "*.sh" -exec chmod +x {} \;
253	    echo "  Permissions set on .sh files"
254
255	    # ── Flash mode: sync Plugin/ to /boot/ so the webUI picks up updates ─────
256	    # In flash mode SCRIPTS_DIR is in appdata — Plugin/ lives in the repo there
257	    # but Unraid serves PHP from /boot/. Sync after every pull to keep them in step.
258	    _BOOT_DIR=$(platform_webui_install_path)
259	    if [[ "$TARGET_DIR" != "$_BOOT_DIR" ]]; then
260	        echo ""
261	        echo "━━━ $ICON_SYNC Flash mode: sync Plugin/ → /boot/ ━━━"
262	        if rsync -a --delete "$TARGET_DIR/Plugin/" "$_BOOT_DIR/Plugin/" 2>/dev/null; then
263	            echo "  Plugin/ synced to /boot/ ✅"
264	        else
265	            warn "Plugin/ sync to /boot/ failed — webUI may be stale until next pull"
266	        fi
267	    fi
268	fi
269
270	END=$(date +%s)
271
272	# ==============================================================================================
273	# ━━━ Conf Upgrade ━━━
274	# ==============================================================================================
275	# Merges new conf structure into the live conf files after every pull.
276	# New keys → added with template defaults (user fills in once).
277	# Removed keys → dropped. Existing values → always preserved.
278	# Silent when already up to date — no overhead on unchanged pulls.
279
280	echo ""
281	echo "━━━ $ICON_GEAR Conf Upgrade ━━━"
282
283	UPGRADE_SCRIPT="$TARGET_DIR/Deployment/conf_upgrade.sh"
284	CONF_DIR="$TARGET_DIR/Configurations"
285
286	if [[ ! -f "$UPGRADE_SCRIPT" ]]; then
287	    log "conf_upgrade.sh not found — skipping (pre-deployment-folder repo)"
288	elif [[ "$DRY_RUN" == true ]]; then
289	    warn "DRY RUN — would upgrade master.conf and ${MY_ID,,}.conf"
290	elif [[ "$SYNC_SUCCESS" == true ]]; then
291	    _DRY=""
292
293	    # master.conf
294	    if [[ -f "$CONF_DIR/master.conf.template" && -f "$CONF_DIR/master.conf" ]]; then
295	        bash "$UPGRADE_SCRIPT" \
296	            --template "$CONF_DIR/master.conf.template" \
297	            --target   "$CONF_DIR/master.conf" \
298	            --backup   $_DRY
299	    else
300	        warn "master.conf.template or master.conf not found — skipping"
301	    fi
302
303	    # This server's host conf only — sparse checkout ensures we have it
304	    HOST_CONF="$CONF_DIR/${MY_ID,,}.conf"
305	    if [[ -f "$CONF_DIR/host.conf.template" && -f "$HOST_CONF" ]]; then
306	        # Template uses HOSTN_ as generic prefix. Substitute MY_ID before merging
307	        # so keys match the target (HOST1_* or HOST2_*) and real values are preserved.
308	        TMPL_RESOLVED=$(mktemp)
309	        sed "s/HOSTN_/${MY_ID}_/g; s/REMOTE_ID/${REMOTE_ID}/g" "$CONF_DIR/host.conf.template" > "$TMPL_RESOLVED"
310	        bash "$UPGRADE_SCRIPT" \
311	            --template "$TMPL_RESOLVED" \
312	            --target   "$HOST_CONF" \
313	            --backup   $_DRY
314	        rm -f "$TMPL_RESOLVED"
315	    else
316	        warn "${MY_ID,,}.conf or host.conf.template not found — skipping"
317	    fi
318	fi
319
320	# ==============================================================================================
321	# ━━━ Summary ━━━
322	# ==============================================================================================
323	echo ""
324	echo "━━━━━ $ICON_SUMMARY GIT SYNC SUMMARY ━━━━━"
325	echo "$ICON_NET  Repo:     $REPO_SSH"
326	echo "$ICON_GEAR Target:   $TARGET_DIR"
327	echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
328	echo "$ICON_LOCK Excluded: peer host*.conf files"
329	echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
330	if [[ "$DRY_RUN" == true ]]; then
331	    echo "$ICON_WARN Status:   DRY RUN — no changes made"
332	elif [[ "$SYNC_SUCCESS" == true ]]; then
333	    echo "$ICON_DONE Status:   $ICON_SUCCESS DONE"
334	    notify "Repository synced successfully on $(hostname)" "Git Sync" "normal"
335	else
336	    echo "$ICON_ERROR Status:  $ICON_ERROR FAILED"
337	fi
338	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"