1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Rsync Stop =================================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Stops rsync intelligently on both local and remote servers. Auto-detects
9	# running orchestrators and chooses the safest stop strategy. If an
10	# orchestrator is running, kills only the rsync subprocess so the orchestrator
11	# exits cleanly after finishing the current share. Use --full-stop to kill
12	# everything immediately.
13	#
14	# ==============================================================================================
15	# OPERATIONAL MODEL
16	# ==============================================================================================
17	#
18	# Two Stop Modes
19	#   Default (smart):
20	#     Detects if an orchestrator (daily/weekly/critical sync) is running.
21	#     If orchestrator found → kills rsync subprocess only. Orchestrator sees
22	#     rsync exit → moves to next share or exits cleanly on its own.
23	#     If no orchestrator → kills rsync directly (standalone rsync.sh run).
24	#     Cleans stale lock files after kill.
25	#     Recovers containers left stopped by interrupted rsync (local only).
26	#
27	#   --full-stop (nuclear):
28	#     Kills orchestrator first → then kills rsync.
29	#     Orchestrator will NOT continue to next share.
30	#     Use when everything needs to stop immediately.
31	#
32	# Orchestrator Detection
33	#   detect_rsync_parent() scans all lock files to find which running process
34	#   has rsync as a descendant. No hardcoded list — works for any orchestrator.
35	#   Returns "script_name:parent_pid" if found, empty if standalone.
36	#
37	# Remote Handling
38	#   Both local and remote handled in one run via SSH.
39	#   Remote containers left as-is — docker_watchdog.sh handles remote recovery.
40	#   If remote unreachable → skips remote cleanly, logs warning.
41	#
42	# ==============================================================================================
43	# OPERATIONAL SAFEGUARDS
44	# ==============================================================================================
45	#
46	# Root Required
47	#   pkill and docker require root.
48	#
49	# Single Instance Lock
50	#   acquire_lock prevents concurrent stop attempts racing each other.
51	#
52	# Timeout Protection
53	#   DOCKER_TIMEOUT (15s) on all docker calls — hung daemon doesn't block.
54	#   SSH_TIMEOUT (15s) on all remote SSH calls.
55	#
56	# SIGTERM → SIGKILL Sequence
57	#   Orchestrators receive SIGTERM first, SIGKILL only if still running after 2s.
58	#
59	# Container Recovery
60	#   Restarts local containers left stopped by the killed rsync session.
61	#   Remote containers deferred to docker_watchdog.sh.
62	#
63	# ==============================================================================================
64	# RUNTIME MODES
65	# ==============================================================================================
66	#
67	# rsync_stop.sh
68	#     Auto-detect orchestrator. Kill rsync-only or full-stop accordingly.
69	#
70	# rsync_stop.sh --full-stop
71	#     Kill orchestrator first, then kill rsync. Nothing continues after this.
72	#
73	# rsync_stop.sh --rsync-only
74	#     Skip container recovery. Used when called by other scripts that handle
75	#     recovery themselves.
76	#
77	# rsync_stop.sh --dry-run
78	#     Show what would be killed without killing anything.
79	#
80	# rsync_stop.sh --status
81	#     Show local and remote rsync PIDs, running orchestrators, and lock files.
82	#
83	# rsync_stop.sh --full-stop --dry-run
84	#     Preview full-stop sequence without making any changes.
85	#
86	# ==============================================================================================
87
88	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
89
90	source "$SCRIPT_DIR/../load_config.sh"
91
92	DOCKER_TIMEOUT=15
93	SSH_TIMEOUT=15
94
95	# ── Parse special flags before parse_args ─────────────────────────────────────────────────────
96	FULL_STOP=false
97	RSYNC_ONLY_MODE=false
98	FILTERED_ARGS=()
99	for arg in "$@"; do
100	    case "$arg" in
101	        --full-stop)   FULL_STOP=true ;;
102	        --rsync-only)  RSYNC_ONLY_MODE=true ;;
103	        *)             FILTERED_ARGS+=("$arg") ;;
104	    esac
105	done
106
107	parse_args "${FILTERED_ARGS[@]}"
108
109	# ==============================================================================================
110	# ━━━ Setup ━━━
111	# ==============================================================================================
112	if [[ "$EUID" -ne 0 ]]; then
113	    error "Must be run as root — pkill and docker require root"
114	    exit 1
115	fi
116
117	platform_require_cmd \
118	    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
119	    "" "" \
120	    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
121
122	acquire_lock
123
124	if ! command -v docker &>/dev/null; then
125	    error "Docker command not found"
126	    exit 1
127	fi
128
129	detect_hosts
130
131	# Soft IP resolution — rsync_stop continues local-only if remote unreachable
132	REMOTE_REACHABLE=false
133	REMOTE_SERVER=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME")
134	if [[ -z "$REMOTE_SERVER" ]]; then
135	    warn "$REMOTE_SERVER_NAME — cannot resolve Tailscale IP, remote operations will be skipped"
136	elif timeout "$SSH_TIMEOUT" ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then
137	    REMOTE_REACHABLE=true
138	    log "$REMOTE_SERVER_NAME reachable ✅"
139	else
140	    warn "$REMOTE_SERVER_NAME unreachable — remote operations will be skipped"
141	fi
142
143	[[ "$DRY_RUN"   == true ]] && warn "DRY RUN — no changes will be made"
144	[[ "$FULL_STOP" == true ]] && warn "FULL STOP mode — orchestrator + rsync will be killed"
145
146	# ==============================================================================================
147	# ━━━ Status ━━━
148	# ==============================================================================================
149	if [[ "$SHOW_STATUS" == true ]]; then
150	    echo ""
151	    echo "━━━━━ $ICON_SUMMARY RSYNC STOP STATUS ━━━━━"
152	    echo "$ICON_HOST  Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
153	    echo "$ICON_NET   Remote:    $REMOTE_ID ($REMOTE_SERVER_NAME)"
154	    echo ""
155
156	    LOCAL_PIDS=$(pgrep -x rsync 2>/dev/null | tr '\n' ' ')
157	    echo "  $ICON_SYNC Local rsync PIDs:  ${LOCAL_PIDS:-none}"
158
159	    for lockfile in "$LOCK_DIR"/*.lock; do
160	        [[ -f "$lockfile" ]] || continue
161	        content=$(cat "$lockfile" 2>/dev/null)
162	        pid="${content%%:*}"
163	        name="${content##*:}"
164	        [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && \
165	            echo "  $ICON_RUNNING Lock: $name (PID $pid)"
166	    done
167
168	    if [[ "$REMOTE_REACHABLE" == true ]]; then
169	        REMOTE_PIDS=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
170	            -o ConnectTimeout="$SSH_TIMEOUT" \
171	            root@"$REMOTE_SERVER" "pgrep -x rsync || true" 2>/dev/null | tr '\n' ' ')
172	        echo "  $ICON_SYNC Remote rsync PIDs: ${REMOTE_PIDS:-none}"
173	    else
174	        echo "  $ICON_WARN Remote: unreachable"
175	    fi
176	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
177	    exit 0
178	fi
179
180	# ==============================================================================================
181	# ── ORCHESTRATOR DETECTION ────────────────────────────────────────────────────────────────────
182	# ==============================================================================================
183	# Scans lock files to find which running process has rsync as a descendant.
184	# No hardcoded script names — detects any orchestrator automatically.
185
186	detect_rsync_parent() {
187	    local rsync_pids
188	    rsync_pids=$(pgrep -x rsync 2>/dev/null || true)
189	    [[ -z "$rsync_pids" ]] && echo "" && return
190
191	    for lockfile in "$LOCK_DIR"/*.lock; do
192	        [[ -f "$lockfile" ]] || continue
193	        local content pid locked_name
194	        content=$(cat "$lockfile" 2>/dev/null)
195	        pid="${content%%:*}"
196	        locked_name="${content##*:}"
197	        [[ -z "$pid" ]] && continue
198	        ! kill -0 "$pid" 2>/dev/null && continue
199	        [[ "$locked_name" == rsync_* ]] && continue
200
201	        local all_descendants
202	        all_descendants=$(pgrep -P "$pid" 2>/dev/null || true)
203
204	        while IFS= read -r rsync_pid; do
205	            [[ -z "$rsync_pid" ]] && continue
206	            local ppid
207	            ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null || echo "")
208	            if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \
209	               [[ "$ppid" == "$pid" ]]; then
210	                echo "${locked_name}:${pid}"
211	                return
212	            fi
213	        done <<< "$rsync_pids"
214	    done
215	    echo ""
216	}
217
218	detect_rsync_parent_remote() {
219	    [[ "$REMOTE_REACHABLE" != true ]] && echo "" && return
220
221	    timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
222	        -o ConnectTimeout="$SSH_TIMEOUT" \
223	        root@"$REMOTE_SERVER" bash << 'REMOTE_SCRIPT' 2>/dev/null
224	LOCK_DIR="/tmp/unraid_locks"
225	rsync_pids=$(pgrep -x rsync 2>/dev/null || true)
226	[[ -z "$rsync_pids" ]] && exit 0
227	for lockfile in "$LOCK_DIR"/*.lock; do
228	    [[ -f "$lockfile" ]] || continue
229	    content=$(cat "$lockfile" 2>/dev/null)
230	    pid="${content%%:*}"
231	    locked_name="${content##*:}"
232	    [[ -z "$pid" ]] && continue
233	    ! kill -0 "$pid" 2>/dev/null && continue
234	    [[ "$locked_name" == rsync_* ]] && continue
235	    all_descendants=$(pgrep -P "$pid" 2>/dev/null || true)
236	    while IFS= read -r rsync_pid; do
237	        [[ -z "$rsync_pid" ]] && continue
238	        ppid=$(awk '/^PPid:/{print $2}' /proc/"$rsync_pid"/status 2>/dev/null || echo "")
239	        if echo "$all_descendants" | grep -qw "$rsync_pid" 2>/dev/null || \
240	           [[ "$ppid" == "$pid" ]]; then
241	            echo "${locked_name}:${pid}"
242	            exit 0
243	        fi
244	    done <<< "$rsync_pids"
245	done
246	REMOTE_SCRIPT
247	}
248
249	LOCAL_ORCH=$(detect_rsync_parent)
250	REMOTE_ORCH=""
251	[[ "$REMOTE_REACHABLE" == true ]] && REMOTE_ORCH=$(detect_rsync_parent_remote)
252
253	# Determine mode
254	if [[ "$FULL_STOP" == true ]]; then
255	    MODE="full-stop"
256	elif [[ -n "$LOCAL_ORCH" ]] || [[ -n "$REMOTE_ORCH" ]]; then
257	    MODE="rsync-only"
258	    [[ -n "$LOCAL_ORCH" ]] && \
259	        warn "Local orchestrator detected: ${LOCAL_ORCH%%:*} — rsync-only mode"
260	    [[ -n "$REMOTE_ORCH" ]] && \
261	        warn "Remote orchestrator detected: ${REMOTE_ORCH%%:*} — rsync-only mode"
262	    warn "Use --full-stop to also kill the orchestrator"
263	else
264	    MODE="rsync-only"
265	    log "No orchestrator detected — killing rsync directly"
266	fi
267
268	# ==============================================================================================
269	# ── Kill Orchestrators (full-stop only) ───────────────────────────────────────────────────────
270	# ==============================================================================================
271	ORCHESTRATORS_KILLED=()
272	REMOTE_ORCHESTRATORS_KILLED=()
273
274	kill_orchestrator() {
275	    local script_name="$1" pid="$2"
276	    local lockfile="$LOCK_DIR/${script_name}.lock"
277
278	    if [[ "$DRY_RUN" == true ]]; then
279	        warn "DRY RUN — would kill $script_name (PID $pid)"
280	        return 0
281	    fi
282
283	    kill -TERM "$pid" 2>/dev/null
284	    sleep 2
285	    kill -0 "$pid" 2>/dev/null && kill -KILL "$pid" 2>/dev/null
286	    sleep 1
287
288	    if ! kill -0 "$pid" 2>/dev/null; then
289	        warn "$script_name stopped (PID $pid) ✅"
290	        rm -f "$lockfile"
291	        return 0
292	    else
293	        error "Failed to kill $script_name (PID $pid)"
294	        return 1
295	    fi
296	}
297
298	if [[ "$MODE" == "full-stop" ]]; then
299	    echo ""
300	    echo "━━━ $ICON_STOP Kill Orchestrators ━━━"
301
302	    if [[ -n "$LOCAL_ORCH" ]]; then
303	        local_name="${LOCAL_ORCH%%:*}"
304	        local_pid="${LOCAL_ORCH##*:}"
305	        warn "Killing local: $local_name (PID $local_pid)"
306	        kill_orchestrator "$local_name" "$local_pid" && \
307	            ORCHESTRATORS_KILLED+=("$local_name")
308	    else
309	        log "No local orchestrator running"
310	    fi
311
312	    if [[ "$REMOTE_REACHABLE" == true ]] && [[ -n "$REMOTE_ORCH" ]]; then
313	        remote_name="${REMOTE_ORCH%%:*}"
314	        remote_pid="${REMOTE_ORCH##*:}"
315	        remote_lock="$LOCK_DIR/${remote_name}.lock"
316	        warn "Killing remote: $remote_name (PID $remote_pid)"
317	        if [[ "$DRY_RUN" == false ]]; then
318	            timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
319	                -o ConnectTimeout="$SSH_TIMEOUT" \
320	                root@"$REMOTE_SERVER" \
321	                "kill -TERM '$remote_pid' 2>/dev/null; sleep 2; \
322	                 kill -0 '$remote_pid' 2>/dev/null && kill -KILL '$remote_pid' 2>/dev/null; \
323	                 rm -f '$remote_lock'" 2>/dev/null
324	            warn "Remote $remote_name stopped ✅"
325	            REMOTE_ORCHESTRATORS_KILLED+=("$remote_name")
326	        else
327	            warn "DRY RUN — would kill remote $remote_name (PID $remote_pid)"
328	        fi
329	    elif [[ "$REMOTE_REACHABLE" == true ]]; then
330	        log "No remote orchestrator running"
331	    fi
332
333	    [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 || \
334	       ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]] && sleep 3
335	fi
336
337	# ==============================================================================================
338	# ━━━ Local Rsync ━━━
339	# ==============================================================================================
340	echo ""
341	echo "━━━ $ICON_STOP Local Rsync ━━━"
342
343	LOCAL_KILLED=false
344	LOCAL_PIDS=$(pgrep -x rsync 2>/dev/null || true)
345
346	if [[ -z "$LOCAL_PIDS" ]]; then
347	    log "No rsync processes running locally"
348	else
349	    warn "Found local rsync PIDs: $(echo "$LOCAL_PIDS" | tr '\n' ' ')"
350	    if [[ "$DRY_RUN" == true ]]; then
351	        warn "DRY RUN — would kill local rsync"
352	    else
353	        pkill -x rsync 2>/dev/null && LOCAL_KILLED=true || \
354	            warn "pkill returned non-zero — rsync may have already exited"
355	        [[ "$LOCAL_KILLED" == true ]] && warn "Local rsync killed ✅"
356	    fi
357	fi
358
359	# Clean stale rsync lock files
360	for lockfile in "$LOCK_DIR"/rsync_*.lock; do
361	    [[ -f "$lockfile" ]] || continue
362	    content=$(cat "$lockfile" 2>/dev/null)
363	    pid="${content%%:*}"
364	    if [[ -n "$pid" ]] && ! kill -0 "$pid" 2>/dev/null; then
365	        log "Cleaning stale lock: $(basename "$lockfile")"
366	        [[ "$DRY_RUN" == false ]] && rm -f "$lockfile"
367	    fi
368	done
369
370	# ==============================================================================================
371	# ━━━ Remote Rsync ━━━
372	# ==============================================================================================
373	echo ""
374	echo "━━━ $ICON_STOP Remote Rsync — $REMOTE_SERVER_NAME ━━━"
375
376	REMOTE_KILLED=false
377
378	if [[ "$REMOTE_REACHABLE" == false ]]; then
379	    warn "Skipping — $REMOTE_SERVER_NAME unreachable"
380	else
381	    REMOTE_PIDS=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
382	        -o ConnectTimeout="$SSH_TIMEOUT" \
383	        root@"$REMOTE_SERVER" "pgrep -x rsync || true" 2>/dev/null || true)
384
385	    if [[ -z "$REMOTE_PIDS" ]]; then
386	        log "No rsync running on $REMOTE_SERVER_NAME"
387	    else
388	        warn "Found remote rsync PIDs: $(echo "$REMOTE_PIDS" | tr '\n' ' ')"
389	        if [[ "$DRY_RUN" == true ]]; then
390	            warn "DRY RUN — would kill remote rsync"
391	        else
392	            timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
393	                -o ConnectTimeout="$SSH_TIMEOUT" \
394	                root@"$REMOTE_SERVER" "pkill -x rsync || true" 2>/dev/null && \
395	                REMOTE_KILLED=true || \
396	                warn "Remote pkill returned non-zero — rsync may have already exited"
397	            [[ "$REMOTE_KILLED" == true ]] && warn "Remote rsync killed ✅"
398	        fi
399	    fi
400	fi
401
402	# ==============================================================================================
403	# ━━━ Container Recovery ━━━
404	# ==============================================================================================
405	# Restart local containers left stopped by interrupted rsync.
406	# Remote containers left for docker_watchdog.sh to recover.
407	# Skipped with --rsync-only flag (called by other scripts that handle recovery themselves).
408	CONTAINERS_RESTARTED=()
409	CONTAINERS_FAILED=()
410
411	if [[ "$RSYNC_ONLY_MODE" == false ]] && \
412	   { [[ "$LOCAL_KILLED" == true ]] || [[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; }; then
413
414	    echo ""
415	    echo "━━━ $ICON_START Container Recovery ━━━"
416	    log "Checking profile containers for recovery..."
417
418	    declare -A SEEN
419	    ALL_CONTAINERS=()
420
421	    for profile_containers in "${PROFILE_CRITICAL_CONTAINER_NAMES[@]:-}"; do
422	        read -r -a container_list <<< "$profile_containers"
423	        for c in "${container_list[@]:-}"; do
424	            [[ -z "$c" ]] && continue
425	            if [[ -z "${SEEN[$c]:-}" ]]; then
426	                SEEN[$c]=1
427	                ALL_CONTAINERS+=("$c")
428	            fi
429	        done
430	    done
431
432	    if [[ ${#ALL_CONTAINERS[@]} -eq 0 ]]; then
433	        log "No profile containers defined — skipping recovery"
434	    else
435	        for c in "${ALL_CONTAINERS[@]}"; do
436	            STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
437	                '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown")
438	            case "$STATUS" in
439	                true)
440	                    log "$c — running ✅"
441	                    ;;
442	                false)
443	                    warn "$c — stopped — restarting..."
444	                    if [[ "$DRY_RUN" == true ]]; then
445	                        warn "DRY RUN — would restart $c"
446	                    else
447	                        if timeout "$DOCKER_TIMEOUT" docker start "$c" >/dev/null 2>&1; then
448	                            warn "$c restarted ✅"
449	                            CONTAINERS_RESTARTED+=("$c")
450	                        else
451	                            error "Failed to restart $c"
452	                            CONTAINERS_FAILED+=("$c")
453	                        fi
454	                    fi
455	                    ;;
456	                *)
457	                    log "$c not found locally — skipping"
458	                    ;;
459	            esac
460	        done
461	    fi
462	fi
463
464	# ==============================================================================================
465	# ━━━ Summary ━━━
466	# ==============================================================================================
467	echo ""
468	echo "━━━━━ $ICON_SUMMARY RSYNC STOP SUMMARY ━━━━━"
469	echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
470	echo "$ICON_GEAR Mode:     $MODE"
471	echo ""
472
473	echo "$ICON_HOST Local ($MY_ID):"
474	[[ ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \
475	    warn "  Orchestrators killed: ${ORCHESTRATORS_KILLED[*]}"
476	if [[ "$LOCAL_KILLED" == true ]]; then
477	    warn "  Rsync killed ✅"
478	else
479	    log "  No rsync was running"
480	fi
481
482	echo "$ICON_NET  Remote ($REMOTE_ID — $REMOTE_SERVER_NAME):"
483	if [[ "$REMOTE_REACHABLE" == false ]]; then
484	    warn "  Unreachable — skipped"
485	else
486	    [[ ${#REMOTE_ORCHESTRATORS_KILLED[@]} -gt 0 ]] && \
487	        warn "  Orchestrators killed: ${REMOTE_ORCHESTRATORS_KILLED[*]}"
488	    if [[ "$REMOTE_KILLED" == true ]]; then
489	        warn "  Rsync killed ✅"
490	    else
491	        log "  No rsync was running"
492	    fi
493	fi
494
495	[[ ${#CONTAINERS_RESTARTED[@]} -gt 0 ]] && \
496	    warn "$ICON_CONTAINERS Containers recovered: ${CONTAINERS_RESTARTED[*]}"
497	[[ ${#CONTAINERS_FAILED[@]} -gt 0 ]] && \
498	    echo "$ICON_ERROR Containers failed to restart: ${CONTAINERS_FAILED[*]}"
499
500	echo ""
501	if [[ "$DRY_RUN" == true ]]; then
502	    warn "DRY RUN — no changes made"
503	else
504	    echo "$ICON_DONE Status: done ✅"
505	fi
506	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
507
508	# Notify if anything was actually killed or failed
509	if [[ "$DRY_RUN" == false ]]; then
510	    if [[ ${#CONTAINERS_FAILED[@]} -gt 0 ]]; then
511	        notify "Rsync stop on $(hostname) ($MY_ID) — containers failed to restart: ${CONTAINERS_FAILED[*]}" \
512	            "Rsync Stop" "warning"
513	    elif [[ "$LOCAL_KILLED" == true || "$REMOTE_KILLED" == true || \
514	            ${#ORCHESTRATORS_KILLED[@]} -gt 0 ]]; then
515	        notify "Rsync stopped on $(hostname) ($MY_ID) — mode: $MODE${CONTAINERS_RESTARTED:+ — recovered: ${CONTAINERS_RESTARTED[*]}}" \
516	            "Rsync Stop" "warning"
517	    fi
518	fi


1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= Conf Cache Sync ================================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Maintains a RAM-resident conf cache at /tmp/.vv/config/cached/.confs/.
9	# Credentials and partner keys live in RAM only — never on disk across hosts.
10	#
11	# On array start (default / --array-start):
12	#   1. Copy own conf to local cache
13	#   2. Pull each available partner's conf from their disk → local cache
14	#   3. Push own conf to each available partner's /tmp/.vv/ cache
15	#
16	# On conf save (--push-only):
17	#   Fast path — push updated own conf to all partners' /tmp/.vv/ cache only.
18	#   No pulls, no local cache rebuild.
19	#
20	# Cache is /tmp (tmpfs) — cleared every reboot, repopulated by this script
21	# on next array start. Scripts source from cache for partner vars; own vars
22	# always come from disk (load_config.sh skips cached copy of own conf).
23	#
24	# ==============================================================================================
25	# RUNTIME MODES
26	# ==============================================================================================
27	#
28	#   conf_sync.sh                   Full sync: pull from all partners + push to all partners
29	#   conf_sync.sh --push-only       Push own conf to all partners (fast, for conf-save hook)
30	#   conf_sync.sh --pull-only       Pull partner confs into local cache only (for intermediate orch)
31	#   conf_sync.sh --dry-run         Show what would happen, no changes
32	#   conf_sync.sh --log             Verbose output
33	#
34	# ==============================================================================================
35
36	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
37	source "$SCRIPT_DIR/../load_config.sh"
38
39	PUSH_ONLY=false
40	PULL_ONLY=false
41	FILTERED_ARGS=()
42	for arg in "$@"; do
43	    case "$arg" in
44	        --push-only) PUSH_ONLY=true ;;
45	        --pull-only) PULL_ONLY=true ;;
46	        *)           FILTERED_ARGS+=("$arg") ;;
47	    esac
48	done
49	parse_args "${FILTERED_ARGS[@]}"
50	detect_hosts
51
52	CACHE_DIR="/tmp/.vv/config/cached/.confs"
53	MY_CONF="$SCRIPTS_ROOT/Configurations/${MY_ID,,}.conf"
54	SSH_TIMEOUT=10
55
56	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
57
58	# ── Ensure cache dir exists ───────────────────────────────────────────────────
59	if [[ "$DRY_RUN" == false ]]; then
60	    mkdir -p "$CACHE_DIR"
61	fi
62
63	# ── Copy own conf into local cache ───────────────────────────────────────────
64	if [[ "$PUSH_ONLY" == false ]] && [[ "$PULL_ONLY" == false ]]; then
65	    if [[ -f "$MY_CONF" ]]; then
66	        if [[ "$DRY_RUN" == true ]]; then
67	            warn "DRY RUN — would copy $(basename "$MY_CONF") → $CACHE_DIR/"
68	        else
69	            cp "$MY_CONF" "$CACHE_DIR/${MY_ID,,}.conf" && \
70	                log "Own conf cached ✅" || warn "Failed to cache own conf"
71	        fi
72	    else
73	        warn "Own conf not found: $MY_CONF"
74	    fi
75	fi
76
77	# ── Per-partner sync ──────────────────────────────────────────────────────────
78	PUSHED=0
79	PULLED=0
80	FAILED=0
81
82	for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
83	    partner_host="${!host_var}"
84	    [[ -z "$partner_host" ]] && continue
85	    [[ "${host_var,,}" == "${MY_ID,,}" ]] && continue
86
87	    partner_slot="${host_var,,}"   # e.g. host2
88	    partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
89
90	    if [[ -z "$partner_ip" ]]; then
91	        warn "$partner_host — cannot resolve Tailscale IP, skipping"
92	        (( FAILED++ ))
93	        continue
94	    fi
95
96	    # ── Pull: grab partner's conf from their disk → our local cache ──────────
97	    if [[ "$PUSH_ONLY" == false ]]; then
98	        remote_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
99	        if [[ "$DRY_RUN" == true ]]; then
100	            warn "DRY RUN — would pull $partner_host:$remote_conf → $CACHE_DIR/${partner_slot}.conf"
101	        elif timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
102	            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
103	            "root@${partner_ip}:${remote_conf}" \
104	            "$CACHE_DIR/${partner_slot}.conf" 2>/dev/null; then
105	            log "Pulled ${partner_slot}.conf from $partner_host ✅"
106	            (( PULLED++ ))
107	        else
108	            warn "Could not pull ${partner_slot}.conf from $partner_host"
109	            (( FAILED++ ))
110	        fi
111	    fi
112
113	    # ── Push: send own conf to partner's /tmp/.vv/ cache ────────────────────
114	    if [[ "$PULL_ONLY" == true ]]; then
115	        continue
116	    fi
117
118	    if [[ "$DRY_RUN" == true ]]; then
119	        warn "DRY RUN — would push ${MY_ID,,}.conf → $partner_host:/tmp/.vv/config/cached/.confs/"
120	        continue
121	    fi
122
123	    # Ensure partner's cache dir exists, then SCP own conf into it
124	    timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
125	        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
126	        "root@${partner_ip}" "mkdir -p '$CACHE_DIR'" 2>/dev/null
127
128	    if timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
129	        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
130	        "$MY_CONF" \
131	        "root@${partner_ip}:${CACHE_DIR}/${MY_ID,,}.conf" 2>/dev/null; then
132	        log "Pushed ${MY_ID,,}.conf to $partner_host ✅"
133	        (( PUSHED++ ))
134	    else
135	        warn "Could not push to $partner_host"
136	        (( FAILED++ ))
137	    fi
138	done
139
140	# ── Summary ───────────────────────────────────────────────────────────────────
141	if [[ "$PUSH_ONLY" == true ]]; then
142	    info "Conf push complete — pushed to $PUSHED host(s)${FAILED:+, $FAILED failed}"
143	elif [[ "$PULL_ONLY" == true ]]; then
144	    info "Conf pull complete — pulled $PULLED partner conf(s)${FAILED:+, $FAILED failed}"
145	else
146	    info "Conf sync complete — pulled $PULLED, pushed $PUSHED${FAILED:+, $FAILED failed}"
147	fi
148


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Array Start Orchestrator ===================================
4	# ==============================================================================================
5	# Single entry point for array start — fired by the Varaverk plugin's
6	# disks_mounted event hook (Plugin/unraid/event/disks_mounted/array_start_jobs).
7	# Launches everything configured in ARRAY_START_SCRIPTS in master.conf.
8	# This script exits after launching all scripts — the event hook sees it complete normally.
9	#
10	# ── WHAT IT LAUNCHES ──────────────────────────────────────────────────────────────────────────
11	# Configured in master.conf ARRAY_START_SCRIPTS — no changes to this script ever needed.
12	# Current order (order matters — see below):
13	#
14	#   ONE-SHOT (run and exit naturally):
15	#     System_Essentials/unraid_api_key_renew.sh — re-register Varaverk API key at boot
16	#     System_Essentials/inotify_tuning.sh       — raise inotify limits before containers start
17	#     System_Essentials/docker_syslog_filter.sh — suppress veth log noise before logs fill
18	#     System_Essentials/php_fpm_max_children.sh — WebGUI performance tuning
19	#     Transcodes/ramdisk_setup.sh               — create tmpfs + symlink before Emby starts
20	#     Docker_Essentials/docker_network_connect.sh — ensure networks + container connections
21	#
22	#   CONTINUOUS (run until array stops):
23	#     Fallback/fallback.sh                      — mutual fallback monitor
24	#
25	#   NOTE: watchdogs (docker, system, stability) are NOT launched here.
26	#   They run via watchdog_orchestrator.sh every 15 min (cron), not as daemons.
27	#
28	# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
29	#   unraid_api_key_renew.sh — before anything else — self-heals API registry at boot
30	#   inotify_tuning.sh     — must run BEFORE Code-Server and other containers start
31	#                           containers that start with low inotify limits keep them ✅
32	#   docker_syslog_filter  — must run BEFORE any container starts creating veth interfaces
33	#   ramdisk_setup.sh      — must run BEFORE Emby starts transcoding
34	#   docker_network_connect — must run BEFORE watchdogs check container states
35	#   fallback.sh           — last — needs everything else stable to make decisions
36	#
37	# ── ONE-SHOT vs CONTINUOUS DETECTION ─────────────────────────────────────────────────────────
38	# Script is launched in background with bash script.sh &
39	# After 1 second: if PID still alive → continuous (running in background)
40	#                  if PID dead + exit 0 → one-shot completed successfully
41	#                  if PID dead + exit N → failure
42	#
43	# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
44	#   Root check           — all launched scripts require root
45	#   acquire_lock         — prevents duplicate array start launches
46	#   detect_hosts()       — MY_ID in notifications
47	#   platform_require_cmd  — notify validated before use
48	#   chmod +x auto-fix    — non-executable scripts fixed before launch
49	#   Full path on failure — shows exact path for debugging
50	#   notify on failures   — alert if any script fails to launch
51	#
52	# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
53	#   ARRAY_START_SCRIPTS — ordered list of scripts to launch at array start
54	#
55	# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
56	#   array_started.sh              — normal launch (called by Varaverk disks_mounted event hook)
57	#   array_started.sh --dry-run    — show what would be launched without launching
58	#   array_started.sh --status     — show configured scripts and their current state
59	#   array_started.sh --log        — verbose output per script
60	# ==============================================================================================
61
62	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
63	ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
64
65	source "$ECOSYSTEM_ROOT/load_config.sh"
66
67	parse_args "$@"
68
69	# ==============================================================================================
70	# ━━━ Setup ━━━
71	# ==============================================================================================
72	if [[ "$EUID" -ne 0 ]]; then
73	    error "Must be run as root"
74	    exit 1
75	fi
76
77	platform_require_cmd \
78	    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
79	    "" "" \
80	    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
81
82	acquire_lock
83
84	detect_hosts
85
86	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — scripts will not be launched"
87
88	# ==============================================================================================
89	# ━━━ Status ━━━
90	# ==============================================================================================
91	if [[ "$SHOW_STATUS" == true ]]; then
92	    echo ""
93	    echo "━━━━━ $ICON_SUMMARY ARRAY START STATUS ━━━━━"
94	    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
95	    echo "$ICON_GEAR  Scripts:  ${#ARRAY_START_SCRIPTS[@]} configured"
96	    echo ""
97
98	    for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
99	        [[ -z "$relative_path" ]] && continue
100	        script_path="$ECOSYSTEM_ROOT/$relative_path"
101	        script_name=$(basename "$script_path")
102
103	        if [[ ! -f "$script_path" ]]; then
104	            echo "  $ICON_ERROR $script_name — FILE NOT FOUND"
105	            echo "    $script_path"
106	            continue
107	        fi
108
109	        [[ ! -x "$script_path" ]] && flag=" (not executable — will auto-fix)" || flag=""
110
111	        # Check if currently running
112	        if pgrep -f "$script_path" >/dev/null 2>&1; then
113	            RUN_PID=$(pgrep -f "$script_path" | head -1)
114	            echo "  $ICON_RUNNING $script_name — RUNNING (PID $RUN_PID)${flag}"
115	        else
116	            echo "  $ICON_NOT_RUNNING $script_name — not running${flag}"
117	        fi
118	    done
119	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
120	    exit 0
121	fi
122
123	# ==============================================================================================
124	# ━━━ Launch Scripts ━━━
125	# ==============================================================================================
126	echo ""
127	echo "━━━ $ICON_GEAR Array Start — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
128	log "Ecosystem root: $ECOSYSTEM_ROOT"
129	log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..."
130	echo ""
131
132	START=$(date +%s)
133	LAUNCHED=0
134	FAILED=0
135	FAILED_SCRIPTS=()
136
137	for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
138	    [[ -z "$relative_path" ]] && continue
139
140	    SCRIPT_PATH="$ECOSYSTEM_ROOT/$relative_path"
141	    SCRIPT_NAME=$(basename "$SCRIPT_PATH")
142
143	    # File existence check
144	    if [[ ! -f "$SCRIPT_PATH" ]]; then
145	        error "$SCRIPT_NAME — not found"
146	        error "  Expected: $SCRIPT_PATH"
147	        (( FAILED++ ))
148	        FAILED_SCRIPTS+=("$SCRIPT_NAME")
149	        continue
150	    fi
151
152	    # Auto-fix permissions — chmod +x if needed
153	    if [[ ! -x "$SCRIPT_PATH" ]]; then
154	        warn "$SCRIPT_NAME — not executable, fixing..."
155	        chmod +x "$SCRIPT_PATH" || {
156	            error "$SCRIPT_NAME — chmod +x failed"
157	            (( FAILED++ ))
158	            FAILED_SCRIPTS+=("$SCRIPT_NAME")
159	            continue
160	        }
161	    fi
162
163	    if [[ "$DRY_RUN" == true ]]; then
164	        warn "DRY RUN — would launch: $SCRIPT_NAME"
165	        (( LAUNCHED++ ))
166	        continue
167	    fi
168
169	    log "$ICON_START Launching $SCRIPT_NAME..."
170	    bash "$SCRIPT_PATH" &
171	    PID=$!
172
173	    # Brief settle — 1s enough to detect immediate failures
174	    sleep 1
175
176	    if kill -0 "$PID" 2>/dev/null; then
177	        # Still running → continuous script
178	        warn "$SCRIPT_NAME — running (PID $PID) ✅"
179	        (( LAUNCHED++ ))
180	    else
181	        # Exited — check if one-shot success or failure
182	        wait "$PID"
183	        EXIT_CODE=$?
184	        if [[ "$EXIT_CODE" -eq 0 ]]; then
185	            log "$SCRIPT_NAME — completed (one-shot) ✅"
186	            (( LAUNCHED++ ))
187	        else
188	            error "$SCRIPT_NAME — exited with code $EXIT_CODE"
189	            error "  Path: $SCRIPT_PATH"
190	            (( FAILED++ ))
191	            FAILED_SCRIPTS+=("$SCRIPT_NAME")
192	        fi
193	    fi
194
195	done
196
197	END=$(date +%s)
198
199	# ==============================================================================================
200	# ━━━ Summary ━━━
201	# ==============================================================================================
202	echo ""
203	echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
204	echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
205	echo "$ICON_SUCCESS Launched: $LAUNCHED"
206	[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR  Failed:   $FAILED — ${FAILED_SCRIPTS[*]}"
207	echo "$ICON_TIME    Duration: $(format_duration $(( END - START )))"
208	echo ""
209
210	if [[ "$DRY_RUN" == true ]]; then
211	    warn "DRY RUN — no scripts launched"
212	elif [[ "$FAILED" -gt 0 ]]; then
213	    warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}"
214	    notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
215	        "Array Start" "warning"
216	else
217	    echo "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
218	fi
219	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"


Now let me check the watchdog scripts and PHP API endpoints:


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Docker Watchdog ============================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Two-tier self-healing container monitoring system. Called by
9	# watchdog_orchestrator.sh via cron every 15 minutes as a single-pass run.
10	# Tier 1 applies specific thresholds to explicitly configured containers.
11	# Tier 2 scans everything else for generic health problems. Silent on clean
12	# cycles, loud when something needs attention.
13	#
14	# ==============================================================================================
15	# OPERATIONAL MODEL
16	# ==============================================================================================
17	#
18	# Tier 1 — Strict Per-Container Monitoring
19	#   Applies only to containers explicitly configured in host*.conf.
20	#
21	#   Memory hard limits    — immediate restart if container exceeds MB ceiling
22	#   Memory soft threshold — warn at SOFT_MEM_THRESHOLD % of limit (no restart)
23	#   CPU thresholds        — strike system: warn at SOFT_CPU_THRESHOLD, restart after
24	#                           CPU_FAIL_LIMIT consecutive strikes at HARD_CPU_THRESHOLD
25	#   HTTP responsiveness   — strike system: restart after RESP_FAIL_LIMIT consecutive
26	#                           failures against the configured endpoint
27	#   Required containers   — must always be running; strike system before restart;
28	#                           skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in window
29	#
30	# Tier 2 — Global Health Scan
31	#   Scans ALL running containers when WATCHDOG_SCAN_ALL=true.
32	#   Containers in WATCHDOG_SCAN_IGNORE are excluded.
33	#
34	#   Unhealthy status      — Docker HEALTHCHECK unhealthy → restart
35	#   OOM killed            — kernel OOM kill detected → restart + notify
36	#   Crash loop            — RestartCount climbing → notify; above WATCHDOG_CRASH_LIMIT
37	#                           → restart → skip list if restart limit hit
38	#   Dead containers       — remove + start (dead state cannot be restarted directly)
39	#   Unexpected exits      — non-zero exit code → restart
40	#
41	# Cross-Cutting Intelligence
42	#   Applies to both tiers on every cycle.
43	#
44	#   Startup grace period  — no restarts for WATCHDOG_STARTUP_GRACE seconds after boot
45	#   Dependency ordering   — dependency restarted first, dependent skipped this cycle
46	#   Restart loop protect  — skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in window
47	#   Skip list auto-clear  — removed when container seen running again
48	#   Notification batching — one summary per cycle, not one ping per event
49	#   Parity awareness      — skips restart actions during parity check
50	#   Timeout protection    — all docker commands wrapped in timeout
51	#   Docker daemon check   — each cycle begins with daemon health check; hung daemon →
52	#                           restart via rc.docker → stability_watchdog.sh escalates if needed
53	#   RAM emergency defer   — reads RW_STATE_FILE; stands down while
54	#                           resource_watchdog.sh is managing a RAM emergency
55	#
56	# ==============================================================================================
57	# DESIGN PRINCIPLES
58	# ==============================================================================================
59	#
60	# Tiered Monitoring
61	#   Not all containers need the same monitoring strategy. Tier 1 gives explicit
62	#   control over the containers that matter most. Tier 2 is the catch-all that
63	#   requires no configuration and protects everything else.
64	#
65	# Strike vs Immediate
66	#   CPU spikes and HTTP failures are transient — brief spikes are normal during
67	#   transcoding or library scans. Memory leaks are not transient. CPU and HTTP
68	#   use a strike system to distinguish sustained problems from momentary ones.
69	#   Memory triggers immediate restart because a container at its ceiling is
70	#   actively leaking, not spiking.
71	#
72	# Loop Protection Over Persistence
73	#   A watchdog that keeps restarting a broken container is not helpful — it risks
74	#   making a database corruption worse. After WATCHDOG_CONTAINER_RESTART_LIMIT
75	#   attempts the container is skip-listed and the operator is notified. Automated
76	#   recovery stops. Human investigation begins.
77	#
78	# Dependency-Safe Ordering
79	#   When a container and its dependency are both down, restart the dependency
80	#   first and skip the dependent this cycle. Prevents false-alarm skip-listing
81	#   of containers whose only failure was starting before their dependency was ready.
82	#
83	# Silent When Healthy
84	#   Runs 96 times per day. Producing output on every clean cycle would make
85	#   logs useless. Output only when something needs attention or a heartbeat fires.
86	#
87	# ==============================================================================================
88	# OPERATIONAL SAFEGUARDS
89	# ==============================================================================================
90	#
91	# Lock Acquisition
92	#   Prevents concurrent execution via acquire_lock(). Safe at array start —
93	#   only one watchdog instance runs at a time.
94	#
95	# Host Detection
96	#   detect_hosts() identifies which server is running the script and aliases
97	#   all HOST*_WATCHDOG_* arrays to the correct host's values.
98	#
99	# Startup Grace Period
100	#   Restart actions suppressed for WATCHDOG_STARTUP_GRACE seconds after the
101	#   watchdog starts. Checks still run and log — only restarts are suppressed.
102	#   Prevents false-positive restarts while containers are still initialising.
103	#
104	# Restart Loop Protection
105	#   WATCHDOG_CONTAINER_RESTART_LIMIT restarts within WATCHDOG_CONTAINER_RESTART_WINDOW
106	#   hours triggers skip-listing and a critical notification. Skip list persists on
107	#   /boot/config/ — survives reboots intentionally. Auto-clears when container
108	#   is seen running again.
109	#
110	# Docker Daemon Health Check
111	#   First operation every cycle. Daemon not responding within DOCKER_TIMEOUT →
112	#   restart via /etc/rc.d/rc.docker → verify recovery. If still hung: log
113	#   critical, skip cycle. stability_watchdog.sh handles further escalation.
114	#
115	# RAM Emergency Deferral
116	#   Reads RW_STATE_FILE each cycle. If resource_watchdog.sh has set
117	#   mem_shutdown_active=true, all restart logic defers until the flag clears.
118	#   Stale state guard: if file is >2 hours old with flag still set,
119	#   resource_watchdog.sh has likely stopped — watchdog resumes normal operation.
120	#
121	# Timeout Protection
122	#   All docker commands wrapped in timeout. Daemon hangs cannot stall the
123	#   watchdog and leave containers unmonitored between cycles.
124	#
125	# Notification Batching
126	#   Events collected across a full cycle and sent as a single summary.
127	#   Prevents notification floods when a shared dependency failure cascades.
128	#
129	# ==============================================================================================
130	# STATE FILES
131	# ==============================================================================================
132	#
133	#   WATCHDOG_STATE_FILE            — strike counts, daemon flags (STATE_DIR — survives reboots)
134	#   DOCKER_WATCHDOG_FAILED_FILE       — container skip list (STATE_DIR — survives reboots)
135	#   WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR)
136	#   RW_STATE_FILE                  — read-only: resource_watchdog RAM emergency flag
137	#
138	# ==============================================================================================
139	# CONFIGURATION
140	# ==============================================================================================
141	#
142	# host*.conf
143	#
144	#   HOST*_WATCHDOG_CONTAINERS
145	#       Memory hard limits per container. Format: "ContainerName:LimitInMB"
146	#       Aliased by detect_hosts() → WATCHDOG_CONTAINERS
147	#
148	#   HOST*_WATCHDOG_CONTAINER_URLS
149	#       HTTP health check endpoints. Format: "ContainerName:http://host:port"
150	#       Aliased by detect_hosts() → WATCHDOG_CONTAINER_URLS
151	#
152	#   HOST*_WATCHDOG_CONTAINER_API_CHECKS
153	#       API liveness checks — deeper than HTTP. Format: "ContainerName:URL|APIKey"
154	#       Use endpoints that require a live DB round-trip (e.g. Emby /System/Info).
155	#       Aliased by detect_hosts() → WATCHDOG_CONTAINER_API_CHECKS
156	#
157	#   HOST*_WATCHDOG_REQUIRED_CONTAINERS
158	#       Containers that must always be running. Aliased by detect_hosts() →
159	#       WATCHDOG_REQUIRED_CONTAINERS
160	#
161	#   HOST*_WATCHDOG_SCAN_IGNORE
162	#       Containers excluded from Tier 2 global scan. Aliased by detect_hosts() →
163	#       WATCHDOG_SCAN_IGNORE
164	#
165	#   HOST*_WATCHDOG_DEPENDENCIES
166	#       Dependency ordering. Format: "Dependent:dep1 dep2". Aliased by
167	#       detect_hosts() → WATCHDOG_DEPENDENCIES
168	#
169	# master.conf
170	#
171	#   WATCHDOG_STARTUP_GRACE
172	#       Seconds before restart actions begin after watchdog starts (default: 600)
173	#
174	#   SOFT_MEM_THRESHOLD
175	#       Warn at this % of hard memory limit — no restart (default: 80)
176	#
177	#   SOFT_CPU_THRESHOLD / HARD_CPU_THRESHOLD / CPU_FAIL_LIMIT
178	#       CPU monitoring thresholds and strike limit
179	#
180	#   CURL_TIMEOUT / RESP_FAIL_LIMIT
181	#       HTTP health check timeout and consecutive failure limit
182	#
183	#   WATCHDOG_SCAN_ALL
184	#       Enable Tier 2 global health scan (default: true)
185	#
186	#   WATCHDOG_RESTART_UNHEALTHY / WATCHDOG_RESTART_DEAD / WATCHDOG_RESTART_CRASHED
187	#       Tier 2 action toggles
188	#
189	#   WATCHDOG_NOTIFY_OOM / WATCHDOG_NOTIFY_CRASHLOOP / WATCHDOG_CRASH_LIMIT
190	#       OOM and crash loop detection toggles and threshold
191	#
192	#   WATCHDOG_CONTAINER_RESTART_LIMIT / WATCHDOG_CONTAINER_RESTART_WINDOW
193	#       Restart loop protection: attempt limit and rolling window in hours
194	#
195	#   WATCHDOG_BATCH_NOTIFY
196	#       Collect cycle events and send as one notification (default: true)
197	#
198	#   DOCKER_WATCHDOG_HEARTBEAT_HOURS
199	#       Hours between alive heartbeat log entries
200	#


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= System Watchdog ============================================
4	# ==============================================================================================
5	# Thin orchestrator — runs SYSTEM_WATCHDOG_SCRIPTS from master.conf sequentially.
6	# Called by watchdog_orchestrator.sh each cycle. Covers system component health:
7	# storage pool growth, runaway logs, WebGUI availability, and network connectivity.
8	#
9	# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
10	#   Driven by SYSTEM_WATCHDOG_SCRIPTS in master.conf — add, remove, or reorder there.
11	#   Default: storage_watchdog → webgui_watchdog → network_watchdog
12	#
13	# ── SEQUENTIAL EXECUTION ─────────────────────────────────────────────────────────────────────
14	# All scripts run in the foreground. Each must complete before the next starts.
15	# A failed script is logged but does not prevent remaining scripts from running.
16	#
17	# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
18	#   Root check           — child scripts require root
19	#   acquire_lock         — prevents concurrent system watchdog runs
20	#   detect_hosts()       — MY_ID in notifications and logs
21	#   Non-fatal steps      — a failed step is logged; remaining steps still run
22	#
23	# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
24	#   SYSTEM_WATCHDOG_SCRIPTS — ordered list of system component watchdog scripts to run
25	#
26	# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
27	#   system_watchdog.sh              — run all system component watchdogs
28	#   system_watchdog.sh --dry-run    — preview without running anything
29	#   system_watchdog.sh --status     — show configured scripts and exit
30	#   system_watchdog.sh --log        — verbose output
31	# ==============================================================================================
32
33	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
34	ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
35
36	source "$ECOSYSTEM_ROOT/load_config.sh"
37
38	parse_args "$@"
39
40	# ==============================================================================================
41	# ━━━ Setup ━━━
42	# ==============================================================================================
43	if [[ "$EUID" -ne 0 ]]; then
44	    error "Must be run as root"
45	    exit 1
46	fi
47
48	platform_require_cmd \
49	    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
50	    "" "" \
51	    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
52
53	acquire_lock
54
55	detect_hosts
56
57	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no watchdog scripts will be executed"
58
59	# ==============================================================================================
60	# ━━━ Status ━━━
61	# ==============================================================================================
62	if [[ "$SHOW_STATUS" == true ]]; then
63	    echo ""
64	    echo "━━━━━ $ICON_SUMMARY SYSTEM WATCHDOG STATUS ━━━━━"
65	    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
66	    echo "$ICON_GEAR  Scripts:  ${#SYSTEM_WATCHDOG_SCRIPTS[@]} configured"
67	    echo ""
68
69	    for entry in "${SYSTEM_WATCHDOG_SCRIPTS[@]}"; do
70	        [[ -z "$entry" ]] && continue
71	        read -r -a parts <<< "$entry"
72	        script_path="$ECOSYSTEM_ROOT/${parts[0]}"
73	        script_name=$(basename "${parts[0]}")
74	        if [[ ! -f "$script_path" ]]; then
75	            echo "  $ICON_ERROR $script_name — FILE NOT FOUND"
76	        else
77	            echo "  $ICON_GEAR $script_name"
78	        fi
79	    done
80	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
81	    exit 0
82	fi
83
84	# ==============================================================================================
85	# ━━━ Run Sequence ━━━
86	# ==============================================================================================
87	echo "━━━ $ICON_SHIELD System Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
88
89	START=$(date +%s)
90	PASSED=()
91	FAILED=()
92	STEP=0
93
94	for entry in "${SYSTEM_WATCHDOG_SCRIPTS[@]}"; do
95	    [[ -z "$entry" ]] && continue
96	    (( STEP++ ))
97
98	    read -r -a parts <<< "$entry"
99	    script_path="$ECOSYSTEM_ROOT/${parts[0]}"
100	    script_name=$(basename "${parts[0]}")
101
102	    if [[ ! -f "$script_path" ]]; then
103	        error "$script_name — not found at $script_path"
104	        FAILED+=("$script_name")
105	        continue
106	    fi
107
108	    if [[ ! -x "$script_path" ]]; then
109	        chmod +x "$script_path" || {
110	            error "$script_name — chmod +x failed"
111	            FAILED+=("$script_name")
112	            continue
113	        }
114	    fi
115
116	    if [[ "$DRY_RUN" == true ]]; then
117	        warn "DRY RUN — would run: $script_name"
118	        PASSED+=("$script_name")
119	        continue
120	    fi
121
122	    _ss=$(date +%s)
123	    if bash "$script_path"; then
124	        log "$script_name — done in $(format_duration $(( $(date +%s) - _ss ))) ✅"
125	        PASSED+=("$script_name")
126	    else
127	        warn "$script_name — exit non-zero in $(format_duration $(( $(date +%s) - _ss ))) (issues found or fixed) — continuing"
128	        FAILED+=("$script_name")
129	    fi
130	done
131
132	END=$(date +%s)
133
134	# ==============================================================================================
135	# ━━━ Summary ━━━
136	# ==============================================================================================
137	log "System watchdog — $STEP script(s) — $(format_duration $(( END - START )))"
138
139	[[ ${#FAILED[@]} -gt 0 ]] && exit 1
140	exit 0
141


Now let me look at PHP API endpoints and search for failure patterns:


Bash

find /boot/config/plugins/varaverk -name "*.php" -type f 2>/dev/null | head -20


1	<?php
2	header('Content-Type: application/json');
3	require_once dirname(__DIR__) . '/include/scheduler.php';
4
5	$id = trim($_POST['id'] ?? '');
6
7	if (!$id || !preg_match('/^[a-zA-Z0-9_.\/\-]+\.sh$/', $id) || str_contains($id, '..')) {
8	    echo json_encode(['ok' => false, 'error' => 'Invalid id']);
9	    exit;
10	}
11
12	$script = SCRIPTS_DIR . '/' . $id;
13	if (!file_exists($script)) {
14	    echo json_encode(['ok' => false, 'error' => 'Script not found: ' . $id]);
15	    exit;
16	}
17
18	$logFile = vv_job_log_path($id);
19	$logDir  = dirname($logFile);
20	if (!is_dir($logDir)) mkdir($logDir, 0755, true);
21
22	$location = trim($_POST['location'] ?? '');
23	if ($location && (!str_starts_with($location, '/') || str_contains($location, '..') || preg_match('/[\x00\n\r]/', $location))) {
24	    echo json_encode(['ok' => false, 'error' => 'Invalid location']);
25	    exit;
26	}
27
28	$extra_args = trim($_POST['extra_args'] ?? '');
29	if ($extra_args && preg_match('/[;&|`$<>\\\\"\']/', $extra_args)) {
30	    echo json_encode(['ok' => false, 'error' => 'Invalid extra_args']);
31	    exit;
32	}
33
34	// Refuse if already running — scripts will lock-exit anyway, but surface it clearly.
35	$statFile = vv_job_stat_path($id);
36	if (file_exists($statFile)) {
37	    $stat   = json_decode(file_get_contents($statFile), true) ?: [];
38	    $pid    = $stat['pid'] ?? null;
39	    $status = $stat['status'] ?? '';
40	    if ($status === 'running' && $pid && file_exists("/proc/{$pid}")) {
41	        echo json_encode(['ok' => false, 'already_running' => true, 'error' => 'Already running']);
42	        exit;
43	    }
44	}
45
46	$runner   = dirname(__DIR__) . '/run_job.sh';
47	$flags    = vv_job_flags($id);
48	$locArg   = $location   ? ' ' . escapeshellarg('--location=' . $location) : '';
49	$extraStr = $extra_args ? ' ' . $extra_args : '';
50	exec('nohup bash ' . escapeshellarg($runner) . ' ' . escapeshellarg($id) . ' ' . escapeshellarg($script) . ($flags ? " $flags" : '') . ' --manual' . $locArg . $extraStr . ' >> ' . escapeshellarg($logFile) . ' 2>&1 </dev/null &');
51
52	echo json_encode(['ok' => true]);
53


1	<?php
2	header('Content-Type: application/json');
3	require_once dirname(__DIR__) . '/include/scheduler.php';
4
5	$id = trim($_POST['id'] ?? '');
6
7	if (!$id || !preg_match('/^[a-zA-Z0-9_.\/\-]+\.sh$/', $id) || str_contains($id, '..')) {
8	    echo json_encode(['ok' => false, 'error' => 'Invalid id']);
9	    exit;
10	}
11
12	$statFile = vv_job_stat_path($id);
13	if (!file_exists($statFile)) {
14	    echo json_encode(['ok' => false, 'error' => 'No stat file — script may not be running']);
15	    exit;
16	}
17
18	$stat = json_decode(file_get_contents($statFile) ?: '{}', true) ?: [];
19
20	if (($stat['status'] ?? '') !== 'running') {
21	    echo json_encode(['ok' => true, 'msg' => 'Not running']);
22	    exit;
23	}
24
25	$pid = (int)($stat['pid'] ?? 0);
26	if ($pid < 2) {
27	    echo json_encode(['ok' => false, 'error' => 'No valid PID in stat file']);
28	    exit;
29	}
30
31	// Kill the whole process group so the script and all its children die together.
32	// pgid is usually the same as the session leader PID from run_job.sh.
33	$pgid = (int)trim(shell_exec("ps -o pgid= -p $pid 2>/dev/null") ?: '0');
34
35	if ($pgid > 1) {
36	    shell_exec("kill -TERM -$pgid 2>/dev/null");
37	} else {
38	    // Fallback: kill the direct PID and its children
39	    shell_exec("pkill -TERM -P $pid 2>/dev/null");
40	    shell_exec("kill -TERM $pid 2>/dev/null");
41	}
42
43	// Give it up to 3s to exit gracefully
44	$dead = false;
45	for ($i = 0; $i < 6; $i++) {
46	    usleep(500000);
47	    if (!file_exists("/proc/$pid")) { $dead = true; break; }
48	}
49
50	// Force-kill if still alive
51	if (!$dead) {
52	    if ($pgid > 1) shell_exec("kill -KILL -$pgid 2>/dev/null");
53	    shell_exec("pkill -KILL -P $pid 2>/dev/null");
54	    shell_exec("kill -KILL $pid 2>/dev/null");
55	    usleep(300000);
56	    $dead = !file_exists("/proc/$pid");
57	}
58
59	// Clear any lock files in /tmp/unraid_locks whose content matches this PID
60	$lockDir  = '/tmp/unraid_locks';
61	$cleared  = [];
62	foreach (glob("$lockDir/*.lock") ?: [] as $lf) {
63	    $content = trim(file_get_contents($lf) ?: '');
64	    $lockPid = (int)explode(':', $content)[0];
65	    if ($lockPid === $pid || !file_exists("/proc/$lockPid")) {
66	        @unlink($lf);
67	        $cleared[] = basename($lf);
68	    }
69	}
70
71	// Also clear by script name in case PID rotated
72	$scriptBase = basename($id, '.sh');
73	$namedLock  = "$lockDir/{$scriptBase}.lock";
74	if (file_exists($namedLock)) {
75	    @unlink($namedLock);
76	    if (!in_array(basename($namedLock), $cleared)) $cleared[] = basename($namedLock);
77	}
78
79	// Update stat file
80	$now = time();
81	$stat['status'] = 'stopped';
82	$stat['end']    = $now;
83	$stat['exit']   = -1;
84	unset($stat['pid']);
85	file_put_contents($statFile, json_encode($stat));
86
87	echo json_encode([
88	    'ok'      => true,
89	    'killed'  => $dead,
90	    'locks'   => $cleared,
91	]);
92


1	<?php
2	// Save rsync standalone config (location + cron) for a specific rsync tier.
3	// POST: flag_name (e.g. "DAILY_RSYNC_ENABLED"), orch_id, location, cron
4	// Stored in schedule.json under "__rsync_{FLAG_NAME}".
5	// Triggers a cron rebuild so the standalone entry takes effect immediately.
6	header('Content-Type: application/json');
7	require_once dirname(__DIR__) . '/include/config.php';
8	require_once dirname(__DIR__) . '/include/scheduler.php';
9
10	if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
11	    echo json_encode(['ok' => false, 'error' => 'POST only']);
12	    exit;
13	}
14
15	$flagName = trim($_POST['flag_name'] ?? '');
16	$orchId   = trim($_POST['orch_id']   ?? '');
17	$location = trim($_POST['location']  ?? '');
18	$cron     = trim($_POST['cron']      ?? '');
19
20	if (!$flagName || !preg_match('/^[A-Z_]+_RSYNC_ENABLED$/', $flagName)) {
21	    echo json_encode(['ok' => false, 'error' => 'Invalid flag_name']);
22	    exit;
23	}
24	if ($orchId && (str_contains($orchId, '..') || !preg_match('/^[A-Za-z0-9_.\-\/]+\.sh$/', $orchId))) {
25	    echo json_encode(['ok' => false, 'error' => 'Invalid orch_id']);
26	    exit;
27	}
28	if ($location && (!str_starts_with($location, '/') || str_contains($location, '..') || preg_match('/[\x00\n\r]/', $location))) {
29	    echo json_encode(['ok' => false, 'error' => 'Invalid location']);
30	    exit;
31	}
32
33	$key      = '__rsync_' . $flagName;
34	$schedule = vv_schedule_load();
35	$schedule[$key] = [
36	    'flag_name' => $flagName,
37	    'orch_id'   => $orchId,
38	    'location'  => $location,
39	    'cron'      => $cron,
40	];
41	if (!vv_schedule_save($schedule)) {
42	    echo json_encode(['ok' => false, 'error' => 'Write failed']);
43	    exit;
44	}
45
46	vv_cron_rebuild($schedule);
47	echo json_encode(['ok' => true]);
48


Let me check for more potential issues in the rsync and sync orchestrator scripts:


Error: File does not exist. Note: your current working directory is /boot/config/plugins/varaverk.


grep -r "notify" /boot/config/plugins/varaverk/Orchestrators/ 2>/dev/null | head -30


/boot/config/plugins/varaverk/Orchestrators/monthly_maintenance.sh:    notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
/boot/config/plugins/varaverk/Orchestrators/monthly_maintenance.sh:    notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \
/boot/config/plugins/varaverk/Orchestrators/critical_sync_maintenance.sh:    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
/boot/config/plugins/varaverk/Orchestrators/critical_sync_maintenance.sh:    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
/boot/config/plugins/varaverk/Orchestrators/critical_sync_maintenance.sh:    notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
/boot/config/plugins/varaverk/Orchestrators/watchdog_orchestrator.sh:        notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
/boot/config/plugins/varaverk/Orchestrators/array_stopping.sh:#   platform_require_cmd  — notify validated before use
/boot/config/plugins/varaverk/Orchestrators/array_stopping.sh:#   notify on failures   — alert if any stop script fails
/boot/config/plugins/varaverk/Orchestrators/array_stopping.sh:    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
/boot/config/plugins/varaverk/Orchestrators/array_stopping.sh:    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
/boot/config/plugins/varaverk/Orchestrators/array_stopping.sh:    notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
/boot/config/plugins/varaverk/Orchestrators/array_stopping.sh:    notify "Array stop on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \
/boot/config/plugins/varaverk/Orchestrators/daily_sync_maintenance.sh:    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
/boot/config/plugins/varaverk/Orchestrators/daily_sync_maintenance.sh:    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
/boot/config/plugins/varaverk/Orchestrators/daily_sync_maintenance.sh:                notify "Daily sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
/boot/config/plugins/varaverk/Orchestrators/daily_sync_maintenance.sh:    notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
/boot/config/plugins/varaverk/Orchestrators/array_started.sh:#     System_Essentials/inotify_tuning.sh       — raise inotify limits before containers start
/boot/config/plugins/varaverk/Orchestrators/array_started.sh:#   inotify_tuning.sh     — must run BEFORE Code-Server and other containers start
/boot/config/plugins/varaverk/Orchestrators/array_started.sh:#                           containers that start with low inotify limits keep them ✅
/boot/config/plugins/varaverk/Orchestrators/array_started.sh:#   platform_require_cmd  — notify validated before use
/boot/config/plugins/varaverk/Orchestrators/array_started.sh:#   notify on failures   — alert if any script fails to launch
/boot/config/plugins/varaverk/Orchestrators/array_started.sh:    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
/boot/config/plugins/varaverk/Orchestrators/array_started.sh:    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
/boot/config/plugins/varaverk/Orchestrators/array_started.sh:    notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
/boot/config/plugins/varaverk/Orchestrators/weekly_sync_maintenance.sh:#   platform_require_cmd  — notify validated before use
/boot/config/plugins/varaverk/Orchestrators/weekly_sync_maintenance.sh:    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
/boot/config/plugins/varaverk/Orchestrators/weekly_sync_maintenance.sh:    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
/boot/config/plugins/varaverk/Orchestrators/weekly_sync_maintenance.sh:    notify "Weekly maintenance failed on $(hostname) ($MY_ID) — sync: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
/boot/config/plugins/varaverk/Orchestrators/intermediate_sync_maintenance.sh:    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
/boot/config/plugins/varaverk/Orchestrators/intermediate_sync_maintenance.sh:    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"


Let me find and read the rsync script:


find /boot/config/plugins/varaverk -name "rsync.sh" -o -name "*sync*maintenance.sh" | head -10


1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= Rsync Core Script ==============================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Core rsync engine for the two-server ecosystem. Called per share or per
9	# appdata profile by orchestrators (daily_sync_maintenance, weekly_sync_maintenance,
10	# critical_sync_maintenance) and directly for manual or scheduled dirty syncs.
11	#
12	# Profile is inferred from the directory basename (lowercased). Override with
13	# --profile=name for explicit selection. If no profile matches, global defaults
14	# from master.conf apply and no containers are stopped.
15	#
16	# After each sync, logs transfer data to bandwidth_monitor.sh for the weekly
17	# bandwidth report. Silent on success — only failures produce visible output.
18	#
19	# ==============================================================================================
20	# OPERATIONAL MODEL
21	# ==============================================================================================
22	#
23	# Profiles define per-share behavior:
24	#   PROFILE_CRITICAL_CONTAINER_NAMES  — containers stopped on both servers before sync
25	#   PROFILE_DELAYED_CONTAINERS        — containers with a delay before restart after sync
26	#   PROFILE_CONTAINER_DELAY           — seconds before delayed containers start
27	#   PROFILE_RSYNC_OPTS                — rsync flags (does not inherit DEFAULT_RSYNC_OPTS)
28	#   PROFILE_BW_LIMIT                  — bandwidth limit in KB/s
29	#   PROFILE_RETRY_COUNT               — retry attempts on failure
30	#   PROFILE_SLEEP                     — seconds between retry attempts
31	#   PROFILE_EXCLUDE_DIRS              — paths excluded from transfer
32	#   PROFILE_REMOTE_RESTART_CONTAINERS — containers restarted on remote after dirty sync
33	#                                       Was running → restart. Was stopped → leave stopped.
34	#
35	# Two-tier rsync enable/disable:
36	#   Tier 1: RSYNC_ENABLED=false → all rsync stops immediately (checked by this script)
37	#   Tier 2: per-orchestrator flag (DAILY_RSYNC_ENABLED etc.) → checked by caller
38	#
39	# Bandwidth logging: after each sync, logs profile/duration/status/bytes to
40	# bandwidth_monitor.sh --log-transfer. Bytes captured from rsync --stats via awk
41	# using version-stable field names.
42	#
43	# ==============================================================================================
44	# OPERATIONAL SAFEGUARDS
45	# ==============================================================================================
46	#
47	# Global Rsync Gate
48	#   check_rsync_enabled() — RSYNC_ENABLED=false exits cleanly before any operation.
49	#
50	# Partnership Blocklist
51	#   Refuses to sync if REMOTE_SERVER_NAME appears in the partnership blocklist.
52	#   Written at offboard — prevents stale access after a partnership ends.
53	#
54	# Version Parity
55	#   check_unraid_version_parity — refuses sync if servers on incompatible unRAID versions.
56	#
57	# Remote Health Pre-flights
58	#   check_connectivity() — Tailscale IP reachable before any SSH
59	#   check_remote_rootfs() — aborts if remote rootfs exceeds ROOTFS_WARN_PCT
60	#   check_remote_share() — aborts if target directory missing or empty on remote
61	#   check_remote_disks() — verifies all backing disks online on remote
62	#
63	# Drive Temperature Check
64	#   check_local_disk_temps() — runs before any transfer. Exit 1 = skip this profile,
65	#   exit 2 = abort all remaining syncs (CRITICAL temperature).
66	#
67	# Remote Docker Daemon Check
68	#   check_remote_docker_daemon — verified before any container stop/start operations.
69	#   If daemon unresponsive: container operations skipped, rsync proceeds without stopping.
70	#
71	# Per-Profile Concurrency Lock
72	#   acquire_rsync_lock() — per-profile lock prevents parallel runs of the same profile.
73	#   Global concurrent limit prevents too many simultaneous rsync processes.
74	#
75	# Notification Validated
76	#   platform_require_cmd confirms the notify script is present before use.
77	#
78	# ==============================================================================================
79	# CONFIGURATION
80	# ==============================================================================================
81	#
82	# master.conf
83	#
84	#   RSYNC_ENABLED
85	#       Global on/off toggle for all rsync operations. (default: true)
86	#
87	#   DEFAULT_RSYNC_OPTS
88	#       Base rsync flags for unproiled shares. Note: --delete is intentionally absent —
89	#       media shares spread files only, arr cleanup scripts own deletions. Profile-specific
90	#       opts set --delete explicitly where needed.
91	#
92	#   BW_LIMIT
93	#       Default bandwidth cap in KB/s when no PROFILE_BW_LIMIT is set. (default: 0 = unlimited)
94	#
95	#   RETRY_COUNT
96	#       Default retry attempts on rsync failure. (default: 3)
97	#
98	#   SLEEP
99	#       Default seconds between retry attempts. (default: 60)
100	#
101	#   ROOTFS_WARN_PCT
102	#       Abort threshold for remote rootfs percentage full. (default: 75)
103	#
104	#   PROFILES["profile_KEY"]
105	#       Profile definitions — one entry per PROFILE_* key per profile name.
106	#       See OPERATIONAL MODEL above for all supported keys.
107	#
108	#   BANDWIDTH_LOG / BANDWIDTH_WARN_GB
109	#       Shared with bandwidth_monitor.sh — set once, used by both.
110	#
111	# ==============================================================================================
112	# RUNTIME MODES
113	# ==============================================================================================
114	#
115	# rsync.sh /path/to/share
116	#     Sync the given path to the remote. Profile inferred from directory basename.
117	#
118	# rsync.sh /path/to/share --profile=name
119	#     Sync with explicit profile override — bypasses basename inference.
120	#
121	# rsync.sh /path/to/share --dry-run
122	#     Run all pre-flight checks and show what rsync would transfer. No transfer,
123	#     no container stops.
124	#
125	# rsync.sh /path/to/share --status
126	#     Show resolved profile, remote identity, and configuration. Then exit.
127	#
128	# rsync.sh /path/to/share --log
129	#     Verbose output throughout — every decision logged.
130	#
131	# ==============================================================================================
132
133	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
134
135	source "$SCRIPT_DIR/../load_config.sh"
136
137	# ── Separate positional directory arg from flags ───────────────────────────────────────────────
138	DIRECTORY=""
139	PROFILE_OVERRIDE=""
140	RAW_ARGS=()
141
142	for ARG in "$@"; do
143	    case "$ARG" in
144	        --profile=*) PROFILE_OVERRIDE="${ARG#--profile=}" ;;
145	        --*|*=*)     RAW_ARGS+=("$ARG") ;;
146	        *)           [[ -z "$DIRECTORY" ]] && DIRECTORY="$ARG" || RAW_ARGS+=("$ARG") ;;
147	    esac
148	done
149
150	parse_args "${RAW_ARGS[@]}"
151
152	[[ -z "$DIRECTORY" ]] && {
153	    error "No directory specified"
154	    error "Usage: rsync.sh <dir> [--dry-run] [--log] [--profile=name]"
155	    exit 1
156	}
157
158	# ==============================================================================================
159	# ━━━ Setup ━━━
160	# ==============================================================================================
161	if [[ "$EUID" -ne 0 ]]; then
162	    error "Must be run as root"
163	    exit 1
164	fi
165
166	acquire_lock
167
168	if ! command -v docker &>/dev/null; then
169	    error "Docker command not found"
170	    exit 1
171	fi
172
173	platform_require_cmd \
174	    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
175	    "" "" \
176	    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
177
178	detect_hosts
179
180	# Tier 1 global gate — Tier 2 (per-orchestrator) checked by caller
181	if ! check_rsync_enabled; then
182	    warn "RSYNC_ENABLED=false — exiting cleanly"
183	    exit 0
184	fi
185
186	# Blocklist gate — refuse to sync with a partner blocked after offboard
187	BLOCKLIST_FILE="${PARTNERSHIP_BLOCKLIST_FILE:-${STATE_DIR:-/boot/config}/partnership_blocklist.db}"
188	if [[ -f "$BLOCKLIST_FILE" ]] && grep -q "^${REMOTE_SERVER_NAME}|" "$BLOCKLIST_FILE" 2>/dev/null; then
189	    error "Rsync blocked — $REMOTE_SERVER_NAME is on the partnership blocklist"
190	    error "Re-onboard the partnership to restore access: partnership_manager.sh --onboard"
191	    exit 1
192	fi
193
194	resolve_remote_ip
195
196	# ── Profile inference ─────────────────────────────────────────────────────────────────────────
197	if [[ -n "$PROFILE_OVERRIDE" ]]; then
198	    PROFILE_NAME="$PROFILE_OVERRIDE"
199	    log "Profile override: $PROFILE_NAME"
200	else
201	    PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]')
202	    log "Profile inferred: $PROFILE_NAME"
203	fi
204
205	# Acquire per-profile lock and check global concurrent limit
206	acquire_rsync_lock "$PROFILE_NAME"
207
208	# ── Load profile settings ─────────────────────────────────────────────────────────────────────
209	BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT}
210	RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT}
211	SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP}
212	CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY}
213
214	read -r -a CRITICAL_CONTAINER_NAMES       <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}"
215	read -r -a DELAYED_CONTAINERS             <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}"
216	read -r -a EXCLUDE_DIRS                   <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}"
217	read -r -a REMOTE_RESTART_CONTAINERS      <<< "${PROFILE_REMOTE_RESTART_CONTAINERS[$PROFILE_NAME]:-}"
218
219	# Local containers use same names as remote (mirrored naming scheme)
220	LOCAL_CRITICAL_CONTAINER_NAMES=("${CRITICAL_CONTAINER_NAMES[@]}")
221
222	log "$ICON_GEAR Config: profile=${PROFILE_NAME} bw-limit=${BW_LIMIT}KB/s retry=${RETRY_COUNT} sleep=${SLEEP}s container-delay=${CONTAINER_DELAY}s"
223	log "$ICON_GEAR Containers: critical=${CRITICAL_CONTAINER_NAMES[*]:-none} delayed=${DELAYED_CONTAINERS[*]:-none} remote-restart=${REMOTE_RESTART_CONTAINERS[*]:-none}"
224
225	[[ "$SHOW_STATUS" == true ]] && show_status && exit 0
226	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
227
228	# ==============================================================================================
229	# ━━━ Pre-flight Checks ━━━
230	# ==============================================================================================
231	echo ""
232	echo "━━━ $ICON_SHIELD Pre-flight ━━━"
233
234	# Disk temp — before touching remote or moving data
235	# Exit 1 = skip this profile | Exit 2 = abort all remaining profiles
236	check_local_disk_temps
237	TEMP_RESULT=$?
238	if [[ "$TEMP_RESULT" -eq 2 ]]; then
239	    error "Drive temps CRITICAL — aborting all remaining syncs"
240	    exit 2
241	elif [[ "$TEMP_RESULT" -eq 1 ]]; then
242	    warn "Drive temps high — skipping profile [$PROFILE_NAME]"
243	    exit 1
244	else
245	    log "Drive temps OK — $TEMP_CHECK_RESULT"
246	fi
247
248	# Version parity — refuse if servers on incompatible unRAID versions
249	check_unraid_version_parity || exit 1
250
251	check_connectivity
252	check_remote_rootfs
253	check_remote_share "$DIRECTORY"
254	check_remote_disks "$DIRECTORY"
255
256	# Remote Docker daemon — check before attempting container operations
257	if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]] || [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
258	    check_remote_docker_daemon || {
259	        warn "Remote Docker daemon unresponsive — skipping container operations"
260	        warn "Proceeding with rsync only — containers will not be stopped or restarted"
261	        CRITICAL_CONTAINER_NAMES=()
262	        LOCAL_CRITICAL_CONTAINER_NAMES=()
263	        REMOTE_RESTART_CONTAINERS=()
264	    }
265	fi
266
267	# ==============================================================================================
268	# ━━━ Stop Containers ━━━
269	# ==============================================================================================
270	if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
271	    echo ""
272	    echo "━━━ $ICON_STOP $ICON_CONTAINERS Stop Containers ━━━"
273	    if [[ "$DRY_RUN" == true ]]; then
274	        for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]:-}"; do
275	            [[ -n "$c" ]] && warn "DRY RUN — would stop local: $c"
276	        done
277	        for c in "${CRITICAL_CONTAINER_NAMES[@]:-}"; do
278	            [[ -n "$c" ]] && warn "DRY RUN — would stop remote: $c"
279	        done
280	    else
281	        # Local first — flush local databases before pushing
282	        stop_local_containers
283	        # Remote next — prevent writes while receiving
284	        stop_containers
285	    fi
286	fi
287
288	# ==============================================================================================
289	# ━━━ Transfer ━━━
290	# ==============================================================================================
291	echo ""
292	echo "━━━ $ICON_SYNC Transfer ━━━"
293	echo "$ICON_RUN  Source:   $DIRECTORY"
294	echo "$ICON_NET  Remote:   $REMOTE_SERVER:$DIRECTORY"
295	echo "$ICON_GEAR Profile:  $PROFILE_NAME"
296	echo "$ICON_HOST Identity: $MY_ID → $REMOTE_ID"
297	echo ""
298
299	get_rsync_opts
300
301	# Append profile excludes
302	for ex in "${EXCLUDE_DIRS[@]:-}"; do
303	    [[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex")
304	done
305
306	# Add --stats to capture bytes transferred for bandwidth logging
307	RSYNC_OPTS+=(--stats)
308
309	[[ "$DRY_RUN" == true ]] && RSYNC_OPTS+=("--dry-run")
310
311	START=$(date +%s)
312	RSYNC_SUCCESS=false
313	BYTES_TRANSFERRED=0
314	ATTEMPT=0
315
316	for (( ATTEMPT=1; ATTEMPT<=RETRY_COUNT; ATTEMPT++ )); do
317	    log "$ICON_RETRY Attempt $ATTEMPT of $RETRY_COUNT..."
318	    echo "$ICON_SYNC  Rsync running — this may take a while..."
319
320	    RSYNC_OUTPUT=$(rsync "${RSYNC_OPTS[@]}" \
321	        -e "ssh -i $SSH_KEY -T -o Compression=no -o IPQoS=throughput" \
322	        "$DIRECTORY" "root@${REMOTE_SERVER}:$(dirname "$DIRECTORY")/" 2>&1)
323
324	    RSYNC_EXIT=$?
325
326	    if [[ "$RSYNC_EXIT" -eq 0 ]]; then
327	        # Parse bytes transferred from --stats output
328	        BYTES_TRANSFERRED=$(echo "$RSYNC_OUTPUT" | \
329	            awk '/Total transferred file size:/{gsub(/,/,"",$NF); gsub(/[^0-9]/,"",$NF); print $NF+0}')
330	        BYTES_TRANSFERRED="${BYTES_TRANSFERRED:-0}"
331
332	        log "$ICON_DONE Rsync complete — $BYTES_TRANSFERRED bytes transferred"
333	        RSYNC_SUCCESS=true
334	        break
335	    else
336	        warn "$ICON_RETRY Rsync failed (attempt $ATTEMPT/$RETRY_COUNT)"
337	        log "Exit code: $RSYNC_EXIT"
338	        if [[ "$ATTEMPT" -lt "$RETRY_COUNT" ]]; then
339	            log "Retrying in ${SLEEP}s..."
340	            sleep "$SLEEP"
341	        fi
342	    fi
343	done
344
345	# ==============================================================================================
346	# ━━━ Start Containers ━━━
347	# ==============================================================================================
348	if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -gt 0 || ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -gt 0 ]]; then
349	    echo ""
350	    echo "━━━ $ICON_START $ICON_CONTAINERS Start Containers ━━━"
351	    if [[ "$DRY_RUN" == true ]]; then
352	        for c in "${CRITICAL_CONTAINER_NAMES[@]:-}"; do
353	            [[ -n "$c" ]] && warn "DRY RUN — would start remote: $c"
354	        done
355	        for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]:-}"; do
356	            [[ -n "$c" ]] && warn "DRY RUN — would start local: $c"
357	        done
358	    else
359	        # Remote first — can be coming up while local restarts
360	        start_containers
361	        # Local next
362	        start_local_containers
363	    fi
364	fi
365
366	# ==============================================================================================
367	# ━━━ Remote Restart (dirty sync profiles) ━━━
368	# ==============================================================================================
369	# For dirty sync profiles (critical-fallback, emby-fallback) — restart containers on remote
370	# that were running before sync so they pick up config changes from the dirty sync window.
371	# Was running → restart. Was stopped → leave stopped.
372	if [[ ${#REMOTE_RESTART_CONTAINERS[@]} -gt 0 ]]; then
373	    echo ""
374	    echo "━━━ $ICON_START $ICON_CONTAINERS Remote Restart (post dirty sync) ━━━"
375	    log "Restarting configured containers on $REMOTE_SERVER_NAME..."
376
377	    for container in "${REMOTE_RESTART_CONTAINERS[@]}"; do
378	        [[ -z "$container" ]] && continue
379
380	        # Check if container was running before sync (still tracked via RUNNING_CONTAINERS)
381	        WAS_RUNNING=false
382	        for prev in "${RUNNING_CONTAINERS[@]:-}"; do
383	            [[ "$prev" == "$container" ]] && WAS_RUNNING=true && break
384	        done
385
386	        if [[ "$WAS_RUNNING" == false ]]; then
387	            # Not in stop list — check current remote state
388	            REMOTE_STATUS=$(timeout 15 ssh -i "$SSH_KEY" \
389	                -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
390	                "docker inspect -f '{{.State.Running}}' $container 2>/dev/null" 2>/dev/null)
391	            [[ "$REMOTE_STATUS" != "true" ]] && \
392	                log "$container not running on $REMOTE_SERVER_NAME — skipping remote restart" && \
393	                continue
394	        fi
395
396	        if [[ "$DRY_RUN" == true ]]; then
397	            warn "DRY RUN — would restart $container on $REMOTE_SERVER_NAME"
398	            continue
399	        fi
400


1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= Daily Sync Maintenance =========================================
4	# ==============================================================================================
5	# Daily orchestrator — runs the full daily maintenance window in the correct order.
6	# Schedule: 0 1 * * * (1am daily via User Scripts plugin)
7	#
8	# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
9	#   Pre-sync:
10	#     git_pull_execute.sh              — pull latest scripts first, always
11	#
12	#   Arr Sync (arr_sync.sh):
13	#     Syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
14	#     All nodes agree on tracked library before any files are transferred.
15	#     Remote nodes that don't have an arr running are skipped gracefully.
16	#
17	#   Rsync window (DAILY_SYNC_SHARES per host):
18	#     HOST*_DAILY_SYNC_SHARES          — media shares spread to all nodes (no --delete)
19	#     HOST*_PERSONAL_SHARES            — encrypted personal shares
20	#
21	#   Post-sync maintenance (DAILY_MAINTENANCE_SCRIPTS):
22	#     media_shares_permissions.sh      — fix ownership before arr cleanup
23	#     media_cleaner.sh anime           — remove junk from anime shares
24	#     media_cleaner.sh media           — remove junk from media shares
25	#     lidarr_cleanup.sh                — remove orphaned music files (local arr = truth)
26	#     sonarr_cleanup.sh                — remove orphaned TV files (local arr = truth)
27	#     radarr_cleanup.sh                — remove orphaned movie files (local arr = truth)
28	#     docker_daily_restart.sh          — restart containers needing daily restart
29	#
30	# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
31	#   git pull first        — maintenance runs on latest code, not yesterday's
32	#   arr sync before rsync — all nodes track the same library before files are spread;
33	#                           prevents remote arrs from searching for content already owned
34	#   rsync before cleanup  — cleanup sees fully spread state, rsync has no --delete
35	#   permissions before arr cleanup — arrs need correct ownership to delete/rename
36	#   arr cleanup after permissions  — clean ownership = successful orphan deletion
37	#   docker restart last   — containers already processed by cleanup
38	#
39	# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
40	# Bidirectional — same script runs on both servers, correct direction automatic.
41	# detect_hosts() aliases DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars.
42	# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server.
43	#
44	# ── DRIVE TEMP HANDLING ───────────────────────────────────────────────────────────────────────
45	# rsync.sh returns exit codes for temperature issues:
46	#   exit 1 = temp WARN — skip this share, continue to next
47	#   exit 2 = temp CRITICAL — abort ALL remaining syncs in this window
48	#   All other failures — skip share, continue to next
49	#
50	# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
51	# Runs daily at 1am — clean run should produce minimal output.
52	# Each job reports log() on success (silent), warn()/error() on failure (visible).
53	# Summary always shown — gives window timing and share/job counts.
54	# Notify only on failure — successful daily maintenance doesn't need notification.
55	#
56	# ── CONFIGURATION (master.conf + host*.conf) ───────────────────────────────────────────
57	#   HOST*_DAILY_SYNC_SHARES      — shares pushed to mirror each day
58	#   HOST*_PERSONAL_SHARES        — encrypted personal shares
59	#   DAILY_MAINTENANCE_SCRIPTS    — maintenance jobs (permissions, cleanup, restart)
60	#   DAILY_RSYNC_ENABLED          — enable/disable rsync section
61	#
62	# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
63	#   daily_sync_maintenance.sh              — normal run
64	#   daily_sync_maintenance.sh --dry-run    — preview without syncing or changing
65	#   daily_sync_maintenance.sh --log        — verbose per-share/per-job output
66	#   daily_sync_maintenance.sh --status     — show configured shares and jobs
67	# ==============================================================================================
68
69	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
70
71	source "$SCRIPT_DIR/../load_config.sh"
72
73	RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
74	SCRIPTS_ROOT="$SCRIPT_DIR/.."
75
76	parse_args "$@"
77
78	# ==============================================================================================
79	# ━━━ Setup ━━━
80	# ==============================================================================================
81	if [[ "$EUID" -ne 0 ]]; then
82	    error "Must be run as root"
83	    exit 1
84	fi
85
86	platform_require_cmd \
87	    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
88	    "" "" \
89	    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
90
91	if ! command -v docker &>/dev/null; then
92	    error "Docker command not found"
93	    exit 1
94	fi
95
96	detect_hosts
97	resolve_remote_ip
98
99	acquire_lock
100
101	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
102
103	# ==============================================================================================
104	# ━━━ Status ━━━
105	# ==============================================================================================
106	if [[ "$SHOW_STATUS" == true ]]; then
107	    echo ""
108	    echo "━━━━━ $ICON_SUMMARY DAILY SYNC STATUS ━━━━━"
109	    echo "$ICON_HOST   Identity:        $MY_ID ($LOCAL_SERVER_NAME)"
110	    echo "$ICON_NET    Remote:          $REMOTE_ID ($REMOTE_SERVER_NAME)"
111	    echo "$ICON_SYNC   Rsync enabled:   ${RSYNC_ENABLED:-true}"
112	    echo "$ICON_SYNC   Daily enabled:   ${DAILY_RSYNC_ENABLED:-false}"
113	    echo ""
114	    echo "━━━ Daily Sync Shares ━━━"
115	    for share in "${DAILY_SYNC_SHARES[@]}"; do
116	        [[ -n "$share" ]] && echo "  $ICON_SYNC $share"
117	    done
118	    for share in "${PERSONAL_SHARES[@]}"; do
119	        [[ -n "$share" ]] && echo "  $ICON_SYNC $share (personal)"
120	    done
121	    echo ""
122	    echo "━━━ Daily Maintenance Scripts ━━━"
123	    for entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
124	        [[ -z "$entry" ]] && continue
125	        echo "  $ICON_GEAR ${entry##*/}"
126	    done
127	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
128	    exit 0
129	fi
130
131	# ==============================================================================================
132	# ── BUILD SHARE LIST via detect_hosts aliases ─────────────────────────────────────────────────
133	# detect_hosts() sets DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars
134	# No manual HOST1/HOST2 comparison needed — aliased automatically per server
135	# ==============================================================================================
136	ALL_SHARES=()
137	for share in "${DAILY_SYNC_SHARES[@]}"; do
138	    [[ -n "$share" ]] && ALL_SHARES+=("$share")
139	done
140	for share in "${PERSONAL_SHARES[@]}"; do
141	    [[ -n "$share" ]] && ALL_SHARES+=("$share")
142	done
143	SHARE_COUNT=${#ALL_SHARES[@]}
144
145	# ── Split maintenance scripts: git pull runs pre-sync, rest run post-sync ─────────────────────
146	PRE_SYNC_SCRIPTS=()
147	POST_SYNC_SCRIPTS=()
148	for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
149	    [[ -z "$script_entry" ]] && continue
150	    script_name=$(basename "${script_entry%% *}")
151	    if [[ "$script_name" == "git_pull_execute.sh" ]]; then
152	        PRE_SYNC_SCRIPTS+=("$script_entry")
153	    else
154	        POST_SYNC_SCRIPTS+=("$script_entry")
155	    fi
156	done
157
158	# Helper — run a maintenance script, track pass/fail
159	run_job() {
160	    local script_entry="$1"
161	    local extra_dry=""
162	    [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
163
164	    read -r -a script_args <<< "$script_entry"
165	    local script_path="$SCRIPTS_ROOT/${script_args[0]}"
166	    local script_name
167	    script_name=$(basename "${script_args[0]}")
168	    local extra_args=("${script_args[@]:1}")
169
170	    if [[ ! -f "$script_path" ]]; then
171	        error "$script_name — not found at $script_path"
172	        JOB_FAIL+=("$script_name")
173	        return 1
174	    fi
175
176	    log "Running: $script_name ${extra_args[*]}"
177	    # shellcheck disable=SC2086
178	    if bash "$script_path" "${extra_args[@]}" $extra_dry; then
179	        log "$script_name — done ✅"
180	        JOB_PASS+=("$script_name ${extra_args[*]}")
181	    else
182	        error "$script_name — failed (exit $?)"
183	        JOB_FAIL+=("$script_name ${extra_args[*]}")
184	    fi
185	}
186
187	WINDOW_START=$(date +%s)
188	JOB_PASS=()
189	JOB_FAIL=()
190	PASS=()
191	FAIL=()
192	SHARE_TIMES=()
193
194	echo ""
195	echo "━━━ $ICON_GEAR Daily Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
196
197	# ==============================================================================================
198	# ━━━ Pre-sync — git pull ━━━
199	# ==============================================================================================
200	if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then
201	    echo ""
202	    echo "━━━ $ICON_GIT Pre-sync ━━━"
203	    for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
204	        run_job "$script_entry"
205	    done
206	fi
207
208	# ==============================================================================================
209	# ━━━ Arr Sync — library reconciliation before file spreading ━━━
210	# ==============================================================================================
211	echo ""
212	echo "━━━ $ICON_SYNC Arr Sync ━━━"
213
214	ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
215	if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
216	    echo "ARR_SYNC_ENABLED=false — skipping"
217	elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
218	    warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
219	    JOB_FAIL+=("arr_sync.sh")
220	else
221	    _arr_sync_args=()
222	    [[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
223	    if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
224	        echo "Arr sync complete ✅"
225	        JOB_PASS+=("arr_sync.sh")
226	    else
227	        warn "Arr sync completed with errors — continuing to rsync"
228	        JOB_FAIL+=("arr_sync.sh")
229	    fi
230	    unset _arr_sync_args
231	fi
232
233	# ==============================================================================================
234	# ━━━ Media Share Sync ━━━
235	# ==============================================================================================
236	echo ""
237	echo "━━━ $ICON_SYNC Media Share Sync — $SHARE_COUNT share(s) ━━━"
238
239	TOTAL_START=$(date +%s)
240	SHARE_INDEX=0
241	ABORT_ALL_SYNCS=false
242
243	if ! check_rsync_enabled "DAILY"; then
244	    warn "Daily rsync disabled — skipping all $SHARE_COUNT share syncs"
245	    warn "Proceeding to maintenance jobs..."
246	elif [[ "$SHARE_COUNT" -eq 0 ]]; then
247	    warn "No shares configured for $MY_ID — check HOST*_DAILY_SYNC_SHARES in host*.conf"
248	else
249	    # Pre-flight — connectivity then remote rootfs
250	    check_connectivity


Let me look for lock file cleanup, state management, and failure patterns in the system:


grep -r "WATCHDOG_CONTAINER_URLS" /boot/config/plugins/varaverk/*.conf 2>/dev/null | head -20


(Bash completed with no output)


find /boot/config/plugins/varaverk -name "*.conf" -type f 2>/dev/null | head -5