1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= Health Digest ==================================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Full ecosystem health aggregation from existing state files. Scheduled daily
9	# (8am). DIGEST_PROFILE controls when notifications actually send — the cron
10	# schedule never changes, only the profile in master.conf.
11	#
12	# Reads state files from across the system (watchdog strikes, fallback state,
13	# skip list, bandwidth history, transcode stats, cert status) and compiles them
14	# into a single digest. Reads only — writes nothing, changes nothing.
15	#
16	# ==============================================================================================
17	# OPERATIONAL MODEL
18	# ==============================================================================================
19	#
20	# Three profiles — switch by changing DIGEST_PROFILE in master.conf:
21	#
22	#   always  — sends every run regardless of findings
23	#             Use: daily digest of everything, even when healthy
24	#
25	#   smart   — sends only when something worth reporting was found
26	#             Stays silent on clean days. DIGEST_SMART_ON_* toggles control
27	#             what triggers a send — all independently configurable.
28	#
29	#   weekly  — sends once per week on DIGEST_DAY (default Sunday), silent all other days
30	#             Use: one weekly summary without daily noise
31	#
32	# Data sources (reads only):
33	#   FALLBACK_STATE_FILE           — current fallback state
34	#   DOCKER_WATCHDOG_FAILED_FILE      — container skip list (manual intervention needed)
35	#   WATCHDOG_STATE_FILE           — active container watchdog strikes
36	#   SYS_WATCHDOG_STATE_FILE       — active system watchdog strikes
37	#   BANDWIDTH_LOG                 — yesterday's transfer totals
38	#   TRANSCODE_DAILY_LOG           — weekly transcode statistics
39	#   CERT_MONITOR_DOMAINS          — live SSL cert check via openssl
40	#   RAMDISK_PATH / TRANSCODE_LINK — current transcode location and usage
41	#
42	# ==============================================================================================
43	# OPERATIONAL SAFEGUARDS
44	# ==============================================================================================
45	#
46	# Single Instance Lock
47	#   acquire_lock prevents duplicate reports — report generation takes time.
48	#
49	# Per-Host Variables
50	#   detect_hosts() aliases CERT_MONITOR_DOMAINS, RAMDISK_WARN_GB, RAMDISK_SIZE,
51	#   RAMDISK_LOW_GB, and all other host-specific vars used in the report.
52	#
53	# Per-Section Guards
54	#   Each data source section checks whether its state file exists before reading.
55	#   A missing state file is skipped cleanly — it does not abort the report.
56	#
57	# Silent Smart Profile
58	#   smart profile produces no output and no notification when nothing worth
59	#   reporting is found.
60	#
61	# Notifications Validated
62	#   platform_require_cmd confirms notify and openssl are present before use.
63	#
64	# ==============================================================================================
65	# CONFIGURATION
66	# ==============================================================================================
67	#
68	# master.conf
69	#
70	#   DIGEST_PROFILE
71	#       Notification frequency: always | smart | weekly. (default: weekly)
72	#
73	#   DIGEST_DAY
74	#       Day name for weekly profile — must match `date +%A` output. (default: Sunday)
75	#
76	#   DIGEST_SMART_ON_WATCHDOG
77	#       Send smart profile notification if any active watchdog strikes. (default: true)
78	#
79	#   DIGEST_SMART_ON_FALLBACK
80	#       Send smart profile notification if fallback state is not NORMAL. (default: true)
81	#
82	#   DIGEST_SMART_ON_CERT_WARN
83	#       Send smart profile notification if any cert is within CERT_WARN_DAYS. (default: true)
84	#
85	#   DIGEST_SMART_ON_BANDWIDTH
86	#       Send smart profile notification if any transfer exceeded BANDWIDTH_WARN_GB. (default: true)
87	#
88	#   CERT_WARN_DAYS / CERT_CRIT_DAYS / CERT_TIMEOUT
89	#       Cert check thresholds — shared with cert_monitor.sh.
90	#
91	#   BANDWIDTH_WARN_GB
92	#       High-transfer threshold — shared with bandwidth_monitor.sh.
93	#
94	#   TRANSCODE_DAILY_LOG
95	#       Path to the transcode statistics log.
96	#
97	# ==============================================================================================
98	# RUNTIME MODES
99	# ==============================================================================================
100	#
101	# weekly_health_digest.sh
102	#     Run digest. DIGEST_PROFILE determines whether a notification is sent.
103	#
104	# weekly_health_digest.sh --dry-run
105	#     Generate and display digest output. No notification sent regardless of profile.
106	#
107	# weekly_health_digest.sh --status
108	#     Show profile, day, and smart trigger configuration. Then exit.
109	#
110	# weekly_health_digest.sh --log
111	#     Verbose per-section output during digest generation.
112	#
113	# ==============================================================================================
114
115	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
116
117	source "$SCRIPT_DIR/../load_config.sh"
118
119	parse_args "$@"
120
121	# ==============================================================================================
122	# ━━━ Setup ━━━
123	# ==============================================================================================
124	if [[ "$EUID" -ne 0 ]]; then
125	    error "Must be run as root"
126	    exit 1
127	fi
128
129
130	platform_require_cmd \
131	    "$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \
132	    "version" "OpenSSL" \
133	    "openssl" || warn "openssl not found — SSL cert checks will be skipped"
134
135	acquire_lock
136
137	# detect_hosts() sets MY_ID and aliases all host-specific vars used in this report
138	detect_hosts
139
140	log "$ICON_GEAR Config: profile=${DIGEST_PROFILE} day=${DIGEST_DAY}"
141	log "$ICON_GEAR Smart triggers: watchdog=${DIGEST_SMART_ON_WATCHDOG} fallback=${DIGEST_SMART_ON_FALLBACK} cert=${DIGEST_SMART_ON_CERT_WARN} bandwidth=${DIGEST_SMART_ON_BANDWIDTH}"
142
143	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — report generated but no notification sent"
144
145	# ==============================================================================================
146	# ━━━ Status ━━━
147	# ==============================================================================================
148	if [[ "$SHOW_STATUS" == true ]]; then
149	    echo ""
150	    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
151	    echo "$ICON_HOST    Identity:       $MY_ID ($LOCAL_SERVER_NAME)"
152	    echo "$ICON_DIGEST  Profile:        $DIGEST_PROFILE"
153	    echo "$ICON_DIGEST  Digest day:     $DIGEST_DAY"
154	    echo "$ICON_DIGEST  Smart triggers: watchdog=$DIGEST_SMART_ON_WATCHDOG fallback=$DIGEST_SMART_ON_FALLBACK cert=$DIGEST_SMART_ON_CERT_WARN bandwidth=$DIGEST_SMART_ON_BANDWIDTH"
155	    echo "$ICON_CERT    Cert domains:   ${CERT_MONITOR_DOMAINS[*]:-none}"
156	    echo "$ICON_BANDWIDTH Bandwidth warn: ${BANDWIDTH_WARN_GB}GB"
157	    echo "$ICON_GEAR    Dry Run:        $DRY_RUN"
158	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
159	    exit 0
160	fi
161
162	# ==============================================================================================
163	# ── Profile gate — should we send today? ──────────────────────────────────────────────────────
164	# ==============================================================================================
165	SHOULD_SEND=false
166
167	case "$DIGEST_PROFILE" in
168	    always)
169	        SHOULD_SEND=true
170	        log "Profile: always — will send"
171	        ;;
172	    weekly)
173	        TODAY_NAME=$(date '+%A')
174	        if [[ "$TODAY_NAME" == "$DIGEST_DAY" ]]; then
175	            SHOULD_SEND=true
176	            log "Profile: weekly — today is $DIGEST_DAY — will send"
177	        else
178	            echo "Profile: weekly — today is $TODAY_NAME, digest day is $DIGEST_DAY — no-op"
179	            exit 0
180	        fi
181	        ;;
182	    smart)
183	        log "Profile: smart — evaluating findings before deciding"
184	        SHOULD_SEND=false
185	        ;;
186	    *)
187	        warn "Unknown DIGEST_PROFILE: $DIGEST_PROFILE — defaulting to weekly"
188	        TODAY_NAME=$(date '+%A')
189	        [[ "$TODAY_NAME" == "$DIGEST_DAY" ]] && SHOULD_SEND=true
190	        ;;
191	esac
192
193	# ==============================================================================================
194	# ── Data Gathering ────────────────────────────────────────────────────────────────────────────
195	# ==============================================================================================
196	FINDINGS=()     # notable but not critical
197	ISSUES=()       # need attention
198	DIGEST_LINES=() # full report lines
199
200	# ── fallback State ────────────────────────────────────────────────────────────────────────────
201	log "Reading: fallback=$FALLBACK_STATE_FILE skip=$DOCKER_WATCHDOG_FAILED_FILE watchdog=$WATCHDOG_STATE_FILE sys=$SYS_WATCHDOG_STATE_FILE"
202	if [[ -f "$FALLBACK_STATE_FILE" ]]; then
203	    FALLBACK_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2)
204	    if [[ -n "$FALLBACK_STATE" ]]; then
205	        DIGEST_LINES+=("$ICON_FALLBACK Fallback: $FALLBACK_STATE")
206	        if [[ "$FALLBACK_STATE" != "NORMAL" ]]; then
207	            ISSUES+=("Fallback state: $FALLBACK_STATE")
208	            [[ "$DIGEST_SMART_ON_FALLBACK" == true ]] && SHOULD_SEND=true
209	        fi
210	    fi
211	else
212	    DIGEST_LINES+=("$ICON_FALLBACK Fallback: state file not found")
213	fi
214
215	# ── Container Skip List ───────────────────────────────────────────────────────────────────────
216	if [[ -f "$DOCKER_WATCHDOG_FAILED_FILE" ]] && [[ -s "$DOCKER_WATCHDOG_FAILED_FILE" ]]; then
217	    SKIP_COUNT=$(wc -l < "$DOCKER_WATCHDOG_FAILED_FILE")
218	    SKIP_LIST=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" | tr '\n' ' ')
219	    DIGEST_LINES+=("$ICON_NOT_RUNNING Skip list: $SKIP_COUNT containers — $SKIP_LIST")
220	    ISSUES+=("Containers on skip list (manual intervention needed): $SKIP_LIST")
221	    SHOULD_SEND=true
222	else
223	    DIGEST_LINES+=("$ICON_RUNNING Skip list: empty ✅")
224	fi
225
226	# ── Container Watchdog Strikes ────────────────────────────────────────────────────────────────
227	if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
228	    ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -c ".")
229	    if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then
230	        STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
231	        DIGEST_LINES+=("$ICON_WATCHDOG Container strikes: $ACTIVE_STRIKES active — $STRIKE_LIST")
232	        FINDINGS+=("Container watchdog: $ACTIVE_STRIKES active strikes")
233	        [[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true
234	    else
235	        DIGEST_LINES+=("$ICON_WATCHDOG Container watchdog: no active strikes ✅")
236	    fi
237	fi
238
239	# ── System Watchdog Strikes ───────────────────────────────────────────────────────────────────
240	if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
241	    SYS_STRIKES=$(grep "^[^=]*:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -cv -E "(:0$|:false$)")
242	    if [[ "$SYS_STRIKES" -gt 0 ]]; then
243	        SYS_STRIKE_LIST=$(grep "^[^=]*:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v -E "(:0$|:false$)" | tr '\n' ' ')
244	        DIGEST_LINES+=("$ICON_REBOOT_SMART System strikes: $SYS_STRIKES active — $SYS_STRIKE_LIST")
245	        FINDINGS+=("System watchdog: $SYS_STRIKES active strikes")
246	        [[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true
247	    else
248	        DIGEST_LINES+=("$ICON_REBOOT_SMART System watchdog: no active strikes ✅")
249	    fi
250	fi
251
252	# ── Transcode Ramdisk ─────────────────────────────────────────────────────────────────────────
253	if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
254	    RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
255	    RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
256	    SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
257	    DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used → $SYMLINK_TARGET")
258
259	    # Weekly transcode stats from TRANSCODE_DAILY_LOG
260	    if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]]; then
261	        WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d')
262	        WEEK_PEAK=$(awk -F'|' -v c="$WEEK_CUTOFF" \
263	            '$1>=c{if($2>max)max=$2} END{printf "%.2f",max+0}' "$TRANSCODE_DAILY_LOG")
264	        WEEK_FLIPS=$(awk -F'|' -v c="$WEEK_CUTOFF" \
265	            '$1>=c{sum+=$3} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
266	        WEEK_RAM=$(awk -F'|' -v c="$WEEK_CUTOFF" \
267	            '$1>=c{sum+=$4} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
268	        WEEK_SSD=$(awk -F'|' -v c="$WEEK_CUTOFF" \
269	            '$1>=c{sum+=$5} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
270	        WEEK_FILES=$(awk -F'|' -v c="$WEEK_CUTOFF" \
271	            '$1>=c{sum+=$6} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
272
273	        DIGEST_LINES+=("$ICON_RAM Transcode week: peak ${WEEK_PEAK}GB | flips: $WEEK_FLIPS | sessions: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD | cleaned: ${WEEK_FILES} files")
274
275	        PEAK_INT=$(printf "%.0f" "$WEEK_PEAK")
276	        WARN_INT=$(printf "%.0f" "${RAMDISK_WARN_GB:-6.8}")
277	        if [[ "$PEAK_INT" -ge "$WARN_INT" ]]; then
278	            DIGEST_LINES+=("$ICON_WARN Peak ${WEEK_PEAK}GB near threshold ${RAMDISK_WARN_GB}GB — consider increasing HOST*_RAMDISK_SIZE")
279	            FINDINGS+=("Transcode ramdisk near threshold: ${WEEK_PEAK}GB / ${RAMDISK_WARN_GB}GB")
280	        fi
281	    fi
282	else
283	    DIGEST_LINES+=("$ICON_RAM Transcodes: ramdisk not mounted")
284	    ISSUES+=("Ramdisk not mounted at $RAMDISK_PATH")
285	    SHOULD_SEND=true
286	fi
287
288	# ── Bandwidth ─────────────────────────────────────────────────────────────────────────────────
289	# Updated for new log format: date|time|profile|duration|status|bytes|warn_flag
290	if [[ -f "${BANDWIDTH_LOG:-}" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
291	    YESTERDAY=$(date -d "yesterday" '+%Y-%m-%d')
292	    YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$6} END{print sum+0}' \
293	        "$BANDWIDTH_LOG")
294	    YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", ${YESTERDAY_BYTES:-0} / 1073741824}")
295	    YESTERDAY_LARGE=$(awk -F'|' -v d="$YESTERDAY" '$1==d && $7=="LARGE"' \
296	        "$BANDWIDTH_LOG" | wc -l)
297
298	    if [[ "${YESTERDAY_LARGE:-0}" -gt 0 ]]; then
299	        DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday: ${YESTERDAY_GB}GB — $YESTERDAY_LARGE large transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB")
300	        FINDINGS+=("High bandwidth day: ${YESTERDAY_GB}GB transferred")
301	        [[ "$DIGEST_SMART_ON_BANDWIDTH" == true ]] && SHOULD_SEND=true
302	    else
303	        DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday: ${YESTERDAY_GB}GB")
304	    fi
305	else
306	    DIGEST_LINES+=("$ICON_BANDWIDTH Bandwidth: no data yet")
307	fi
308
309	# ── SSL Certificates ──────────────────────────────────────────────────────────────────────────
310	if [[ ${#CERT_MONITOR_DOMAINS[@]} -gt 0 ]] && command -v openssl >/dev/null 2>&1; then
311	    CERT_ISSUES=()
312	    for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
313	        [[ -z "$domain" ]] && continue
314	        expiry_str=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \
315	            -connect "${domain}:443" -servername "$domain" \
316	            2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
317	        if [[ -n "$expiry_str" ]]; then
318	            expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
319	            days_remaining=$(( (expiry_epoch - $(date +%s)) / 86400 ))
320	            if [[ "$days_remaining" -le "${CERT_CRIT_DAYS:-7}" ]]; then
321	                CERT_ISSUES+=("$domain: ${days_remaining}d CRITICAL")
322	                SHOULD_SEND=true
323	            elif [[ "$days_remaining" -le "${CERT_WARN_DAYS:-30}" ]]; then
324	                CERT_ISSUES+=("$domain: ${days_remaining}d warning")
325	                [[ "$DIGEST_SMART_ON_CERT_WARN" == true ]] && SHOULD_SEND=true
326	            else
327	                log "$ICON_CERT $domain: ${days_remaining}d remaining ✅"
328	            fi
329	        fi
330	    done
331	    if [[ ${#CERT_ISSUES[@]} -gt 0 ]]; then
332	        DIGEST_LINES+=("$ICON_CERT Certificates: ${CERT_ISSUES[*]}")
333	        FINDINGS+=("Cert issues: ${CERT_ISSUES[*]}")
334	    else
335	        DIGEST_LINES+=("$ICON_CERT Certificates: all healthy ✅")
336	    fi
337	fi
338
339	# ==============================================================================================
340	# ── Smart profile — exit silently if nothing to report ────────────────────────────────────────
341	# ==============================================================================================
342	if [[ "$DIGEST_PROFILE" == "smart" && "$SHOULD_SEND" == false ]]; then
343	    echo "Profile: smart — no findings worth reporting — silent exit"
344	    exit 0
345	fi
346
347	# ==============================================================================================
348	# ━━━ Build and Send Digest ━━━
349	# ==============================================================================================
350	echo ""
351	echo "━━━ $ICON_DIGEST Health Digest — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
352	echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
353	echo ""
354
355	for line in "${DIGEST_LINES[@]}"; do
356	    echo "  $line"
357	done
358
359	echo ""
360	echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━"
361	echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
362	echo "$ICON_DIGEST  Profile:  $DIGEST_PROFILE"
363	echo "$ICON_ERROR   Issues:   ${#ISSUES[@]}"
364	echo "$ICON_WARN    Findings: ${#FINDINGS[@]}"
365	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
366
367	# Build notification message
368	NOTIFY_MSG="Health Digest — $MY_ID ($LOCAL_SERVER_NAME)"
369	[[ ${#ISSUES[@]} -gt 0 ]]   && NOTIFY_MSG+=" | Issues: ${ISSUES[*]}"
370	[[ ${#FINDINGS[@]} -gt 0 ]] && NOTIFY_MSG+=" | Findings: ${FINDINGS[*]}"
371	[[ ${#ISSUES[@]} -eq 0 && ${#FINDINGS[@]} -eq 0 ]] && NOTIFY_MSG+=" | All systems healthy"
372
373	NOTIFY_SEV="normal"
374	[[ ${#ISSUES[@]} -gt 0 ]] && NOTIFY_SEV="warning"
375
376	if [[ "$DRY_RUN" == true ]]; then
377	    warn "DRY RUN — digest generated but not sent"
378	elif [[ "$SHOULD_SEND" == true ]]; then
379	    notify "$NOTIFY_MSG" "Health Digest" "$NOTIFY_SEV"
380	    echo "Digest sent"
381	fi


1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= ZFS Memory Snapshot ============================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Weekly ZFS pool health and memory diagnostic report. Scheduled Sunday 6am —
9	# first in the Sunday monitoring block, before other scripts run. Informational
10	# only — system_watchdog.sh handles threshold-based intervention.
11	#
12	# Combines ZFS pool status, ARC statistics, Docker memory usage, and kernel
13	# memory pressure into a single snapshot. Output goes to both console (for User
14	# Scripts output log) and ZFS_REPORT_LOG for week-over-week comparison.
15	#
16	# ==============================================================================================
17	# OPERATIONAL MODEL
18	# ==============================================================================================
19	#
20	# Five report sections (each skips gracefully if its data source is unavailable):
21	#
22	#   ZFS pool health   — status, state, errors per pool. Pools in
23	#                       ZFS_REPORT_IGNORE_POOLS excluded from the report
24	#                       (still fully monitored by unRAID — report-only exclusion).
25	#   ARC statistics    — current ARC vs max, metadata pressure, hit rate.
26	#                       Warns if ARC utilisation exceeds ZFS_REPORT_ARC_WARN_PCT.
27	#   Memory status     — total, free, available RAM.
28	#                       Warns if free < ZFS_REPORT_FREE_WARN_GB or
29	#                       available < ZFS_REPORT_AVAIL_WARN_GB.
30	#   Docker memory     — top ZFS_REPORT_DOCKER_TOP containers by memory usage.
31	#                       Useful for spotting containers approaching watchdog limits.
32	#   Kernel pressure   — vmstat snapshot (3 samples).
33	#
34	# ==============================================================================================
35	# OPERATIONAL SAFEGUARDS
36	# ==============================================================================================
37	#
38	# Single Instance Lock
39	#   acquire_lock prevents duplicate runs — zpool and docker stats are slow.
40	#
41	# Per-Host Pool Ignore List
42	#   detect_hosts() aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS.
43	#   Single-disk JBOD members excluded from report noise per server.
44	#
45	# ZFS Availability Guard
46	#   Skips pool and ARC sections gracefully if ZFS is not available on this server.
47	#
48	# Docker Availability Guard
49	#   Skips container memory section gracefully if Docker is not responding.
50	#
51	# Docker Stats Timeout
52	#   DOCKER_TIMEOUT caps docker stats calls. A hung daemon does not block the report.
53	#
54	# Notification Validated
55	#   platform_require_cmd confirms the notify script is present before use.
56	#
57	# ==============================================================================================
58	# STATE FILES
59	# ==============================================================================================
60	#
61	#   ZFS_REPORT_LOG   — /var/log/zfs-weekly-health.log (tmpfs, resets on reboot)
62	#       Weekly report written here for comparison across runs. Open the log to
63	#       see pool health trend week over week without remembering last week's values.
64	#       In dry-run mode, console only — nothing written.
65	#
66	# ==============================================================================================
67	# CONFIGURATION
68	# ==============================================================================================
69	#
70	# host*.conf
71	#
72	#   HOST*_ZFS_REPORT_IGNORE_POOLS
73	#       Pools excluded from health reporting. Single-disk JBOD members generate
74	#       expected high-usage warnings — exclude them to reduce report noise.
75	#       Aliased by detect_hosts() → ZFS_REPORT_IGNORE_POOLS.
76	#
77	# master.conf
78	#
79	#   ZFS_REPORT_LOG
80	#       Log file path for weekly reports. (default: /var/log/zfs-weekly-health.log)
81	#
82	#   ZFS_REPORT_ARC_WARN_PCT
83	#       Warn if ARC is using more than this percentage of its configured max. (default: 90)
84	#
85	#   ZFS_REPORT_FREE_WARN_GB
86	#       Warn if free RAM is below this threshold in GB. (default: 10)
87	#
88	#   ZFS_REPORT_AVAIL_WARN_GB
89	#       Warn if available RAM is below this threshold in GB. (default: 20)
90	#
91	#   ZFS_REPORT_DOCKER_TOP
92	#       Number of top Docker containers by memory usage to include. (default: 10)
93	#
94	# ==============================================================================================
95	# RUNTIME MODES
96	# ==============================================================================================
97	#
98	# zfs_memory_snapshot.sh
99	#     Generate report, write to ZFS_REPORT_LOG and console. Notify on warnings.
100	#
101	# zfs_memory_snapshot.sh --dry-run
102	#     Generate report to console only. No log write, no notifications.
103	#
104	# zfs_memory_snapshot.sh --status
105	#     Show pool ignore list and threshold configuration. Then exit.
106	#
107	# zfs_memory_snapshot.sh --log
108	#     Verbose output during report generation.
109	#
110	# ==============================================================================================
111
112	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
113
114	source "$SCRIPT_DIR/../load_config.sh"
115
116	parse_args "$@"
117
118	DOCKER_TIMEOUT=15
119
120	# ==============================================================================================
121	# ━━━ Setup ━━━
122	# ==============================================================================================
123	if [[ "$EUID" -ne 0 ]]; then
124	    error "Must be run as root"
125	    exit 1
126	fi
127
128
129	acquire_lock
130
131	# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS
132	detect_hosts
133
134	# Build ignore pool lookup map — O(1) check per pool
135	declare -A IGNORE_POOL_MAP
136	for pool in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do
137	    [[ -n "$pool" ]] && IGNORE_POOL_MAP["$pool"]=1
138	done
139
140	log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
141	log "Ignoring pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
142	log "$ICON_GEAR Config: arc-warn=${ZFS_REPORT_ARC_WARN_PCT}% free-warn=${ZFS_REPORT_FREE_WARN_GB}GB avail-warn=${ZFS_REPORT_AVAIL_WARN_GB}GB docker-top=${ZFS_REPORT_DOCKER_TOP}"
143
144	# Tee output to log file unless dry run
145	if [[ "$DRY_RUN" == false ]]; then
146	    mkdir -p "$(dirname "$ZFS_REPORT_LOG")"
147	    exec > >(tee -a "$ZFS_REPORT_LOG") 2>&1
148	fi
149
150	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — output will not be written to log"
151
152	# ==============================================================================================
153	# ━━━ Status ━━━
154	# ==============================================================================================
155	if [[ "$SHOW_STATUS" == true ]]; then
156	    echo ""
157	    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
158	    echo "$ICON_HOST       Identity:       $MY_ID ($LOCAL_SERVER_NAME)"
159	    echo "$ICON_ZFS        Log file:       $ZFS_REPORT_LOG"
160	    echo "$ICON_ZFS        ARC warn:       ${ZFS_REPORT_ARC_WARN_PCT}%"
161	    echo "$ICON_MEM        Free RAM warn:  ${ZFS_REPORT_FREE_WARN_GB}GB"
162	    echo "$ICON_MEM        Avail warn:     ${ZFS_REPORT_AVAIL_WARN_GB}GB"
163	    echo "$ICON_CONTAINERS Docker top:     $ZFS_REPORT_DOCKER_TOP"
164	    echo "$ICON_ZFS        Ignore pools:   ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
165	    echo "$ICON_GEAR       Dry Run:        $DRY_RUN"
166	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
167	    exit 0
168	fi
169
170	# ==============================================================================================
171	# ━━━ Report ━━━
172	# ==============================================================================================
173	WARNINGS=()
174	START=$(date +%s)
175	DATE=$(date '+%Y-%m-%d %H:%M:%S')
176
177	echo ""
178	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
179	echo "  $ICON_ZFS  ZFS WEEKLY HEALTH REPORT — $DATE"
180	echo "  $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME"
181	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
182
183	# ── ZFS Pool Health ───────────────────────────────────────────────────────────────────────────
184	echo ""
185	echo "━━━ $ICON_ZFS ZFS Pool Health ━━━"
186
187	if ! command -v zpool >/dev/null 2>&1; then
188	    warn "ZFS not available on this system — skipping pool checks"
189	else
190	    # Pool status — filtered to key lines, ignoring specified pools
191	    CURRENT_POOL=""
192	    while IFS= read -r line; do
193	        if [[ "$line" =~ ^[[:space:]]*pool:[[:space:]]*(.+) ]]; then
194	            CURRENT_POOL="${BASH_REMATCH[1]// /}"
195	        fi
196	        [[ -n "${IGNORE_POOL_MAP[$CURRENT_POOL]:-}" ]] && continue
197	        echo "  $line"
198	    done < <(zpool status 2>/dev/null | grep -E "pool:|state:|status:|errors:|scan:")
199
200	    echo ""
201
202	    # Pool list — filter out ignored pools
203	    zpool list 2>/dev/null | while IFS= read -r line; do
204	        if [[ "$line" == NAME* ]]; then
205	            echo "  $line"
206	            continue
207	        fi
208	        pool_name=$(echo "$line" | awk '{print $1}')
209	        [[ -n "${IGNORE_POOL_MAP[$pool_name]:-}" ]] && continue
210	        echo "  $line"
211	    done
212
213	    # Check for unhealthy non-ignored pools
214	    UNHEALTHY=$(zpool list -H -o name,health 2>/dev/null | \
215	        while IFS=$'\t' read -r name health; do
216	            [[ -n "${IGNORE_POOL_MAP[$name]:-}" ]] && continue
217	            [[ "$health" != "ONLINE" ]] && echo "$name: $health"
218	        done)
219
220	    if [[ -n "$UNHEALTHY" ]]; then
221	        error "One or more ZFS pools are NOT ONLINE: $UNHEALTHY"
222	        WARNINGS+=("ZFS pool unhealthy: $UNHEALTHY")
223	    else
224	        echo "All monitored ZFS pools are ONLINE ✅"
225	    fi
226
227	    if [[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]]; then
228	        log "Ignored pools: ${ZFS_REPORT_IGNORE_POOLS[*]}"
229	    fi
230	fi
231
232	# ── ARC Statistics ────────────────────────────────────────────────────────────────────────────
233	echo ""
234	echo "━━━ $ICON_ZFS ARC Statistics ━━━"
235
236	if [[ ! -f /proc/spl/kstat/zfs/arcstats ]]; then
237	    warn "ZFS arcstats not available — skipping ARC section"
238	else
239	    ARC_MAX=$(cat /sys/module/zfs/parameters/zfs_arc_max 2>/dev/null || \
240	              awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
241	    ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
242	    ARC_META_USED=$(awk '/^arc_meta_used / {print $3}' /proc/spl/kstat/zfs/arcstats)
243
244	    ARC_MAX_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_MAX / 1073741824}")
245	    ARC_CUR_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE / 1073741824}")
246	    ARC_META_GB=$(awk "BEGIN {printf \"%.1f\", $ARC_META_USED / 1073741824}")
247	    ARC_PCT=$(awk "BEGIN {printf \"%.1f\", $ARC_SIZE * 100 / $ARC_MAX}")
248	    ARC_PCT_INT=$(printf "%.0f" "$ARC_PCT")
249
250	    echo "  $ICON_ZFS ARC Max:         ${ARC_MAX_GB}GB"
251	    echo "  $ICON_ZFS ARC Current:     ${ARC_CUR_GB}GB"
252	    echo "  $ICON_ZFS ARC Meta Used:   ${ARC_META_GB}GB"
253	    echo "  $ICON_ZFS ARC Utilization: ${ARC_PCT}%"
254
255	    if [[ "$ARC_PCT_INT" -ge "$ZFS_REPORT_ARC_WARN_PCT" ]]; then
256	        warn "ARC utilization ${ARC_PCT}% — above ${ZFS_REPORT_ARC_WARN_PCT}% threshold"
257	        WARNINGS+=("ARC high: ${ARC_PCT}%")
258	    else
259	        log "ARC utilization ${ARC_PCT}% — within threshold ✅"
260	    fi
261
262	    echo ""
263	    META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' \
264	        /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
265	    META_MFU_GHOST=$(awk '/^mfu_ghost_metadata / {print $3}' \
266	        /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
267	    META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' \
268	        /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
269
270	    MRU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MRU_GHOST / 1073741824}")
271	    MFU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MFU_GHOST / 1073741824}")
272
273	    echo "  $ICON_ZFS MRU Ghost:       ${MRU_GB}GB"
274	    echo "  $ICON_ZFS MFU Ghost:       ${MFU_GB}GB"
275	    echo "  $ICON_ZFS Metadata Misses: ${META_MISSES}"
276	fi
277
278	# ── Memory Status ─────────────────────────────────────────────────────────────────────────────
279	echo ""
280	echo "━━━ $ICON_MEM Memory Status ━━━"
281
282	FREE_HUMAN=$(free -h  | awk '/Mem:/ {print $4}')
283	AVAIL_HUMAN=$(free -h | awk '/Mem:/ {print $7}')
284	TOTAL_HUMAN=$(free -h | awk '/Mem:/ {print $2}')
285	FREE_GB=$(free -g     | awk '/Mem:/ {print $4}')
286	AVAIL_GB=$(free -g    | awk '/Mem:/ {print $7}')
287
288	echo "  $ICON_MEM Total RAM:      $TOTAL_HUMAN"
289	echo "  $ICON_MEM Free RAM:       $FREE_HUMAN"
290	echo "  $ICON_MEM Available RAM:  $AVAIL_HUMAN"
291
292	if [[ "$FREE_GB" -lt "$ZFS_REPORT_FREE_WARN_GB" ]]; then
293	    warn "Free RAM ${FREE_HUMAN} — below ${ZFS_REPORT_FREE_WARN_GB}GB threshold"
294	    WARNINGS+=("Low free RAM: ${FREE_HUMAN}")
295	else
296	    log "Free RAM ${FREE_HUMAN} — within threshold ✅"
297	fi
298
299	if [[ "$AVAIL_GB" -lt "$ZFS_REPORT_AVAIL_WARN_GB" ]]; then
300	    warn "Available RAM ${AVAIL_HUMAN} — below ${ZFS_REPORT_AVAIL_WARN_GB}GB threshold"
301	    WARNINGS+=("Low available RAM: ${AVAIL_HUMAN}")
302	else
303	    log "Available RAM ${AVAIL_HUMAN} — within threshold ✅"
304	fi
305
306	# ── Docker Memory ─────────────────────────────────────────────────────────────────────────────
307	echo ""
308	echo "━━━ $ICON_CONTAINERS Top $ZFS_REPORT_DOCKER_TOP Docker Memory Users ━━━"
309
310	if ! command -v docker >/dev/null 2>&1; then
311	    warn "Docker not available — skipping container memory section"
312	else
313	    timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
314	        --format "table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}" \
315	        2>/dev/null | head -n $(( ZFS_REPORT_DOCKER_TOP + 1 )) | \
316	        while IFS= read -r line; do
317	            echo "  $line"
318	        done
319	fi
320
321	# ── Kernel Pressure ───────────────────────────────────────────────────────────────────────────
322	echo ""
323	echo "━━━ $ICON_GEAR Kernel Pressure ━━━"
324
325	if ! command -v vmstat >/dev/null 2>&1; then
326	    warn "vmstat not available — skipping kernel pressure section"
327	else
328	    vmstat 1 3 2>/dev/null | while IFS= read -r line; do
329	        echo "  $line"
330	    done
331	fi
332
333	END=$(date +%s)
334
335	# ── Summary ───────────────────────────────────────────────────────────────────────────────────
336	echo ""
337	echo "━━━━━ $ICON_SUMMARY ZFS REPORT SUMMARY ━━━━━"
338	echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
339	echo "$ICON_TIME    Duration: $(format_duration $(( END - START )))"
340	echo "$ICON_ZFS     Log:      $ZFS_REPORT_LOG"
341	[[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]] && \
342	    log "Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]}"
343	echo ""
344
345	if [[ ${#WARNINGS[@]} -eq 0 ]]; then
346	    echo "$ICON_DONE All checks within thresholds ✅"
347	else
348	    echo "$ICON_WARN Warnings: ${#WARNINGS[@]}"
349	    for w in "${WARNINGS[@]}"; do
350	        echo "  $ICON_WARN $w"
351	    done
352	    notify "ZFS weekly report on $(hostname) — ${#WARNINGS[@]} warning(s): ${WARNINGS[*]}" \
353	        "ZFS Report" "warning"
354	fi
355	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
356
357	[[ ${#WARNINGS[@]} -gt 0 ]] && exit 1
358	exit 0


Now let me read the Orchestrator scripts.


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Array Start Orchestrator ===================================
4	# ==============================================================================================
5	# Single entry point for array start — fired by the Varaverk plugin's
6	# disks_mounted event hook (Plugin/unraid/event/disks_mounted/array_start_jobs).
7	# Launches everything configured in ARRAY_START_SCRIPTS in master.conf.
8	# This script exits after launching all scripts — the event hook sees it complete normally.
9	#
10	# ── WHAT IT LAUNCHES ──────────────────────────────────────────────────────────────────────────
11	# Configured in master.conf ARRAY_START_SCRIPTS — no changes to this script ever needed.
12	# Current order (order matters — see below):
13	#
14	#   ONE-SHOT (run and exit naturally):
15	#     System_Essentials/unraid_api_key_renew.sh — re-register Varaverk API key at boot
16	#     System_Essentials/inotify_tuning.sh       — raise inotify limits before containers start
17	#     System_Essentials/docker_syslog_filter.sh — suppress veth log noise before logs fill
18	#     System_Essentials/php_fpm_max_children.sh — WebGUI performance tuning
19	#     Transcodes/ramdisk_setup.sh               — create tmpfs + symlink before Emby starts
20	#     Docker_Essentials/docker_network_connect.sh — ensure networks + container connections
21	#
22	#   CONTINUOUS (run until array stops):
23	#     Fallback/fallback.sh                      — mutual fallback monitor
24	#
25	#   NOTE: watchdogs (docker, system, stability) are NOT launched here.
26	#   They run via watchdog_orchestrator.sh every 15 min (cron), not as daemons.
27	#
28	# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
29	#   unraid_api_key_renew.sh — before anything else — self-heals API registry at boot
30	#   inotify_tuning.sh     — must run BEFORE Code-Server and other containers start
31	#                           containers that start with low inotify limits keep them ✅
32	#   docker_syslog_filter  — must run BEFORE any container starts creating veth interfaces
33	#   ramdisk_setup.sh      — must run BEFORE Emby starts transcoding
34	#   docker_network_connect — must run BEFORE watchdogs check container states
35	#   fallback.sh           — last — needs everything else stable to make decisions
36	#
37	# ── ONE-SHOT vs CONTINUOUS DETECTION ─────────────────────────────────────────────────────────
38	# Script is launched in background with bash script.sh &
39	# After 1 second: if PID still alive → continuous (running in background)
40	#                  if PID dead + exit 0 → one-shot completed successfully
41	#                  if PID dead + exit N → failure
42	#
43	# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
44	#   Root check           — all launched scripts require root
45	#   acquire_lock         — prevents duplicate array start launches
46	#   detect_hosts()       — MY_ID in notifications
47	#   platform_require_cmd  — notify validated before use
48	#   chmod +x auto-fix    — non-executable scripts fixed before launch
49	#   Full path on failure — shows exact path for debugging
50	#   notify on failures   — alert if any script fails to launch
51	#
52	# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
53	#   ARRAY_START_SCRIPTS — ordered list of scripts to launch at array start
54	#
55	# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
56	#   array_started.sh              — normal launch (called by Varaverk disks_mounted event hook)
57	#   array_started.sh --dry-run    — show what would be launched without launching
58	#   array_started.sh --status     — show configured scripts and their current state
59	#   array_started.sh --log        — verbose output per script
60	# ==============================================================================================
61
62	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
63	ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
64
65	source "$ECOSYSTEM_ROOT/load_config.sh"
66
67	parse_args "$@"
68
69	# ==============================================================================================
70	# ━━━ Setup ━━━
71	# ==============================================================================================
72	if [[ "$EUID" -ne 0 ]]; then
73	    error "Must be run as root"
74	    exit 1
75	fi
76
77
78	acquire_lock
79
80	detect_hosts
81
82	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — scripts will not be launched"
83
84	# ==============================================================================================
85	# ━━━ Status ━━━
86	# ==============================================================================================
87	if [[ "$SHOW_STATUS" == true ]]; then
88	    echo ""
89	    echo "━━━━━ $ICON_SUMMARY ARRAY START STATUS ━━━━━"
90	    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
91	    echo "$ICON_GEAR  Scripts:  ${#ARRAY_START_SCRIPTS[@]} configured"
92	    echo ""
93
94	    for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
95	        [[ -z "$relative_path" ]] && continue
96	        script_path="$ECOSYSTEM_ROOT/$relative_path"
97	        script_name=$(basename "$script_path")
98
99	        if [[ ! -f "$script_path" ]]; then
100	            echo "  $ICON_ERROR $script_name — FILE NOT FOUND"
101	            echo "    $script_path"
102	            continue
103	        fi
104
105	        [[ ! -x "$script_path" ]] && flag=" (not executable — will auto-fix)" || flag=""
106
107	        # Check if currently running
108	        if pgrep -f "$script_path" >/dev/null 2>&1; then
109	            RUN_PID=$(pgrep -f "$script_path" | head -1)
110	            echo "  $ICON_RUNNING $script_name — RUNNING (PID $RUN_PID)${flag}"
111	        else
112	            echo "  $ICON_NOT_RUNNING $script_name — not running${flag}"
113	        fi
114	    done
115	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
116	    exit 0
117	fi
118
119	# ==============================================================================================
120	# ━━━ Launch Scripts ━━━
121	# ==============================================================================================
122	if [[ ${#ARRAY_START_SCRIPTS[@]} -eq 0 ]]; then
123	    notify "Array started on $LOCAL_SERVER_NAME ($MY_ID) but ARRAY_START_SCRIPTS is empty — boot sequence skipped. Check master.conf." \
124	        "Array Start" "alert"
125	    error "ARRAY_START_SCRIPTS is empty — check master.conf"
126	    exit 1
127	fi
128
129	echo ""
130	echo "━━━ $ICON_GEAR Array Start — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
131	log "Ecosystem root: $ECOSYSTEM_ROOT"
132	log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..."
133	echo ""
134
135	START=$(date +%s)
136	LAUNCHED=0
137	FAILED=0
138	FAILED_SCRIPTS=()
139
140	for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
141	    [[ -z "$relative_path" ]] && continue
142
143	    SCRIPT_PATH="$ECOSYSTEM_ROOT/$relative_path"
144	    SCRIPT_NAME=$(basename "$SCRIPT_PATH")
145
146	    # File existence check
147	    if [[ ! -f "$SCRIPT_PATH" ]]; then
148	        error "$SCRIPT_NAME — not found"
149	        error "  Expected: $SCRIPT_PATH"
150	        (( FAILED++ ))
151	        FAILED_SCRIPTS+=("$SCRIPT_NAME")
152	        continue
153	    fi
154
155	    # Auto-fix permissions — chmod +x if needed
156	    if [[ ! -x "$SCRIPT_PATH" ]]; then
157	        warn "$SCRIPT_NAME — not executable, fixing..."
158	        chmod +x "$SCRIPT_PATH" || {
159	            error "$SCRIPT_NAME — chmod +x failed"
160	            (( FAILED++ ))
161	            FAILED_SCRIPTS+=("$SCRIPT_NAME")
162	            continue
163	        }
164	    fi
165
166	    if [[ "$DRY_RUN" == true ]]; then
167	        warn "DRY RUN — would launch: $SCRIPT_NAME"
168	        (( LAUNCHED++ ))
169	        continue
170	    fi
171
172	    log "$ICON_START Launching $SCRIPT_NAME..."
173	    bash "$SCRIPT_PATH" &
174	    PID=$!
175
176	    # Brief settle — 1s enough to detect immediate failures
177	    sleep 1
178
179	    if kill -0 "$PID" 2>/dev/null; then
180	        # Still running → continuous script
181	        warn "$SCRIPT_NAME — running (PID $PID) ✅"
182	        (( LAUNCHED++ ))
183	    else
184	        # Exited — check if one-shot success or failure
185	        wait "$PID"
186	        EXIT_CODE=$?
187	        if [[ "$EXIT_CODE" -eq 0 ]]; then
188	            log "$SCRIPT_NAME — completed (one-shot) ✅"
189	            (( LAUNCHED++ ))
190	        else
191	            error "$SCRIPT_NAME — exited with code $EXIT_CODE"
192	            error "  Path: $SCRIPT_PATH"
193	            (( FAILED++ ))
194	            FAILED_SCRIPTS+=("$SCRIPT_NAME")
195	        fi
196	    fi
197
198	done
199
200	END=$(date +%s)
201
202	# ==============================================================================================
203	# ━━━ Summary ━━━
204	# ==============================================================================================
205	echo ""
206	echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
207	echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
208	echo "$ICON_SUCCESS Launched: $LAUNCHED"
209	[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR  Failed:   $FAILED — ${FAILED_SCRIPTS[*]}"
210	echo "$ICON_TIME    Duration: $(format_duration $(( END - START )))"
211	echo ""
212
213	if [[ "$DRY_RUN" == true ]]; then
214	    warn "DRY RUN — no scripts launched"
215	elif [[ "$FAILED" -gt 0 ]]; then
216	    warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}"
217	    notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
218	        "Array Start" "warning"
219	else
220	    echo "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
221	fi
222	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Array Stop Orchestrator ====================================
4	# ==============================================================================================
5	# Planned shutdown orchestrator — stops all active processes cleanly before array maintenance.
6	# Runs ARRAY_STOP_SCRIPTS from master.conf sequentially, each confirmed complete before next.
7	#
8	# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
9	#   1. user_scripts_stop.sh        — kill background user scripts (prevents new operations)
10	#   2. rsync_stop.sh --rsync-only  — kill rsync; skip container recovery (handled in step 4)
11	#   3. mover_stop.sh               — stop mover after rsync (both write to same paths)
12	#   4. docker_container_stop.sh    — stop all containers one-by-one with verification
13	#
14	# ── WHY THIS ORDER ────────────────────────────────────────────────────────────────────────────
15	# User scripts stopped first — they can spawn new rsync/docker operations mid-shutdown.
16	# Rsync before mover — both write to the same paths; running together risks corruption.
17	# Containers last — apps should stay available as long as possible during shutdown prep.
18	#
19	# ── SEQUENTIAL vs BACKGROUND ─────────────────────────────────────────────────────────────────
20	# Unlike array_started.sh, all scripts run in the foreground. Each must complete (pass or fail)
21	# before the next starts — a failed stop is noted but does not prevent remaining steps.
22	#
23	# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
24	#   Root check           — all stop scripts require root
25	#   acquire_lock         — prevents concurrent array stop runs
26	#   detect_hosts()       — MY_ID in notifications and logs
27	#   platform_require_cmd  — notify validated before use
28	#   Non-fatal steps      — a failed step is logged but remaining steps still run
29	#   notify on failures   — alert if any stop script fails
30	#
31	# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
32	#   ARRAY_STOP_SCRIPTS — ordered list of stop scripts to run
33	#
34	# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
35	#   array_stopping.sh              — run full stop sequence
36	#   array_stopping.sh --dry-run    — preview without stopping anything
37	#   array_stopping.sh --status     — show configured scripts and exit
38	#   array_stopping.sh --log        — verbose output
39	# ==============================================================================================
40
41	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
42	ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
43
44	source "$ECOSYSTEM_ROOT/load_config.sh"
45
46	parse_args "$@"
47
48	# ==============================================================================================
49	# ━━━ Setup ━━━
50	# ==============================================================================================
51	if [[ "$EUID" -ne 0 ]]; then
52	    error "Must be run as root"
53	    exit 1
54	fi
55
56
57	acquire_lock
58
59	if ! command -v docker &>/dev/null; then
60	    error "Docker command not found"
61	    exit 1
62	fi
63
64	detect_hosts
65
66	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no stop scripts will be executed"
67
68	# ==============================================================================================
69	# ━━━ Status ━━━
70	# ==============================================================================================
71	if [[ "$SHOW_STATUS" == true ]]; then
72	    echo ""
73	    echo "━━━━━ $ICON_SUMMARY ARRAY STOP STATUS ━━━━━"
74	    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
75	    echo "$ICON_GEAR  Scripts:  ${#ARRAY_STOP_SCRIPTS[@]} configured"
76	    echo ""
77
78	    for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
79	        [[ -z "$entry" ]] && continue
80	        read -r -a parts <<< "$entry"
81	        script_path="$ECOSYSTEM_ROOT/${parts[0]}"
82	        script_name=$(basename "${parts[0]}")
83	        extra_args=("${parts[@]:1}")
84	        if [[ ! -f "$script_path" ]]; then
85	            echo "  $ICON_ERROR $script_name — FILE NOT FOUND"
86	        else
87	            echo "  $ICON_GEAR $script_name${extra_args:+ ${extra_args[*]}}"
88	        fi
89	    done
90	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
91	    exit 0
92	fi
93
94	# ==============================================================================================
95	# ━━━ Stop Sequence ━━━
96	# ==============================================================================================
97	echo ""
98	echo "━━━ $ICON_STOP Array Stop — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
99	echo "$ICON_GEAR Running ${#ARRAY_STOP_SCRIPTS[@]} stop script(s) sequentially..."
100	echo ""
101
102	START=$(date +%s)
103	PASSED=()
104	FAILED=()
105	STEP=0
106
107	for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
108	    [[ -z "$entry" ]] && continue
109	    (( STEP++ ))
110
111	    read -r -a parts <<< "$entry"
112	    script_path="$ECOSYSTEM_ROOT/${parts[0]}"
113	    script_name=$(basename "${parts[0]}")
114	    extra_args=("${parts[@]:1}")
115
116	    echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
117
118	    if [[ ! -f "$script_path" ]]; then
119	        error "$script_name — not found at $script_path"
120	        FAILED+=("$script_name")
121	        echo ""
122	        continue
123	    fi
124
125	    if [[ ! -x "$script_path" ]]; then
126	        warn "$script_name — not executable, fixing..."
127	        chmod +x "$script_path" || {
128	            error "$script_name — chmod +x failed"
129	            FAILED+=("$script_name")
130	            echo ""
131	            continue
132	        }
133	    fi
134
135	    if [[ "$DRY_RUN" == true ]]; then
136	        warn "DRY RUN — would run: $script_name ${extra_args[*]}"
137	        PASSED+=("$script_name")
138	        echo ""
139	        continue
140	    fi
141
142	    if bash "$script_path" "${extra_args[@]}"; then
143	        log "$script_name — done ✅"
144	        PASSED+=("$script_name")
145	    else
146	        warn "$script_name — failed (exit $?) — continuing to next step"
147	        FAILED+=("$script_name")
148	    fi
149
150	    echo ""
151	done
152
153	END=$(date +%s)
154
155	# ==============================================================================================
156	# ━━━ Summary ━━━
157	# ==============================================================================================
158	echo "━━━━━ $ICON_SUMMARY ARRAY STOP SUMMARY ━━━━━"
159	echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
160	echo "$ICON_TIME    Duration: $(format_duration $(( END - START )))"
161	[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE    Passed:   ${PASSED[*]}"
162	[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR   Failed:   ${FAILED[*]}"
163	echo ""
164
165	if [[ "$DRY_RUN" == true ]]; then
166	    warn "DRY RUN — no changes made"
167	elif [[ ${#FAILED[@]} -eq 0 ]]; then
168	    echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
169	    notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
170	        "Array Stop" "normal"
171	else
172	    warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}"
173	    notify "Array stop on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \
174	        "Array Stop" "warning"
175	fi
176	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
177
178	[[ ${#FAILED[@]} -gt 0 ]] && exit 1
179	exit 0
180


1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= Critical Sync Maintenance ======================================
4	# ==============================================================================================
5	# Orchestrator for time-sensitive syncs that run every 30 minutes.
6	# Keeps the mirror current between the less frequent daily and weekly windows.
7	# Schedule: */30 * * * * (every 30 minutes via User Scripts plugin)
8	#
9	# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
10	#   1. Critical-Data rsync    — auth stack, NPM config, certs (containers stopped both sides)
11	#   2. emby-fallback rsync    — dirty Emby sync (watch states, library — Emby stays running)
12	#   3. CRITICAL_MAINTENANCE_SCRIPTS — any scripts configured for critical window
13	#   4. partnership --check    — read both state files, detect changes, act accordingly
14	#
15	# ── WHY EVERY 30 MINUTES ──────────────────────────────────────────────────────────────────────
16	#   Auth stack changes (new users, proxy rules, certs) propagate within 30min ✅
17	#   Emby watch states stay in sync — mirror users see correct playback position ✅
18	#   Partnership state changes detected and acted on quickly ✅
19	#   Lock prevents: daily rsync doing Critical-Data mid-critical window ✅
20	#
21	# ── RSYNC GATE ────────────────────────────────────────────────────────────────────────────────
22	#   RSYNC_ENABLED=false           → skips all syncs (global gate)
23	#   CRITICAL_RSYNC_ENABLED=false  → skips critical syncs only (per-orchestrator gate)
24	#   partnership --check always runs regardless — state check doesn't need rsync
25	#
26	# ── LOCK BEHAVIOUR ────────────────────────────────────────────────────────────────────────────
27	#   acquire_lock "strict" — if previous 30min run still going, skip this cycle entirely
28	#   Critical-Data taking > 30min is a problem worth knowing about
29	#   Strict mode prevents pile-up without waiting — log and move on ✅
30	#
31	# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
32	#   Runs 48 times per day — clean runs must produce zero output ✅
33	#   Only failures and notable events produce visible output
34	#
35	# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
36	#   CRITICAL_RSYNC_ENABLED       — enable/disable rsync section
37	#   CRITICAL_SYNC_SHARES         — shares synced every 30min (HOST*_CRITICAL_SYNC_SHARES)
38	#   CRITICAL_MAINTENANCE_SCRIPTS — scripts run in critical window (optional)
39	#   PARTNERSHIP_ENABLED          — enable/disable partnership check
40	#
41	# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
42	#   critical_sync_maintenance.sh              — normal run
43	#   critical_sync_maintenance.sh --dry-run    — preview syncs without transferring
44	#   critical_sync_maintenance.sh --log        — verbose per-share output
45	#   critical_sync_maintenance.sh --status     — show configuration and exit
46	# ==============================================================================================
47
48	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
49
50	source "$SCRIPT_DIR/../load_config.sh"
51
52	parse_args "$@"
53
54	# ==============================================================================================
55	# ━━━ Setup ━━━
56	# ==============================================================================================
57	if [[ "$EUID" -ne 0 ]]; then
58	    error "Must be run as root"
59	    exit 1
60	fi
61
62
63	acquire_lock "strict"
64
65	detect_hosts
66	resolve_remote_ip
67
68	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
69
70	# ==============================================================================================
71	# ━━━ Status ━━━
72	# ==============================================================================================
73	if [[ "$SHOW_STATUS" == true ]]; then
74	    echo ""
75	    echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC STATUS ━━━━━"
76	    echo "$ICON_HOST    Identity:           $MY_ID ($LOCAL_SERVER_NAME)"
77	    echo "$ICON_NET     Remote:             $REMOTE_ID ($REMOTE_SERVER_NAME)"
78	    echo "$ICON_SYNC    Rsync enabled:      ${RSYNC_ENABLED:-true}"
79	    echo "$ICON_SYNC    Critical enabled:   ${CRITICAL_RSYNC_ENABLED:-false}"
80	    echo "$ICON_SHIELD  Partnership:        ${PARTNERSHIP_ENABLED:-false}"
81	    echo ""
82	    echo "━━━ Critical Sync Shares ━━━"
83	    if [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
84	        warn "  No CRITICAL_SYNC_SHARES configured"
85	    else
86	        for share in "${CRITICAL_SYNC_SHARES[@]}"; do
87	            [[ -z "$share" ]] && continue
88	            SHARE_PATH="${share%%|*}"
89	            SHARE_PROFILE="${share##*|}"
90	            SHARE_NAME=$(basename "$SHARE_PATH")
91	            [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]] && \
92	                echo "  $ICON_SYNC $SHARE_NAME — no profile" || \
93	                echo "  $ICON_SYNC $SHARE_NAME — profile: $SHARE_PROFILE"
94	        done
95	    fi
96	    echo ""
97	    echo "━━━ Critical Maintenance Scripts ━━━"
98	    if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
99	        echo "  None configured"
100	    else
101	        for entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
102	            [[ -z "$entry" || "$entry" == \#* ]] && continue
103	            echo "  $ICON_GEAR $(basename "${entry%% *}")"
104	        done
105	    fi
106	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
107	    exit 0
108	fi
109
110	# ==============================================================================================
111	# ━━━ Critical Shares Sync ━━━
112	# ==============================================================================================
113	START=$(date +%s)
114	RSYNC_OK=false
115	PASS=()
116	FAIL=()
117
118	log "$ICON_SYNC Critical shares (${#CRITICAL_SYNC_SHARES[@]}): $(for s in "${CRITICAL_SYNC_SHARES[@]}"; do printf '%s ' "$(basename "${s%%|*}")"; done)"
119	[[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]] && \
120	    log "$ICON_GEAR Maintenance scripts: $(for s in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do printf '%s ' "$(basename "${s%% *}")"; done)"
121
122	if ! check_rsync_enabled "CRITICAL"; then
123	    echo "Critical rsync disabled — skipping sync, running partnership check only"
124	elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
125	    warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID"
126	    warn "Check HOST*_CRITICAL_SYNC_SHARES in host*.conf"
127	else
128	    echo "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')"
129
130	    # Build dry-run flag to pass through
131	    RSYNC_DRY=""
132	    [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
133
134	    for share in "${CRITICAL_SYNC_SHARES[@]}"; do
135	        [[ -z "$share" ]] && continue
136
137	        # Parse optional profile flag: "/path/to/share|profile-name"
138	        SHARE_PATH="${share%%|*}"
139	        SHARE_PROFILE="${share##*|}"
140	        SHARE_NAME=$(basename "$SHARE_PATH")
141
142	        SHARE_START=$(date +%s)
143
144	        if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then
145	            bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" $RSYNC_DRY
146	        else
147	            bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" \
148	                --profile="$SHARE_PROFILE" $RSYNC_DRY
149	        fi
150
151	        RSYNC_EXIT=$?
152	        SHARE_DUR=$(format_duration $(( $(date +%s) - SHARE_START )))
153
154	        if [[ "$RSYNC_EXIT" -eq 0 ]]; then
155	            PASS+=("$SHARE_NAME")
156	            log "$SHARE_NAME — done in $SHARE_DUR ✅"
157	            RSYNC_OK=true
158	        else
159	            FAIL+=("$SHARE_NAME")
160	            error "$SHARE_NAME — failed after $SHARE_DUR (exit $RSYNC_EXIT)"
161	        fi
162	    done
163	fi
164
165	# ==============================================================================================
166	# ━━━ Critical Maintenance Scripts ━━━
167	# ==============================================================================================
168	if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
169	    for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
170	        [[ -z "$script_entry" || "$script_entry" == \#* ]] && continue
171
172	        SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
173	        SCRIPT_ARGS="${script_entry#* }"
174	        [[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
175	        [[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run"
176
177	        SCRIPT_NAME=$(basename "$SCRIPT_PATH")
178
179	        if [[ ! -f "$SCRIPT_PATH" ]]; then
180	            warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
181	            continue
182	        fi
183
184	        log "Running: $SCRIPT_NAME"
185	        bash "$SCRIPT_PATH" $SCRIPT_ARGS
186	        EXIT_CODE=$?
187	        [[ "$EXIT_CODE" -ne 0 ]] && \
188	            warn "$SCRIPT_NAME exited with code $EXIT_CODE"
189	    done
190	fi
191
192	# ==============================================================================================
193	# ━━━ Partnership Check ━━━
194	# ==============================================================================================
195	if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
196	    PARTNER_DRY=""
197	    [[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run"
198
199	    if [[ "$RSYNC_OK" == true ]]; then
200	        bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
201	            --check --remote-seen $PARTNER_DRY
202	    else
203	        bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
204	            --check --remote-unseen $PARTNER_DRY
205	    fi
206	else
207	    echo "Partnership disabled — skipping check"
208	fi
209
210	# ==============================================================================================
211	# ━━━ Summary ━━━
212	# ==============================================================================================
213	END=$(date +%s)
214	DURATION=$(format_duration $(( END - START )))
215
216	# Silent when healthy — only show summary if there were failures or notable events
217	if [[ ${#FAIL[@]} -gt 0 ]]; then
218	    echo ""
219	    echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
220	    echo "$ICON_HOST   Identity: $MY_ID ($LOCAL_SERVER_NAME)"
221	    echo "$ICON_TIME   Duration: $DURATION"
222	    [[ ${#PASS[@]} -gt 0 ]] && echo "Synced:  ${PASS[*]}"
223	    echo "$ICON_ERROR  Failed:  ${FAIL[*]}"
224	    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
225	    notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
226	        "Critical Sync" "warning"
227	    exit 1
228	else
229	    echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)"
230	fi
231
232	exit 0


1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= Daily Sync Maintenance =========================================
4	# ==============================================================================================
5	# Daily orchestrator — runs the full daily maintenance window in the correct order.
6	# Schedule: 0 1 * * * (1am daily via User Scripts plugin)
7	#
8	# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
9	#   Pre-sync:
10	#     git_pull_execute.sh              — pull latest scripts first, always
11	#
12	#   Arr Sync (arr_sync.sh):
13	#     Syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
14	#     All nodes agree on tracked library before any files are transferred.
15	#     Remote nodes that don't have an arr running are skipped gracefully.
16	#
17	#   Rsync window (DAILY_SYNC_SHARES per host):
18	#     HOST*_DAILY_SYNC_SHARES          — media shares spread to all nodes (no --delete)
19	#     HOST*_PERSONAL_SHARES            — encrypted personal shares
20	#
21	#   Post-sync maintenance (DAILY_MAINTENANCE_SCRIPTS):
22	#     media_shares_permissions.sh      — fix ownership before arr cleanup
23	#     media_cleaner.sh anime           — remove junk from anime shares
24	#     media_cleaner.sh media           — remove junk from media shares
25	#     lidarr_cleanup.sh                — remove orphaned music files (local arr = truth)
26	#     sonarr_cleanup.sh                — remove orphaned TV files (local arr = truth)
27	#     radarr_cleanup.sh                — remove orphaned movie files (local arr = truth)
28	#     docker_daily_restart.sh          — restart containers needing daily restart
29	#
30	# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
31	#   git pull first        — maintenance runs on latest code, not yesterday's
32	#   arr sync before rsync — all nodes track the same library before files are spread;
33	#                           prevents remote arrs from searching for content already owned
34	#   rsync before cleanup  — cleanup sees fully spread state, rsync has no --delete
35	#   permissions before arr cleanup — arrs need correct ownership to delete/rename
36	#   arr cleanup after permissions  — clean ownership = successful orphan deletion
37	#   docker restart last   — containers already processed by cleanup
38	#
39	# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
40	# Bidirectional — same script runs on both servers, correct direction automatic.
41	# detect_hosts() aliases DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars.
42	# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server.
43	#
44	# ── DRIVE TEMP HANDLING ───────────────────────────────────────────────────────────────────────
45	# rsync.sh returns exit codes for temperature issues:
46	#   exit 1 = temp WARN — skip this share, continue to next
47	#   exit 2 = temp CRITICAL — abort ALL remaining syncs in this window
48	#   All other failures — skip share, continue to next
49	#
50	# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
51	# Runs daily at 1am — clean run should produce minimal output.
52	# Each job reports log() on success (silent), warn()/error() on failure (visible).
53	# Summary always shown — gives window timing and share/job counts.
54	# Notify only on failure — successful daily maintenance doesn't need notification.
55	#
56	# ── CONFIGURATION (master.conf + host*.conf) ───────────────────────────────────────────
57	#   HOST*_DAILY_SYNC_SHARES      — shares pushed to mirror each day
58	#   HOST*_PERSONAL_SHARES        — encrypted personal shares
59	#   DAILY_MAINTENANCE_SCRIPTS    — maintenance jobs (permissions, cleanup, restart)
60	#   DAILY_RSYNC_ENABLED          — enable/disable rsync section
61	#
62	# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
63	#   daily_sync_maintenance.sh              — normal run
64	#   daily_sync_maintenance.sh --dry-run    — preview without syncing or changing
65	#   daily_sync_maintenance.sh --log        — verbose per-share/per-job output
66	#   daily_sync_maintenance.sh --status     — show configured shares and jobs
67	# ==============================================================================================
68
69	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
70
71	source "$SCRIPT_DIR/../load_config.sh"
72
73	RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
74	SCRIPTS_ROOT="$SCRIPT_DIR/.."
75
76	parse_args "$@"
77
78	# ==============================================================================================
79	# ━━━ Setup ━━━
80	# ==============================================================================================
81	if [[ "$EUID" -ne 0 ]]; then
82	    error "Must be run as root"
83	    exit 1
84	fi
85
86
87	if ! command -v docker &>/dev/null; then
88	    error "Docker command not found"
89	    exit 1
90	fi
91
92	detect_hosts
93	resolve_remote_ip
94
95	acquire_lock
96
97	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
98
99	# ==============================================================================================
100	# ━━━ Status ━━━
101	# ==============================================================================================
102	if [[ "$SHOW_STATUS" == true ]]; then
103	    echo ""
104	    echo "━━━━━ $ICON_SUMMARY DAILY SYNC STATUS ━━━━━"
105	    echo "$ICON_HOST   Identity:        $MY_ID ($LOCAL_SERVER_NAME)"
106	    echo "$ICON_NET    Remote:          $REMOTE_ID ($REMOTE_SERVER_NAME)"
107	    echo "$ICON_SYNC   Rsync enabled:   ${RSYNC_ENABLED:-true}"
108	    echo "$ICON_SYNC   Daily enabled:   ${DAILY_RSYNC_ENABLED:-false}"
109	    echo ""
110	    echo "━━━ Daily Sync Shares ━━━"
111	    for share in "${DAILY_SYNC_SHARES[@]}"; do
112	        [[ -n "$share" ]] && echo "  $ICON_SYNC $share"
113	    done
114	    for share in "${PERSONAL_SHARES[@]}"; do
115	        [[ -n "$share" ]] && echo "  $ICON_SYNC $share (personal)"
116	    done
117	    echo ""
118	    echo "━━━ Daily Maintenance Scripts ━━━"
119	    for entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
120	        [[ -z "$entry" ]] && continue
121	        echo "  $ICON_GEAR ${entry##*/}"
122	    done
123	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
124	    exit 0
125	fi
126
127	# ==============================================================================================
128	# ── BUILD SHARE LIST via detect_hosts aliases ─────────────────────────────────────────────────
129	# detect_hosts() sets DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars
130	# No manual HOST1/HOST2 comparison needed — aliased automatically per server
131	# ==============================================================================================
132	ALL_SHARES=()
133	for share in "${DAILY_SYNC_SHARES[@]}"; do
134	    [[ -n "$share" ]] && ALL_SHARES+=("$share")
135	done
136	for share in "${PERSONAL_SHARES[@]}"; do
137	    [[ -n "$share" ]] && ALL_SHARES+=("$share")
138	done
139	SHARE_COUNT=${#ALL_SHARES[@]}
140
141	# ── Split maintenance scripts: git pull runs pre-sync, rest run post-sync ─────────────────────
142	PRE_SYNC_SCRIPTS=()
143	POST_SYNC_SCRIPTS=()
144	for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
145	    [[ -z "$script_entry" ]] && continue
146	    script_name=$(basename "${script_entry%% *}")
147	    if [[ "$script_name" == "git_pull_execute.sh" ]]; then
148	        PRE_SYNC_SCRIPTS+=("$script_entry")
149	    else
150	        POST_SYNC_SCRIPTS+=("$script_entry")
151	    fi
152	done
153
154	# Helper — run a maintenance script, track pass/fail
155	run_job() {
156	    local script_entry="$1"
157	    local extra_dry=""
158	    [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
159
160	    read -r -a script_args <<< "$script_entry"
161	    local script_path="$SCRIPTS_ROOT/${script_args[0]}"
162	    local script_name
163	    script_name=$(basename "${script_args[0]}")
164	    local extra_args=("${script_args[@]:1}")
165
166	    if [[ ! -f "$script_path" ]]; then
167	        error "$script_name — not found at $script_path"
168	        JOB_FAIL+=("$script_name")
169	        return 1
170	    fi
171
172	    log "Running: $script_name ${extra_args[*]}"
173	    # shellcheck disable=SC2086
174	    if bash "$script_path" "${extra_args[@]}" $extra_dry; then
175	        log "$script_name — done ✅"
176	        JOB_PASS+=("$script_name ${extra_args[*]}")
177	    else
178	        error "$script_name — failed (exit $?)"
179	        JOB_FAIL+=("$script_name ${extra_args[*]}")
180	    fi
181	}
182
183	WINDOW_START=$(date +%s)
184	JOB_PASS=()
185	JOB_FAIL=()
186	PASS=()
187	FAIL=()
188	SHARE_TIMES=()
189
190	echo ""
191	echo "━━━ $ICON_GEAR Daily Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
192
193	# ==============================================================================================
194	# ━━━ Pre-sync — git pull ━━━
195	# ==============================================================================================
196	if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then
197	    echo ""
198	    echo "━━━ $ICON_GIT Pre-sync ━━━"
199	    for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
200	        run_job "$script_entry"
201	    done
202	fi
203
204	# ==============================================================================================
205	# ━━━ Arr Sync — library reconciliation before file spreading ━━━
206	# ==============================================================================================
207	echo ""
208	echo "━━━ $ICON_SYNC Arr Sync ━━━"
209
210	ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
211	if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
212	    echo "ARR_SYNC_ENABLED=false — skipping"
213	elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
214	    warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
215	    JOB_FAIL+=("arr_sync.sh")
216	else
217	    _arr_sync_args=()
218	    [[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
219	    if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
220	        echo "Arr sync complete ✅"
221	        JOB_PASS+=("arr_sync.sh")
222	    else
223	        warn "Arr sync completed with errors — continuing to rsync"
224	        JOB_FAIL+=("arr_sync.sh")
225	    fi
226	    unset _arr_sync_args
227	fi
228
229	# ==============================================================================================
230	# ━━━ Media Share Sync ━━━
231	# ==============================================================================================
232	echo ""
233	echo "━━━ $ICON_SYNC Media Share Sync — $SHARE_COUNT share(s) ━━━"
234
235	TOTAL_START=$(date +%s)
236	SHARE_INDEX=0
237	ABORT_ALL_SYNCS=false
238
239	if ! check_rsync_enabled "DAILY"; then
240	    warn "Daily rsync disabled — skipping all $SHARE_COUNT share syncs"
241	    warn "Proceeding to maintenance jobs..."
242	elif [[ "$SHARE_COUNT" -eq 0 ]]; then
243	    warn "No shares configured for $MY_ID — check HOST*_DAILY_SYNC_SHARES in host*.conf"
244	else
245	    # Pre-flight — connectivity then remote rootfs
246	    check_connectivity
247	    check_remote_rootfs
248
249	    RSYNC_DRY=""
250	    [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
251
252	    for SHARE in "${ALL_SHARES[@]}"; do
253	        (( SHARE_INDEX++ ))
254	        SHARE_NAME=$(basename "$SHARE")
255	        SHARE_START=$(date +%s)
256
257	        echo ""
258	        echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"
259
260	        if [[ "$ABORT_ALL_SYNCS" == true ]]; then
261	            warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
262	            FAIL+=("$SHARE_NAME:temp-critical")
263	            continue
264	        fi
265
266	        bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
267	        RSYNC_EXIT=$?
268
269	        SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")
270
271	        case "$RSYNC_EXIT" in
272	            0)
273	                PASS+=("$SHARE_NAME")
274	                log "$SHARE_NAME — done ✅"
275	                ;;
276	            1)
277	                FAIL+=("$SHARE_NAME:temp-warn")
278	                warn "$SHARE_NAME skipped — drive temps too high"
279	                ;;
280	            2)
281	                FAIL+=("$SHARE_NAME:temp-critical")
282	                ABORT_ALL_SYNCS=true
283	                error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
284	                notify "Daily sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
285	                    "Daily Sync" "warning"
286	                ;;
287	            *)
288	                FAIL+=("$SHARE_NAME")
289	                error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
290	                ;;
291	        esac
292
293	    done
294	fi
295
296	TOTAL_END=$(date +%s)
297
298	# ==============================================================================================
299	# ━━━ Post-sync Maintenance Jobs ━━━
300	# ==============================================================================================
301	if [[ ${#POST_SYNC_SCRIPTS[@]} -gt 0 ]]; then
302	    echo ""
303	    echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━"
304	    for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
305	        echo ""
306	        run_job "$script_entry"
307	    done
308	fi
309
310	WINDOW_END=$(date +%s)
311
312	# ==============================================================================================
313	# ━━━ Summary ━━━
314	# ==============================================================================================
315	echo ""
316	echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━"
317	echo "$ICON_HOST   Identity: $MY_ID ($LOCAL_SERVER_NAME)"
318	echo "$ICON_TIME   Window:   $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
319	echo "$ICON_TIME   Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
320	echo ""
321
322	echo "$ICON_SYNC  Shares ($SHARE_COUNT):"
323	for entry in "${SHARE_TIMES[@]}"; do
324	    sname="${entry%%:*}"
325	    sdur="${entry##*:}"
326	    if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
327	        echo "  $ICON_ERROR $sname — $(format_duration "$sdur")"
328	    else
329	        echo "  $ICON_DONE  $sname — $(format_duration "$sdur")"
330	    fi
331	done
332	[[ "$SHARE_COUNT" -eq 0 || "${DAILY_RSYNC_ENABLED:-false}" == "false" ]] && \
333	    echo "  (rsync disabled)"
334	echo "  Passed: ${#PASS[@]}/$SHARE_COUNT  Failed: ${#FAIL[@]}/$SHARE_COUNT"
335	echo ""
336
337	if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
338	    echo "$ICON_GEAR  Jobs:"
339	    for job in "${JOB_PASS[@]}"; do echo "  $ICON_DONE  $job"; done
340	    for job in "${JOB_FAIL[@]}"; do echo "  $ICON_ERROR $job"; done
341	    echo ""
342	fi
343
344	TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
345
346	if [[ "$TOTAL_FAIL" -gt 0 ]]; then
347	    warn "Status: $TOTAL_FAIL failure(s)"
348	    notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
349	        "Daily Maintenance" "warning"
350	    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
351	    exit 1
352	else
353	    echo "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
354	    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
355	    exit 0
356	fi


1	#!/bin/bash
2	# ==============================================================================================
3	# =========================== Intermediate Sync Maintenance ====================================
4	# ==============================================================================================
5	# 4-hour orchestrator — arr library reconciliation, artwork fetching, and optional rsync.
6	# Schedule: 0 */4 * * * (every 4 hours)
7	#
8	# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
9	#   1. conf_sync.sh --pull-only    — refresh partner conf cache in RAM (/tmp/.vv/)
10	#   2. arr_sync.sh                 — sync Lidarr/Sonarr/Radarr libraries across all nodes
11	#   3. Rsync window (optional)     — INTERMEDIATE_SYNC_SHARES, if any configured
12	#   4. INTERMEDIATE_MAINTENANCE_SCRIPTS — artwork fetch and any future 4-hour jobs
13	#
14	# ── WHY A SEPARATE ORCHESTRATOR ───────────────────────────────────────────────────────────────
15	# arr libraries need to converge more frequently than once a day. If a remote node adds
16	# something at 2am, the next daily window is 23 hours away — remote arrs search for content
17	# they don't know is already owned. Running every 4 hours closes that gap.
18	#
19	# lidarr_missing_art.sh is idempotent — skips existing files, runs fast after initial fill.
20	# Pairing it here means artwork catches up within 4 hours of a new album landing.
21	#
22	# Rsync is optional — INTERMEDIATE_SYNC_SHARES empty by default. Add shares to the config
23	# if a subset of data needs mid-day propagation (e.g. watch state, metadata). Full media
24	# share sync stays in the daily window.
25	#
26	# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
27	# detect_hosts() sets MY_ID and aliases HOST*_INTERMEDIATE_SYNC_SHARES → INTERMEDIATE_SYNC_SHARES.
28	# Each server can have a different set of mid-day shares — configure in host*.conf.
29	# Each script in INTERMEDIATE_MAINTENANCE_SCRIPTS handles its own host logic.
30	#
31	# ── DRIVE TEMP HANDLING ───────────────────────────────────────────────────────────────────────
32	# Same as daily_sync_maintenance.sh:
33	#   exit 1 = temp WARN — skip this share, continue to next
34	#   exit 2 = temp CRITICAL — abort ALL remaining syncs in this window
35	#
36	# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
37	#   Root check           — scripts called here require root
38	#   acquire_lock         — prevents concurrent intermediate windows
39	#   check_connectivity   — verified before any rsync (skipped if no shares)
40	#   check_remote_rootfs  — aborts rsync if remote rootfs nearly full
41	#   Non-fatal jobs       — a failed arr_sync warns but does not block rsync or artwork fetch
42	#   Silent on success    — runs 4x/day, only failures warrant notification
43	#
44	# ── CONFIGURATION ─────────────────────────────────────────────────────────────────────────────
45	#   host*.conf:   HOST*_INTERMEDIATE_SYNC_SHARES — shares synced mid-day (empty = rsync skipped)
46	#   master.conf:         INTERMEDIATE_RSYNC_ENABLED     — enable/disable rsync section (default: true)
47	#   master.conf:         INTERMEDIATE_MAINTENANCE_SCRIPTS — jobs run after rsync
48	#   master.conf:         ARR_SYNC_ENABLED               — toggle inside arr_sync.sh
49	#
50	# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
51	#   intermediate_sync_maintenance.sh              — normal run
52	#   intermediate_sync_maintenance.sh --dry-run    — preview without changes
53	#   intermediate_sync_maintenance.sh --log        — verbose per-job output
54	#   intermediate_sync_maintenance.sh --status     — show configured shares/jobs and exit
55	# ==============================================================================================
56
57	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
58
59	source "$SCRIPT_DIR/../load_config.sh"
60
61	RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
62	SCRIPTS_ROOT="$SCRIPT_DIR/.."
63
64	parse_args "$@"
65
66	# ==============================================================================================
67	# ━━━ Setup ━━━
68	# ==============================================================================================
69	if [[ "$EUID" -ne 0 ]]; then
70	    error "Must be run as root"
71	    exit 1
72	fi
73
74
75	acquire_lock
76
77	detect_hosts
78	resolve_remote_ip
79
80	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
81
82	# ── Helper — run a maintenance job, track pass/fail ───────────────────────────────────────────
83	run_job() {
84	    local script_entry="$1"
85	    local extra_dry=""
86	    [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
87
88	    read -r -a script_args <<< "$script_entry"
89	    local script_path="$SCRIPTS_ROOT/${script_args[0]}"
90	    local script_name
91	    script_name=$(basename "${script_args[0]}")
92	    local extra_args=("${script_args[@]:1}")
93
94	    if [[ ! -f "$script_path" ]]; then
95	        error "$script_name — not found at $script_path"
96	        JOB_FAIL+=("$script_name")
97	        return 1
98	    fi
99
100	    log "Running: $script_name ${extra_args[*]}"
101	    # shellcheck disable=SC2086
102	    if bash "$script_path" "${extra_args[@]}" $extra_dry; then
103	        log "$script_name — done ✅"
104	        JOB_PASS+=("$script_name ${extra_args[*]}")
105	    else
106	        warn "$script_name — failed (exit $?)"
107	        JOB_FAIL+=("$script_name ${extra_args[*]}")
108	    fi
109	}
110
111	# ==============================================================================================
112	# ━━━ Status ━━━
113	# ==============================================================================================
114	if [[ "$SHOW_STATUS" == true ]]; then
115	    echo ""
116	    echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC STATUS ━━━━━"
117	    echo "$ICON_HOST   Identity:        $MY_ID ($LOCAL_SERVER_NAME)"
118	    echo "$ICON_NET    Remote:          $REMOTE_ID ($REMOTE_SERVER_NAME)"
119	    echo "$ICON_SYNC   Rsync enabled:   ${RSYNC_ENABLED:-true}"
120	    echo "$ICON_SYNC   Interm. enabled: ${INTERMEDIATE_RSYNC_ENABLED:-true}"
121	    echo "$ICON_GEAR   Arr sync:        ${ARR_SYNC_ENABLED:-true}"
122	    echo ""
123	    echo "━━━ Intermediate Sync Shares ━━━"
124	    if [[ ${#INTERMEDIATE_SYNC_SHARES[@]} -eq 0 ]]; then
125	        echo "  None configured — add to INTERMEDIATE_SYNC_SHARES in master.conf to enable"
126	    else
127	        for share in "${INTERMEDIATE_SYNC_SHARES[@]}"; do
128	            [[ -n "$share" ]] && echo "  $ICON_SYNC $(basename "$share") ($share)"
129	        done
130	    fi
131	    echo ""
132	    echo "━━━ Intermediate Maintenance Scripts ━━━"
133	    if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
134	        echo "  None configured"
135	    else
136	        for entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
137	            [[ -n "$entry" ]] && echo "  $ICON_GEAR ${entry##*/}"
138	        done
139	    fi
140	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
141	    exit 0
142	fi
143
144	WINDOW_START=$(date +%s)
145	JOB_PASS=()
146	JOB_FAIL=()
147	PASS=()
148	FAIL=()
149	SHARE_TIMES=()
150
151	echo ""
152	echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
153
154	# ==============================================================================================
155	# ━━━ Conf Pull ━━━
156	# ==============================================================================================
157	echo ""
158	echo "━━━ $ICON_GEAR Conf Pull ━━━"
159
160	CONF_SYNC_SCRIPT="$SCRIPTS_ROOT/System_Essentials/conf_sync.sh"
161	if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
162	    warn "conf_sync.sh not found — skipping partner conf refresh"
163	else
164	    _conf_args=("--pull-only")
165	    [[ "$DRY_RUN" == true ]] && _conf_args+=("--dry-run")
166	    if bash "$CONF_SYNC_SCRIPT" "${_conf_args[@]}"; then
167	        log "Partner conf cache refreshed ✅"
168	        JOB_PASS+=("conf_sync.sh --pull-only")
169	    else
170	        warn "Partner conf pull failed — cache may be stale"
171	        JOB_FAIL+=("conf_sync.sh --pull-only")
172	    fi
173	    unset _conf_args
174	fi
175
176	# ==============================================================================================
177	# ━━━ Arr Sync ━━━
178	# ==============================================================================================
179	echo ""
180	echo "━━━ $ICON_SYNC Arr Sync ━━━"
181
182	ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Media/arr_sync.sh"
183	if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
184	    echo "ARR_SYNC_ENABLED=false — skipping"
185	elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
186	    warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
187	    JOB_FAIL+=("arr_sync.sh")
188	else
189	    _arr_sync_args=()
190	    [[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
191	    if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
192	        echo "Arr sync complete ✅"
193	        JOB_PASS+=("arr_sync.sh")
194	    else
195	        warn "Arr sync completed with errors — continuing"
196	        JOB_FAIL+=("arr_sync.sh")
197	    fi
198	    unset _arr_sync_args
199	fi
200
201	# ==============================================================================================
202	# ━━━ Rsync (optional) ━━━
203	# ==============================================================================================
204	SHARE_COUNT=${#INTERMEDIATE_SYNC_SHARES[@]}
205
206	echo ""
207	echo "━━━ $ICON_SYNC Mid-day Share Sync — $SHARE_COUNT share(s) ━━━"
208
209	TOTAL_START=$(date +%s)
210	SHARE_INDEX=0
211	ABORT_ALL_SYNCS=false
212
213	if [[ "$SHARE_COUNT" -eq 0 ]]; then
214	    echo "No INTERMEDIATE_SYNC_SHARES configured — skipping"
215	    echo "Add shares to INTERMEDIATE_SYNC_SHARES in master.conf to enable mid-day sync"
216	elif ! check_rsync_enabled "INTERMEDIATE"; then
217	    warn "Intermediate rsync disabled — skipping all $SHARE_COUNT share sync(s)"
218	else
219	    check_connectivity
220	    check_remote_rootfs
221
222	    RSYNC_DRY=""
223	    [[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
224
225	    for SHARE in "${INTERMEDIATE_SYNC_SHARES[@]}"; do
226	        (( SHARE_INDEX++ ))
227	        SHARE_NAME=$(basename "$SHARE")
228	        SHARE_START=$(date +%s)
229
230	        echo ""
231	        echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"
232
233	        if [[ "$ABORT_ALL_SYNCS" == true ]]; then
234	            warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
235	            FAIL+=("$SHARE_NAME:temp-critical")
236	            continue
237	        fi
238
239	        bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
240	        RSYNC_EXIT=$?
241
242	        SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")
243
244	        case "$RSYNC_EXIT" in
245	            0)
246	                PASS+=("$SHARE_NAME")
247	                log "$SHARE_NAME — done ✅"
248	                ;;
249	            1)
250	                FAIL+=("$SHARE_NAME:temp-warn")
251	                warn "$SHARE_NAME skipped — drive temps too high"
252	                ;;
253	            2)
254	                FAIL+=("$SHARE_NAME:temp-critical")
255	                ABORT_ALL_SYNCS=true
256	                error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
257	                notify "Intermediate sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
258	                    "Intermediate Sync" "warning"
259	                ;;
260	            *)
261	                FAIL+=("$SHARE_NAME")
262	                error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
263	                ;;
264	        esac
265	    done
266	fi
267
268	TOTAL_END=$(date +%s)
269
270	# ==============================================================================================
271	# ━━━ Maintenance Jobs ━━━
272	# ==============================================================================================
273	if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
274	    echo ""
275	    echo "━━━ $ICON_GEAR Maintenance Jobs ━━━"
276	    for script_entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
277	        [[ -z "$script_entry" ]] && continue
278	        echo ""
279	        run_job "$script_entry"
280	    done
281	fi
282
283	WINDOW_END=$(date +%s)
284
285	# ==============================================================================================
286	# ━━━ Summary ━━━
287	# ==============================================================================================
288	echo ""
289	echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━"
290	echo "$ICON_HOST   Identity: $MY_ID ($LOCAL_SERVER_NAME)"
291	echo "$ICON_TIME   Window:   $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
292	echo "$ICON_TIME   Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
293	echo ""
294
295	if [[ "$SHARE_COUNT" -gt 0 ]]; then
296	    echo "$ICON_SYNC  Shares ($SHARE_COUNT):"
297	    for entry in "${SHARE_TIMES[@]}"; do
298	        sname="${entry%%:*}"
299	        sdur="${entry##*:}"
300	        if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
301	            echo "  $ICON_ERROR $sname — $(format_duration "$sdur")"
302	        else
303	            echo "  $ICON_DONE  $sname — $(format_duration "$sdur")"
304	        fi
305	    done
306	    echo "  Passed: ${#PASS[@]}/$SHARE_COUNT  Failed: ${#FAIL[@]}/$SHARE_COUNT"
307	    echo ""
308	fi
309
310	if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
311	    echo "$ICON_GEAR  Jobs:"
312	    for job in "${JOB_PASS[@]}"; do echo "  $ICON_DONE  $job"; done
313	    for job in "${JOB_FAIL[@]}"; do echo "  $ICON_ERROR $job"; done
314	    echo ""
315	fi
316
317	TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
318
319	if [[ "$DRY_RUN" == true ]]; then
320	    warn "DRY RUN — no changes made"
321	elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
322	    echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
323	else
324	    warn "Status: $TOTAL_FAIL failure(s)"
325	    notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
326	        "Intermediate Sync" "warning"
327	fi
328	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
329
330	[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
331	exit 0
332