1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Ramdisk Stop ===============================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Safely stops the transcode ramdisk: redirects the transcode symlink to the
9	# SSD fallback before unmounting so Emby continues writing without interruption,
10	# then unmounts the tmpfs and updates the state file.
11	#
12	# Primary use case: stopping the current ramdisk before re-running
13	# ramdisk_setup.sh with new size or threshold values (setup is idempotent —
14	# if the ramdisk is mounted, it skips the mount and reports status, so you
15	# must stop it first to change the size).
16	#
17	# ==============================================================================================
18	# OPERATIONAL MODEL
19	# ==============================================================================================
20	#
21	# Executes in safe order:
22	#   1. Validate — ramdisk mounted, SSD fallback exists
23	#   2. Redirect symlink → SSD (Emby immediately writes to SSD instead)
24	#   3. Warn if active transcode files still on ramdisk (informational — not a blocker)
25	#   4. Unmount ramdisk tmpfs
26	#   5. Update /tmp/transcode_state.db → current_target=TRANSCODE_SSD
27	#
28	# The symlink redirect happens before unmount so there is no window where Emby
29	# has nowhere to write. Existing in-progress transcode files on the ramdisk are
30	# lost on unmount — warn the user but proceed (this is expected for maintenance).
31	#
32	# ==============================================================================================
33	# OPERATIONAL SAFEGUARDS
34	# ==============================================================================================
35	#
36	# Root Required
37	#   umount requires root.
38	#
39	# Single Instance Lock
40	#   acquire_lock prevents concurrent stop attempts.
41	#
42	# Mounted Check
43	#   Exits cleanly if ramdisk is not mounted — nothing to do.
44	#
45	# Symlink-First Order
46	#   Symlink is redirected before unmount — Emby never sees a broken path.
47	#
48	# transcode_manager Warning
49	#   Warns if transcode_manager.sh is running — it may flip the symlink back
50	#   to ramdisk on its next cycle. Stop transcode_manager before running this
51	#   if you need the SSD redirect to hold.
52	#
53	# ==============================================================================================
54	# CONFIGURATION
55	# ==============================================================================================
56	#
57	# host*.conf
58	#   HOST*_TRANSCODE_SSD     SSD fallback directory — redirect target during stop.
59	#                           Aliased by detect_hosts() → TRANSCODE_SSD.
60	#
61	# master.conf
62	#   TRANSCODE_LINK          Symlink Emby uses. Must match Emby's transcode path setting.
63	#   RAMDISK_PATH            tmpfs mount point.
64	#
65	# ==============================================================================================
66	# STATE FILES
67	# ==============================================================================================
68	#
69	#   /tmp/transcode_state.db    — updated to current_target=TRANSCODE_SSD after stop.
70	#                                transcode_manager.sh reads this on its next cycle.
71	#
72	# ==============================================================================================
73	# RUNTIME MODES
74	# ==============================================================================================
75	#
76	# ramdisk_stop.sh
77	#     Stop the ramdisk: redirect symlink → SSD, unmount, update state.
78	#
79	# ramdisk_stop.sh --dry-run
80	#     Show what would happen without making any changes.
81	#
82	# ramdisk_stop.sh --status
83	#     Show current mount state, symlink target, active files on ramdisk. Exit.
84	#
85	# ramdisk_stop.sh --log
86	#     Verbose output for each step.
87	#
88	# ==============================================================================================
89
90	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
91
92	source "$SCRIPT_DIR/../load_config.sh"
93
94	parse_args "$@"
95
96	# ==============================================================================================
97	# ━━━ Setup ━━━
98	# ==============================================================================================
99	if [[ "$EUID" -ne 0 ]]; then
100	    error "Must be run as root — umount requires root"
101	    exit 1
102	fi
103
104
105	acquire_lock
106
107	detect_hosts
108
109	STATE_FILE="${TRANSCODE_STATE_FILE:-${STATE_DIR:-/tmp}/transcode_state.db}"
110
111	log "Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
112	log "Ramdisk:   $RAMDISK_PATH"
113	log "Fallback:  $TRANSCODE_SSD"
114	log "Symlink:   $TRANSCODE_LINK"
115
116	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
117
118	# ==============================================================================================
119	# ━━━ Status ━━━
120	# ==============================================================================================
121	if [[ "$SHOW_STATUS" == true ]]; then
122	    echo ""
123	    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
124	    echo "$ICON_HOST   Identity:     $MY_ID ($LOCAL_SERVER_NAME)"
125	    echo "$ICON_RAM    Ramdisk path: $RAMDISK_PATH"
126	    echo "$ICON_DISK   SSD fallback: $TRANSCODE_SSD"
127	    echo "$ICON_LINK   Symlink:      $TRANSCODE_LINK"
128	    echo ""
129
130	    if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
131	        USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
132	        AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
133	        echo "  $ICON_RAM  Ramdisk: mounted — $USAGE used / $AVAIL available ✅"
134	        FILE_COUNT=$(find "$RAMDISK_PATH" -type f 2>/dev/null | wc -l)
135	        echo "  $ICON_RAM  Active files on ramdisk: $FILE_COUNT"
136	    else
137	        echo "  $ICON_RAM  Ramdisk: NOT mounted"
138	    fi
139
140	    if [[ -L "$TRANSCODE_LINK" ]]; then
141	        TARGET=$(readlink "$TRANSCODE_LINK")
142	        echo "  $ICON_LINK Symlink: $TRANSCODE_LINK → $TARGET"
143	    else
144	        echo "  $ICON_LINK Symlink: not set"
145	    fi
146
147	    if [[ -f "$STATE_FILE" ]]; then
148	        echo ""
149	        echo "  State file ($STATE_FILE):"
150	        while IFS='=' read -r key value; do
151	            [[ -z "$key" ]] && continue
152	            echo "    $key = $value"
153	        done < "$STATE_FILE"
154	    else
155	        echo "  $ICON_INFO State file: not found (ramdisk never started this boot)"
156	    fi
157
158	    if pgrep -f "transcode_manager.sh" >/dev/null 2>&1; then
159	        echo ""
160	        warn "transcode_manager.sh is currently RUNNING"
161	    fi
162
163	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
164	    exit 0
165	fi
166
167	# ==============================================================================================
168	# ━━━ Preflight ━━━
169	# ==============================================================================================
170	echo ""
171	echo "━━━ $ICON_GEAR Preflight ━━━"
172
173	# Bail if not mounted — nothing to do
174	if ! mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
175	    warn "Ramdisk is not mounted at $RAMDISK_PATH — nothing to stop"
176	    exit 0
177	fi
178
179	log "Ramdisk is mounted ✅"
180
181	# Warn if transcode_manager is running — it may flip symlink back on next cycle
182	if pgrep -f "transcode_manager.sh" >/dev/null 2>&1; then
183	    warn "transcode_manager.sh is currently RUNNING"
184	    warn "It may flip the symlink back to ramdisk on its next cycle"
185	    warn "Stop transcode_manager.sh first if you need the SSD redirect to hold"
186	    echo ""
187	fi
188
189	# Confirm SSD fallback exists
190	if [[ ! -d "$TRANSCODE_SSD" ]]; then
191	    if [[ "$DRY_RUN" == true ]]; then
192	        warn "DRY RUN — SSD fallback does not exist: $TRANSCODE_SSD"
193	        warn "DRY RUN — would create it before redirecting symlink"
194	    else
195	        warn "SSD fallback does not exist — creating: $TRANSCODE_SSD"
196	        mkdir -p "$TRANSCODE_SSD" || {
197	            error "Failed to create SSD fallback: $TRANSCODE_SSD"
198	            error "Cannot safely redirect symlink — aborting"
199	            exit 1
200	        }
201	        log "SSD fallback created ✅"
202	    fi
203	else
204	    log "SSD fallback exists: $TRANSCODE_SSD ✅"
205	fi
206
207	START=$(date +%s)
208	STOP_SUCCESS=true
209
210	# ==============================================================================================
211	# ━━━ Redirect Symlink → SSD ━━━
212	# ==============================================================================================
213	# Redirect BEFORE unmount — Emby continues writing to SSD with no broken path window.
214	echo ""
215	echo "━━━ $ICON_LINK Redirect Symlink → SSD ━━━"
216
217	if [[ -L "$TRANSCODE_LINK" ]]; then
218	    CURRENT_TARGET=$(readlink "$TRANSCODE_LINK")
219	    if [[ "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
220	        echo "Symlink already points to SSD — no change needed ✅"
221	    else
222	        if [[ "$DRY_RUN" == true ]]; then
223	            warn "DRY RUN — would redirect: $TRANSCODE_LINK → $TRANSCODE_SSD"
224	        else
225	            ln -sfn "$TRANSCODE_SSD" "$TRANSCODE_LINK" && \
226	                warn "Symlink redirected: $TRANSCODE_LINK → $TRANSCODE_SSD ✅" || {
227	                error "Failed to redirect symlink"
228	                STOP_SUCCESS=false
229	            }
230	        fi
231	    fi
232	elif [[ -e "$TRANSCODE_LINK" ]]; then
233	    warn "$TRANSCODE_LINK exists but is not a symlink — leaving as-is"
234	else
235	    warn "Symlink $TRANSCODE_LINK does not exist — nothing to redirect"
236	fi
237
238	# ==============================================================================================
239	# ━━━ Active Files Warning ━━━
240	# ==============================================================================================
241	echo ""
242	echo "━━━ $ICON_RAM Active Files Check ━━━"
243
244	FILE_COUNT=$(find "$RAMDISK_PATH" -type f 2>/dev/null | wc -l)
245	if [[ "$FILE_COUNT" -gt 0 ]]; then
246	    warn "⚠  $FILE_COUNT file(s) still on ramdisk — will be lost on unmount"
247	    warn "Active transcode sessions should be stopped before unmounting"
248	    warn "Proceeding regardless (this is expected for maintenance)"
249	    if [[ "$LOG" == true ]]; then
250	        find "$RAMDISK_PATH" -type f 2>/dev/null | while read -r f; do
251	            log "  $f"
252	        done
253	    fi
254	else
255	    log "No active files on ramdisk ✅"
256	fi
257
258	# ==============================================================================================
259	# ━━━ Unmount ━━━
260	# ==============================================================================================
261	echo ""
262	echo "━━━ $ICON_RAM Unmount Ramdisk ━━━"
263
264	if [[ "$DRY_RUN" == true ]]; then
265	    warn "DRY RUN — would unmount: $RAMDISK_PATH"
266	else
267	    if umount "$RAMDISK_PATH" 2>/dev/null; then
268	        warn "Ramdisk unmounted: $RAMDISK_PATH ✅"
269	    else
270	        # Regular unmount failed — check if only directory handles are open (no active writes)
271	        OPEN_FILES=$(lsof +D "$RAMDISK_PATH" 2>/dev/null | awk 'NR>1 && $5 != "DIR"' | wc -l)
272	        if [[ "$OPEN_FILES" -eq 0 ]]; then
273	            warn "Busy — only directory handles open, no active writes — trying lazy unmount"
274	            if umount -l "$RAMDISK_PATH"; then
275	                warn "Ramdisk lazy-unmounted: $RAMDISK_PATH ✅"
276	                warn "Handles will release when owning processes next check the directory"
277	            else
278	                error "Lazy unmount also failed — $RAMDISK_PATH"
279	                STOP_SUCCESS=false
280	            fi
281	        else
282	            error "Failed to unmount $RAMDISK_PATH — $OPEN_FILES file(s) still open for writing"
283	            error "Stop active transcode sessions and retry"
284	            STOP_SUCCESS=false
285	        fi
286	    fi
287	fi
288
289	# ==============================================================================================
290	# ━━━ Update State File ━━━
291	# ==============================================================================================
292	echo ""
293	echo "━━━ $ICON_GEAR State File ━━━"
294
295	if [[ "$DRY_RUN" == true ]]; then
296	    warn "DRY RUN — would update $STATE_FILE: current_target=$TRANSCODE_SSD"
297	elif [[ "$STOP_SUCCESS" == true ]]; then
298	    NOW=$(date +%s)
299	    cat > "$STATE_FILE" <<EOF
300	current_target=$TRANSCODE_SSD
301	last_flip_time=$NOW
302	flip_count_hour=0
303	flip_hour_start=$NOW
304	EOF
305	    log "State file updated: current_target=$TRANSCODE_SSD"
306	else
307	    warn "Skipping state file update — stop had errors"
308	fi
309
310	END=$(date +%s)
311
312	# ==============================================================================================
313	# ━━━ Summary ━━━
314	# ==============================================================================================
315	echo ""
316	echo "━━━━━ $ICON_SUMMARY RAMDISK STOP SUMMARY ━━━━━"
317	echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
318	echo "$ICON_RAM   Ramdisk:  $RAMDISK_PATH"
319	echo "$ICON_DISK  Fallback: $TRANSCODE_SSD"
320	echo "$ICON_LINK  Symlink:  $TRANSCODE_LINK → $(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "not set")"
321	echo "$ICON_TIME  Duration: $(format_duration $(( END - START )))"
322	echo ""
323
324	if [[ "$DRY_RUN" == true ]]; then
325	    warn "DRY RUN — no changes made"
326	elif [[ "$STOP_SUCCESS" == true ]]; then
327	    echo "$ICON_DONE Status: done ✅"
328	    echo "Run ramdisk_setup.sh to remount with new configuration"
329	else
330	    echo "$ICON_ERROR Status: STOP HAD ERRORS"
331	    notify "Ramdisk stop errors on $(hostname) ($MY_ID) — check output" \
332	        "Ramdisk Stop" "warning"
333	    exit 1
334	fi
335	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
336


Let me continue reading the remaining scripts.


1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= SMART Extended Self-Test =======================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Runs a SMART extended (long) self-test on all drives (or a specific drive),
9	# waits for completion, and reports results. Extended tests do a full read-scan
10	# of every sector — catches bad sectors that the short test skips. Monthly cadence
11	# via the monthly maintenance orchestrator.
12	#
13	# NVMe drives are included — smartctl supports NVMe self-test via the same
14	# -t long interface (NVMe 1.3+ specification).
15	#
16	# ==============================================================================================
17	# OPERATIONAL MODEL
18	# ==============================================================================================
19	#
20	# Starts long self-tests on all drives sequentially (NOT in parallel) to avoid
21	# saturating I/O. One drive at a time: start, wait for completion, move to next.
22	# Sequential order keeps test duration predictable and avoids thermal stacking.
23	#
24	# Drives already running a self-test are skipped. If a previous test did not
25	# finish (interrupted mid-way), it is reported and the drive is re-tested.
26	#
27	# Each test is polled every 60 seconds. Typical durations:
28	#   HDD  2–4 TB : 60–120 min
29	#   HDD  8–12 TB: 90–180 min
30	#   SSD  any    :  5–15 min
31	#   NVMe any    :  5–10 min
32	#
33	# ==============================================================================================
34	# OPERATIONAL SAFEGUARDS
35	# ==============================================================================================
36	#
37	# Single Instance Lock
38	#   acquire_lock prevents concurrent test runs — one long-test pass per server.
39	#
40	# Sequential Execution
41	#   Drives tested one at a time. Parallel long-tests thrash I/O and inflate temps.
42	#
43	# SIGTERM Trap
44	#   Poll loop exits cleanly on signal. Tests continue running in drive firmware.
45	#
46	# Tool Validation
47	#   platform_require_cmd confirms smartctl is present before use.
48	#
49	# Silent When Clean
50	#   Only failures and warnings produce notifications.
51	#
52	# ==============================================================================================
53	# CONFIGURATION
54	# ==============================================================================================
55	#
56	# host*.conf
57	#
58	#   HOST*_SMART_IGNORE_DRIVES
59	#       Drives skipped in SMART testing. Aliased by detect_hosts() →
60	#       SMART_IGNORE_DRIVES. Typically includes the boot USB flash drive.
61	#
62	# master.conf
63	#
64	#   SMART_TEMP_WARN / SMART_TEMP_CRIT
65	#       Fallback thresholds used for post-test temperature reporting if
66	#       dynamix.cfg is not found.
67	#
68	# ==============================================================================================
69	# RUNTIME MODES
70	# ==============================================================================================
71	#
72	# smart_long_test.sh
73	#     Run extended self-test on all drives not in SMART_IGNORE_DRIVES.
74	#
75	# smart_long_test.sh /dev/sda
76	#     Run extended self-test on a specific drive. Bypasses the ignore list.
77	#
78	# smart_long_test.sh --status
79	#     Show last self-test result for all drives and exit.
80	#
81	# smart_long_test.sh --dry-run
82	#     Show which drives would be tested. No tests started.
83	#
84	# smart_long_test.sh --log
85	#     Verbose progress output during polling.
86	#
87	# ==============================================================================================
88
89	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
90
91	source "$SCRIPT_DIR/../load_config.sh"
92
93	parse_args "$@"
94
95	TARGET_DRIVE="${PARSED_ARGS[0]:-}"
96
97	# ==============================================================================================
98	# ━━━ Setup ━━━
99	# ==============================================================================================
100	if [[ "$EUID" -ne 0 ]]; then
101	    error "Must be run as root"
102	    exit 1
103	fi
104
105	platform_require_cmd \
106	    "$(command -v smartctl 2>/dev/null || echo /usr/bin/smartctl)" \
107	    "--version" "smartmontools" \
108	    "smartctl" || {
109	        error "smartctl not found — install smartmontools"
110	        notify "SMART long test failed on $(hostname) — smartmontools not installed" \
111	            "SMART Long Test" "warning"
112	        exit 1
113	    }
114
115
116	acquire_lock
117
118	detect_hosts
119	get_unraid_temp_thresholds
120
121	log "Identity:     $MY_ID ($LOCAL_SERVER_NAME)"
122	log "Ignore:       ${SMART_IGNORE_DRIVES[*]:-none}"
123
124	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no tests will be started"
125
126	trap 'warn "SMART long test script interrupted — tests continue in drive firmware"; exit 0' \
127	    SIGTERM SIGINT
128
129	# ==============================================================================================
130	# ── Build drive list ──────────────────────────────────────────────────────────────────────────
131	# ==============================================================================================
132	_build_drive_list() {
133	    local drives=()
134	    if [[ -n "$TARGET_DRIVE" ]]; then
135	        if [[ ! -e "$TARGET_DRIVE" ]]; then
136	            error "Drive not found: $TARGET_DRIVE"
137	            exit 1
138	        fi
139	        drives=("$TARGET_DRIVE")
140	    else
141	        for drive in /dev/sd? /dev/nvme?; do
142	            [[ ! -e "$drive" ]] && continue
143	            local drive_name
144	            drive_name=$(basename "$drive")
145	            local ignored=false
146	            for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
147	                [[ "$drive_name" == "$ignore" ]] && ignored=true && break
148	            done
149	            [[ "$ignored" == true ]] && { log "Skipping $drive_name (SMART_IGNORE_DRIVES)"; continue; }
150	            if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
151	                log "Skipping $drive_name — SMART not enabled"
152	                continue
153	            fi
154	            drives+=("$drive")
155	        done
156	    fi
157	    echo "${drives[@]}"
158	}
159
160	# ==============================================================================================
161	# ━━━ Status ━━━
162	# ==============================================================================================
163	if [[ "$SHOW_STATUS" == true ]]; then
164	    echo ""
165	    echo "━━━━━ $ICON_SUMMARY SMART LONG TEST STATUS ━━━━━"
166	    echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
167	    echo ""
168	    for drive in /dev/sd? /dev/nvme?; do
169	        [[ ! -e "$drive" ]] && continue
170	        drive_name=$(basename "$drive")
171	        local_result=$(smartctl -l selftest "$drive" 2>/dev/null | \
172	            grep -m1 "Extended" | awk '{print $NF}')
173	        ignored=false
174	        for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
175	            [[ "$drive_name" == "$ignore" ]] && ignored=true && break
176	        done
177	        if [[ "$ignored" == true ]]; then
178	            echo "  $ICON_WARN $drive_name — ignored"
179	        else
180	            echo "  $ICON_SMART $drive_name — last extended: ${local_result:-no result}"
181	        fi
182	    done
183	    echo ""
184	    echo "  Ignored: ${SMART_IGNORE_DRIVES[*]:-none}"
185	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
186	    exit 0
187	fi
188
189	# ==============================================================================================
190	# ━━━ Run Extended Tests — Sequential ━━━
191	# ==============================================================================================
192	read -r -a DRIVES_TO_TEST <<< "$(_build_drive_list)"
193
194	if [[ ${#DRIVES_TO_TEST[@]} -eq 0 ]]; then
195	    warn "No drives to test — all may be on ignore list or SMART not enabled"
196	    exit 0
197	fi
198
199	echo ""
200	echo "━━━ $ICON_SMART SMART Extended Self-Test — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
201	echo "$ICON_HOST  ${#DRIVES_TO_TEST[@]} drive(s) to test — running sequentially"
202	echo ""
203
204	WINDOW_START=$(date +%s)
205	DRIVES_OK=()
206	DRIVES_FAIL=()
207	DRIVES_SKIP=()
208
209	for drive in "${DRIVES_TO_TEST[@]}"; do
210	    drive_name=$(basename "$drive")
211	    echo "━━━ $ICON_SMART $drive_name ━━━"
212
213	    if [[ "$DRY_RUN" == true ]]; then
214	        warn "DRY RUN — would run extended self-test on $drive_name"
215	        echo ""
216	        continue
217	    fi
218
219	    # Check if test already in progress
220	    if smartctl -l selftest "$drive" 2>/dev/null | grep -q "Self-test routine in progress"; then
221	        warn "$drive_name — self-test already in progress — skipping"
222	        DRIVES_SKIP+=("$drive_name")
223	        echo ""
224	        continue
225	    fi
226
227	    # Start extended test
228	    if ! smartctl -t long "$drive" >/dev/null 2>&1; then
229	        error "$drive_name — failed to start extended self-test"
230	        DRIVES_FAIL+=("$drive_name")
231	        echo ""
232	        continue
233	    fi
234	    echo "  Extended test started on $drive_name"
235
236	    # Poll until complete
237	    DRIVE_START=$(date +%s)
238	    while true; do
239	        sleep 60
240
241	        STATUS_LINE=$(smartctl -l selftest "$drive" 2>/dev/null | grep -m1 "Extended")
242
243	        if echo "$STATUS_LINE" | grep -q "Self-test routine in progress"; then
244	            PCT=$(echo "$STATUS_LINE" | grep -oE "[0-9]+% of test remaining" || true)
245	            log "$drive_name — test in progress ${PCT:+($PCT)}"
246	            continue
247	        fi
248
249	        # Test finished — determine result
250	        DRIVE_ELAPSED=$(format_duration $(( $(date +%s) - DRIVE_START )))
251	        if echo "$STATUS_LINE" | grep -iq "Completed without error\|Successful"; then
252	            echo "  $ICON_DONE $drive_name — extended test passed ✅ ($DRIVE_ELAPSED)"
253	            DRIVES_OK+=("$drive_name")
254	        elif echo "$STATUS_LINE" | grep -iq "Failed\|failed"; then
255	            FAIL_DETAIL=$(echo "$STATUS_LINE" | awk '{print $NF}')
256	            error "$drive_name — extended test FAILED — $FAIL_DETAIL ($DRIVE_ELAPSED)"
257	            DRIVES_FAIL+=("$drive_name")
258	        elif [[ -z "$STATUS_LINE" ]]; then
259	            warn "$drive_name — no test result found — may not support extended test"
260	            DRIVES_SKIP+=("$drive_name")
261	        else
262	            warn "$drive_name — unexpected result: $STATUS_LINE ($DRIVE_ELAPSED)"
263	            DRIVES_SKIP+=("$drive_name")
264	        fi
265	        echo ""
266	        break
267	    done
268	done
269
270	WINDOW_END=$(date +%s)
271
272	# ==============================================================================================
273	# ━━━ Summary ━━━
274	# ==============================================================================================
275	echo "━━━━━ $ICON_SUMMARY SMART LONG TEST SUMMARY ━━━━━"
276	echo "$ICON_HOST   Identity: $MY_ID ($LOCAL_SERVER_NAME)"
277	echo "$ICON_TIME   Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
278	echo "$ICON_SMART  Drives:   ${#DRIVES_TO_TEST[@]} tested"
279	[[ ${#DRIVES_OK[@]}   -gt 0 ]] && echo "$ICON_DONE   Passed:   ${DRIVES_OK[*]}"
280	[[ ${#DRIVES_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR  Failed:   ${DRIVES_FAIL[*]}"
281	[[ ${#DRIVES_SKIP[@]} -gt 0 ]] && echo "$ICON_WARN   Skipped:  ${DRIVES_SKIP[*]}"
282	echo ""
283
284	if [[ "$DRY_RUN" == true ]]; then
285	    warn "DRY RUN — no tests started"
286	elif [[ ${#DRIVES_FAIL[@]} -gt 0 ]]; then
287	    echo "$ICON_ERROR Status: FAILURES — ${DRIVES_FAIL[*]}"
288	    notify "SMART long test failures on $(hostname) ($MY_ID) — drives: ${DRIVES_FAIL[*]}" \
289	        "SMART Long Test" "warning"
290	elif [[ ${#DRIVES_OK[@]} -gt 0 ]]; then
291	    echo "$ICON_DONE Status: all ${#DRIVES_OK[@]} drive(s) passed ✅"
292	else
293	    warn "Status: no results — all drives skipped"
294	fi
295	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
296
297	[[ ${#DRIVES_FAIL[@]} -gt 0 ]] && exit 1
298	exit 0
299


1	#!/bin/bash
2	# ==============================================================================================
3	# =========================== Watchdog Skip List Manager =======================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# View and manage the persistent container skip list used by docker_watchdog.sh.
9	# docker_watchdog.sh adds a container to the skip list when it exceeds
10	# WATCHDOG_CONTAINER_RESTART_LIMIT restarts within WATCHDOG_CONTAINER_RESTART_WINDOW
11	# hours — prevents infinite restart loops on containers that keep crashing.
12	#
13	# Skip list persists on /boot/config (survives reboots). Auto-clears when
14	# docker_watchdog.sh sees the container running on a later cycle. Use this
15	# script to clear manually after fixing the underlying problem.
16	#
17	# ==============================================================================================
18	# OPERATIONAL MODEL
19	# ==============================================================================================
20	#
21	# Skip List Lifecycle
22	#   1. Container crashes repeatedly → watchdog adds to skip list, notifies
23	#   2. Watchdog stops restarting the container on subsequent cycles
24	#   3a. If container recovers on its own (Docker restart policy), watchdog
25	#       sees it running, removes from skip list automatically
26	#   3b. If stuck stopped → fix the root cause, clear via this script, then
27	#       docker start ContainerName manually
28	#   4. Watchdog monitors normally on next cycle. If it crashes again → re-added.
29	#
30	# ==============================================================================================
31	# OPERATIONAL SAFEGUARDS
32	# ==============================================================================================
33	#
34	# Single Instance Lock
35	#   acquire_lock prevents concurrent access with docker_watchdog.sh writing
36	#   the same files.
37	#
38	# Active Watchdog Detection
39	#   Warns if docker_watchdog.sh is currently running when a clear is attempted
40	#   — the watchdog could re-add the container to the skip list within seconds.
41	#
42	# Docker Timeout
43	#   DOCKER_TIMEOUT caps docker inspect calls against a hung daemon.
44	#
45	# Confirmation Required
46	#   Interactive mode prompts for YES before clearing. Use --force for scripts.
47	#
48	# ==============================================================================================
49	# STATE FILES
50	# ==============================================================================================
51	#
52	#   DOCKER_WATCHDOG_FAILED_FILE        — persistent container skip list (on /boot/config)
53	#   WATCHDOG_CONTAINER_RESTART_LOG  — restart history used for loop detection
54	#
55	# ==============================================================================================
56	# RUNTIME MODES
57	# ==============================================================================================
58	#
59	# watchdog_skip_list_manager.sh [--status]
60	#     Show skip list, container states, and recent restart history.
61	#
62	# watchdog_skip_list_manager.sh --clear ContainerName
63	#     Remove a specific container from the skip list and clear its restart history.
64	#     Prompts for YES unless --force is passed.
65	#
66	# watchdog_skip_list_manager.sh --clear-all
67	#     Clear all skip lists and all restart history.
68	#     Prompts for YES unless --force is passed.
69	#
70	# All actions support --dry-run (show what would change) and --force (skip prompt).
71	#
72	# ==============================================================================================
73
74	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
75
76	source "$SCRIPT_DIR/../load_config.sh"
77
78	DOCKER_TIMEOUT=15
79
80	# ── Parse action flags before parse_args ──────────────────────────────────────────────────────
81	ACTION="status"
82	TARGET_CONTAINER=""
83	FORCE=false
84	FILTERED_ARGS=()
85
86	for arg in "$@"; do
87	    case "$arg" in
88	        --clear-all)  ACTION="clear-all" ;;
89	        --clear)      ACTION="clear" ;;
90	        --status)     ACTION="status" ;;
91	        --force)      FORCE=true ;;
92	        *)
93	            if [[ "$ACTION" == "clear" && -z "$TARGET_CONTAINER" ]]; then
94	                TARGET_CONTAINER="$arg"
95	            else
96	                FILTERED_ARGS+=("$arg")
97	            fi
98	            ;;
99	    esac
100	done
101
102	parse_args "${FILTERED_ARGS[@]}"
103
104	# ==============================================================================================
105	# ━━━ Setup ━━━
106	# ==============================================================================================
107	if [[ "$EUID" -ne 0 ]]; then
108	    error "Must be run as root"
109	    exit 1
110	fi
111
112
113	acquire_lock
114
115	if ! command -v docker &>/dev/null; then
116	    error "Docker command not found"
117	    exit 1
118	fi
119
120	# detect_hosts() sets MY_ID — used in output
121	detect_hosts
122
123	log "$ICON_GEAR Config: action=${ACTION} container=${TARGET_CONTAINER:-all} skip-file=${DOCKER_WATCHDOG_FAILED_FILE}"
124	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
125	[[ "$FORCE"   == true ]] && warn "FORCE mode — confirmation prompt skipped"
126
127	# Ensure state files exist
128	touch "$DOCKER_WATCHDOG_FAILED_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
129
130	# ==============================================================================================
131	# ━━━ Status — always shown regardless of action ━━━
132	# ==============================================================================================
133	echo ""
134	echo "━━━ $ICON_WATCHDOG Skip List Status — $MY_ID ━━━"
135
136	SKIP_COUNT=$(grep -c "." "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null || echo 0)
137	SKIP_COUNT="${SKIP_COUNT//[^0-9]/}"; SKIP_COUNT="${SKIP_COUNT:-0}"
138	RESTART_COUNT=$(wc -l < "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
139	RESTART_COUNT="${RESTART_COUNT//[^0-9]/}"; RESTART_COUNT="${RESTART_COUNT:-0}"
140
141	# docker_watchdog.sh running check
142	WATCHDOG_RUNNING=false
143	if pgrep -f "docker_watchdog.sh" >/dev/null 2>&1; then
144	    WATCHDOG_RUNNING=true
145	    warn "docker_watchdog.sh is currently RUNNING"
146	    [[ "$ACTION" != "status" ]] && \
147	        warn "Clearing during an active cycle — watchdog may re-add container on next iteration"
148	fi
149
150	echo ""
151	if [[ "$SKIP_COUNT" -eq 0 ]]; then
152	    echo "Skip list: empty — all containers monitored normally ✅"
153	else
154	    warn "$SKIP_COUNT container(s) on skip list — manual intervention needed:"
155	    echo ""
156	    while IFS= read -r container; do
157	        [[ -z "$container" ]] && continue
158	        STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
159	            '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
160	        case "$STATUS" in
161	            true)
162	                echo "  $ICON_RUNNING $container — RUNNING (watchdog will auto-clear next cycle)"
163	                ;;
164	            false)
165	                echo "  $ICON_NOT_RUNNING $container — STOPPED — fix and start manually"
166	                ;;
167	            *)
168	                echo "  $ICON_WARN $container — not found on this server"
169	                ;;
170	        esac
171	    done < "$DOCKER_WATCHDOG_FAILED_FILE"
172	fi
173
174	echo ""
175	echo "━━━ $ICON_WATCHDOG Restart History ━━━"
176	if [[ "$RESTART_COUNT" -eq 0 ]]; then
177	    echo "No restart history"
178	else
179	    echo "$RESTART_COUNT restart entries (window: ${WATCHDOG_CONTAINER_RESTART_WINDOW}h)"
180	    echo ""
181	    awk -F'|' '{counts[$1]++} END {
182	        for (c in counts)
183	            printf "  %-30s %d restart(s)\n", c, counts[c]
184	    }' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | sort
185	fi
186
187	[[ "$ACTION" == "status" ]] && exit 0
188
189	# ==============================================================================================
190	# ━━━ Clear All ━━━
191	# ==============================================================================================
192	if [[ "$ACTION" == "clear-all" ]]; then
193	    echo ""
194	    echo "━━━ $ICON_TRASH Clear All Skip Lists ━━━"
195	    warn "This will clear the skip list and restart history for ALL containers"
196	    echo ""
197
198	    if [[ "$DRY_RUN" == false ]]; then
199	        if [[ "$FORCE" == true ]]; then
200	            log "FORCE flag set — skipping confirmation"
201	        elif [[ -t 0 ]]; then
202	            read -r -p "Type YES to confirm: " CONFIRM
203	            if [[ "$CONFIRM" != "YES" ]]; then
204	                warn "Cancelled"
205	                exit 0
206	            fi
207	        else
208	            error "Non-interactive mode — use --force flag to skip confirmation"
209	            exit 1
210	        fi
211
212	        > "$DOCKER_WATCHDOG_FAILED_FILE"
213	        > "$WATCHDOG_CONTAINER_RESTART_LOG"
214	        warn "Skip list cleared ✅"
215	        warn "Restart history cleared ✅"
216	        [[ "$WATCHDOG_RUNNING" == true ]] && \
217	            warn "Note: watchdog is running — containers will be monitored on next cycle"
218	        notify "Watchdog skip list cleared on $(hostname) ($MY_ID) — all containers will be monitored normally" \
219	            "Watchdog Manager" "warning"
220	    else
221	        warn "DRY RUN — would clear: $DOCKER_WATCHDOG_FAILED_FILE"
222	        warn "DRY RUN — would clear: $WATCHDOG_CONTAINER_RESTART_LOG"
223	    fi
224	fi
225
226	# ==============================================================================================
227	# ━━━ Clear Specific Container ━━━
228	# ==============================================================================================
229	if [[ "$ACTION" == "clear" ]]; then
230	    echo ""
231	    echo "━━━ $ICON_TRASH Clear Container: $TARGET_CONTAINER ━━━"
232
233	    if [[ -z "$TARGET_CONTAINER" ]]; then
234	        error "No container specified"
235	        error "Usage: watchdog_skip_list_manager.sh --clear ContainerName"
236	        exit 1
237	    fi
238
239	    # Remove from skip list
240	    if ! grep -q "^${TARGET_CONTAINER}$" "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null; then
241	        warn "$TARGET_CONTAINER is not on the skip list"
242	    else
243	        if [[ "$DRY_RUN" == false ]]; then
244	            sed -i "/^${TARGET_CONTAINER}$/d" "$DOCKER_WATCHDOG_FAILED_FILE"
245	            warn "$TARGET_CONTAINER removed from skip list ✅"
246	        else
247	            warn "DRY RUN — would remove $TARGET_CONTAINER from skip list"
248	        fi
249	    fi
250
251	    # Clear restart history for this container
252	    HIST_COUNT=$(grep -c "^${TARGET_CONTAINER}|" \
253	        "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null || echo 0)
254	    HIST_COUNT="${HIST_COUNT//[^0-9]/}"; HIST_COUNT="${HIST_COUNT:-0}"
255
256	    if [[ "$HIST_COUNT" -gt 0 ]]; then
257	        if [[ "$DRY_RUN" == false ]]; then
258	            sed -i "/^${TARGET_CONTAINER}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG"
259	            warn "Cleared $HIST_COUNT restart history entries for $TARGET_CONTAINER ✅"
260	        else
261	            warn "DRY RUN — would clear $HIST_COUNT restart history entries"
262	        fi
263	    else
264	        log "No restart history for $TARGET_CONTAINER"
265	    fi
266
267	    [[ "$WATCHDOG_RUNNING" == true ]] && \
268	        warn "Note: watchdog is running — $TARGET_CONTAINER may be re-added if still failing"
269
270	    if [[ "$DRY_RUN" == false ]]; then
271	        echo ""
272	        echo "━━━ $ICON_INFO Next Steps ━━━"
273	        echo "  1. Fix whatever was causing $TARGET_CONTAINER to fail"
274	        echo "  2. Start it manually: docker start $TARGET_CONTAINER"
275	        echo "  3. docker_watchdog.sh monitors it on the next cycle"
276	        echo "  4. If it crashes again → watchdog adds it back and notifies"
277	        notify "$TARGET_CONTAINER cleared from watchdog skip list on $(hostname) ($MY_ID)" \
278	            "Watchdog Manager" "warning"
279	    fi
280	fi
281
282	echo ""
283	echo "━━━━━ $ICON_SUMMARY DONE — $MY_ID ━━━━━"


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= ZFS Pool Scrub ============================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Triggers a ZFS scrub on all pools (or a specific pool), waits for completion,
9	# and sends a notification with any errors found. Reads every block on every pool
10	# and verifies checksums — catches silent corruption that would otherwise only
11	# surface when the corrupted data is read (possibly after redundancy can no
12	# longer help). Monthly recommended for all pools; quarterly minimum for large pools.
13	#
14	# ==============================================================================================
15	# OPERATIONAL MODEL
16	# ==============================================================================================
17	#
18	# Starts a scrub on each pool, then polls every 60 seconds until all complete.
19	# Progress is shown every poll — safe to leave running or interrupt. ZFS scrub
20	# continues in the kernel even if the script is stopped — it does not depend on
21	# this script remaining alive.
22	#
23	# Scrub is safe to run while the pool is in use. It does consume I/O bandwidth —
24	# schedule during off-peak hours or maintenance windows.
25	#
26	# Pools in HOST*_ZFS_REPORT_IGNORE_POOLS are skipped automatically (single-disk
27	# VM pools, temp pools, etc.). Specifying a pool by name bypasses the ignore list.
28	#
29	# ==============================================================================================
30	# OPERATIONAL SAFEGUARDS
31	# ==============================================================================================
32	#
33	# Single Instance Lock
34	#   acquire_lock prevents concurrent scrub starts on the same server.
35	#
36	# Scrub-in-Progress Check
37	#   Detects pools already scrubbing and skips them rather than erroring — safe
38	#   to run when a scrub may have been started by another path.
39	#
40	# SIGTERM Trap
41	#   The poll loop exits cleanly on signal. The ZFS scrub continues regardless.
42	#
43	# Tool Validation
44	#   platform_require_cmd confirms zpool and the notify script are present before use.
45	#
46	# Silent When Clean
47	#   Only errors produce visible output and a notification.
48	#
49	# ==============================================================================================
50	# CONFIGURATION
51	# ==============================================================================================
52	#
53	# host*.conf
54	#
55	#   HOST*_ZFS_REPORT_IGNORE_POOLS
56	#       Pools to exclude from automatic scrub. Typically single-disk VM pools
57	#       or temporary pools that do not need integrity checking.
58	#       Aliased by detect_hosts() → ZFS_REPORT_IGNORE_POOLS.
59	#
60	# ==============================================================================================
61	# RUNTIME MODES
62	# ==============================================================================================
63	#
64	# zfs_pool_scrub.sh
65	#     Scrub all pools not in ZFS_REPORT_IGNORE_POOLS. Wait for completion.
66	#
67	# zfs_pool_scrub.sh poolname
68	#     Scrub a specific pool by name. Bypasses the ignore list.
69	#
70	# zfs_pool_scrub.sh --status
71	#     Show current scrub status for all pools and exit.
72	#
73	# zfs_pool_scrub.sh --dry-run
74	#     Show which pools would be scrubbed. No scrub started.
75	#
76	# zfs_pool_scrub.sh --log
77	#     Verbose progress output every poll cycle.
78	#
79	# ==============================================================================================
80
81	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
82
83	source "$SCRIPT_DIR/../load_config.sh"
84
85	parse_args "$@"
86
87	TARGET_POOL="${PARSED_ARGS[0]:-}"
88	SCRUB_RUNNING=true
89
90	# ==============================================================================================
91	# ━━━ Setup ━━━
92	# ==============================================================================================
93	if [[ "$EUID" -ne 0 ]]; then
94	    error "Must be run as root"
95	    exit 1
96	fi
97
98	platform_require_cmd \
99	    "$(command -v zpool 2>/dev/null || echo /sbin/zpool)" \
100	    "--version" "" \
101	    "zpool" || {
102	        error "ZFS not available on this system — zpool not found"
103	        exit 1
104	    }
105
106
107	acquire_lock
108
109	# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS
110	detect_hosts
111
112	# Build ignore pool map
113	declare -A IGNORE_MAP
114	for pool in "${ZFS_REPORT_IGNORE_POOLS[@]:-}"; do
115	    [[ -n "$pool" ]] && IGNORE_MAP["$pool"]=1
116	done
117
118	log "Identity:     $MY_ID ($LOCAL_SERVER_NAME)"
119	log "Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
120
121	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scrubs will be started"
122
123	# SIGTERM trap — exit poll loop cleanly
124	trap 'warn "ZFS scrub script interrupted — scrub continues in background"; SCRUB_RUNNING=false; exit 0' \
125	    SIGTERM SIGINT
126
127	# ==============================================================================================
128	# ━━━ Status ━━━
129	# ==============================================================================================
130	if [[ "$SHOW_STATUS" == true ]]; then
131	    echo ""
132	    echo "━━━━━ $ICON_SUMMARY ZFS SCRUB STATUS ━━━━━"
133	    echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
134	    echo ""
135	    while IFS= read -r pool; do
136	        [[ -z "$pool" ]] && continue
137	        SCAN=$(zpool status "$pool" 2>/dev/null | grep "scan:")
138	        IGNORED=""
139	        [[ -n "${IGNORE_MAP[$pool]:-}" ]] && IGNORED=" (ignored)"
140	        echo "  $ICON_ZFS $pool${IGNORED} — ${SCAN:-no scan data}"
141	    done < <(zpool list -H -o name 2>/dev/null)
142	    echo ""
143	    echo "  Ignored pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
144	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
145	    exit 0
146	fi
147
148	# ==============================================================================================
149	# ── Build pool list ────────────────────────────────────────────────────────────────────────────
150	# ==============================================================================================
151	POOLS_TO_SCRUB=()
152
153	if [[ -n "$TARGET_POOL" ]]; then
154	    # Specific pool — bypass ignore list, validate exists
155	    if ! zpool list "$TARGET_POOL" >/dev/null 2>&1; then
156	        error "Pool not found: $TARGET_POOL"
157	        exit 1
158	    fi
159	    POOLS_TO_SCRUB=("$TARGET_POOL")
160	else
161	    # All pools — skip ignored ones
162	    while IFS= read -r pool; do
163	        [[ -z "$pool" ]] && continue
164	        if [[ -n "${IGNORE_MAP[$pool]:-}" ]]; then
165	            log "Skipping $pool (in ZFS_REPORT_IGNORE_POOLS)"
166	            continue
167	        fi
168	        POOLS_TO_SCRUB+=("$pool")
169	    done < <(zpool list -H -o name 2>/dev/null)
170	fi
171
172	if [[ ${#POOLS_TO_SCRUB[@]} -eq 0 ]]; then
173	    warn "No pools to scrub — all pools may be on the ignore list"
174	    warn "Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
175	    exit 0
176	fi
177
178	log "Pools to scrub: ${POOLS_TO_SCRUB[*]}"
179
180	# ==============================================================================================
181	# ━━━ Start Scrubs ━━━
182	# ==============================================================================================
183	echo ""
184	echo "━━━ $ICON_ZFS Starting ZFS Scrubs — $MY_ID ━━━"
185	START=$(date +%s)
186
187	STARTED=()
188	SKIPPED_POOLS=()
189
190	for pool in "${POOLS_TO_SCRUB[@]}"; do
191
192	    # Check if scrub already in progress
193	    ALREADY=$(zpool status "$pool" 2>/dev/null | grep "scan:" | grep -c "in progress" || true)
194	    if [[ "$ALREADY" -gt 0 ]]; then
195	        warn "$pool — scrub already in progress — joining existing scrub"
196	        STARTED+=("$pool")
197	        continue
198	    fi
199
200	    log "Starting scrub on $pool..."
201
202	    if [[ "$DRY_RUN" == true ]]; then
203	        warn "DRY RUN — would scrub: $pool"
204	        STARTED+=("$pool")
205	    elif zpool scrub "$pool" 2>/dev/null; then
206	        log "$pool scrub started ✅"
207	        STARTED+=("$pool")
208	    else
209	        error "Failed to start scrub on $pool"
210	        SKIPPED_POOLS+=("$pool")
211	    fi
212	done
213
214	if [[ "$DRY_RUN" == true ]]; then
215	    echo ""
216	    echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━"
217	    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
218	    warn "DRY RUN — no scrubs started"
219	    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
220	    exit 0
221	fi
222
223	if [[ ${#STARTED[@]} -eq 0 ]]; then
224	    error "No scrubs were started — check pool status"
225	    exit 1
226	fi
227
228	# ==============================================================================================
229	# ━━━ Poll Until Complete ━━━
230	# ==============================================================================================
231	echo ""
232	echo "━━━ $ICON_TIME Waiting for Scrubs to Complete ━━━"
233	log "Polling every 60 seconds — scrubs may take hours on large pools"
234	log "Safe to interrupt — scrubs continue in background if script is stopped"
235
236	while [[ "$SCRUB_RUNNING" == true ]]; do
237	    sleep 60
238
239	    STILL_RUNNING=false
240	    for pool in "${STARTED[@]}"; do
241	        IN_PROGRESS=$(zpool status "$pool" 2>/dev/null | \
242	            grep "scan:" | grep -c "in progress" || true)
243	        if [[ "$IN_PROGRESS" -gt 0 ]]; then
244	            STILL_RUNNING=true
245	            # Show progress — always visible so user knows it's running
246	            PROGRESS=$(zpool status "$pool" 2>/dev/null | \
247	                grep "scan:" | grep -oE "[0-9]+\.[0-9]+% done")
248	            REPAIRED=$(zpool status "$pool" 2>/dev/null | \
249	                grep "scan:" | grep -oE "[0-9]+ repaired")
250	            warn "$pool — scrub in progress ${PROGRESS:+$PROGRESS}${REPAIRED:+ ($REPAIRED)}"
251	        fi
252	    done
253
254	    [[ "$STILL_RUNNING" == false ]] && SCRUB_RUNNING=false
255	done
256
257	END=$(date +%s)
258	warn "All scrubs complete — $(format_duration $(( END - START )))"
259
260	# ==============================================================================================
261	# ━━━ Results ━━━
262	# ==============================================================================================
263	echo ""
264	echo "━━━ $ICON_ZFS Scrub Results ━━━"
265
266	POOLS_OK=()
267	POOLS_ERRORS=()
268
269	for pool in "${STARTED[@]}"; do
270	    SCAN_LINE=$(zpool status "$pool" 2>/dev/null | grep "scan:")
271	    ERRORS=$(zpool status "$pool" 2>/dev/null | \
272	        grep "errors:" | grep -v "No known data errors")
273
274	    if [[ -n "$ERRORS" ]]; then
275	        error "$pool — ERRORS FOUND"
276	        error "  $SCAN_LINE"
277	        error "  $ERRORS"
278	        POOLS_ERRORS+=("$pool")
279	    else
280	        echo "$pool — $SCAN_LINE"
281	        POOLS_OK+=("$pool")
282	    fi
283	done
284
285	# ==============================================================================================
286	# ━━━ Summary ━━━
287	# ==============================================================================================
288	echo ""
289	echo "━━━━━ $ICON_SUMMARY ZFS SCRUB SUMMARY ━━━━━"
290	echo "$ICON_HOST   Identity:     $MY_ID ($LOCAL_SERVER_NAME)"
291	echo "$ICON_ZFS    Pools:        ${#POOLS_TO_SCRUB[@]} to scrub"
292	echo "$ICON_SUCCESS Clean:       ${#POOLS_OK[@]}"
293	[[ ${#POOLS_ERRORS[@]} -gt 0 ]] && echo "$ICON_ERROR  Errors:      ${#POOLS_ERRORS[@]}"
294	[[ ${#SKIPPED_POOLS[@]} -gt 0 ]] && warn "Failed start: ${SKIPPED_POOLS[*]}"
295	echo "$ICON_TIME   Duration:     $(format_duration $(( END - START )))"
296	echo ""
297
298	if [[ ${#POOLS_ERRORS[@]} -gt 0 ]]; then
299	    echo "$ICON_ERROR Status: ERRORS FOUND — ${POOLS_ERRORS[*]}"
300	    notify "ZFS scrub errors on $(hostname) ($MY_ID) — pools with errors: ${POOLS_ERRORS[*]}" \
301	        "ZFS Scrub" "warning"
302	elif [[ ${#POOLS_OK[@]} -gt 0 ]]; then
303	    echo "$ICON_DONE Status: all ${#POOLS_OK[@]} pools clean ✅"
304	fi
305	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
306
307	[[ ${#POOLS_ERRORS[@]} -gt 0 ]] && exit 1
308	exit 0


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Ramdisk Setup ==============================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Creates the tmpfs ramdisk, SSD fallback directory, transcode symlink, and
9	# pre-creates transcoding-temp on the ramdisk. Run once at array start via
10	# array_started.sh (System_Essentials/). Idempotent — already-mounted ramdisk
11	# reports status and exits cleanly. Always resets the symlink to the ramdisk
12	# on boot, ensuring a clean state regardless of what state it was in before
13	# shutdown.
14	#
15	# ==============================================================================================
16	# OPERATIONAL MODEL
17	# ==============================================================================================
18	#
19	# Creates four things in order:
20	#   1. RAMDISK_PATH  — tmpfs mount (size: HOST*_RAMDISK_SIZE ceiling, not a reservation)
21	#   2. TRANSCODE_SSD — SSD fallback directory and transcoding-temp inside it
22	#   3. TRANSCODE_LINK — symlink reset to RAMDISK_PATH (clean state at every boot)
23	#   4. transcoding-temp/ inside RAMDISK_PATH — pre-created before Emby starts
24	#
25	# The transcoding-temp pre-creation is critical: if it doesn't exist on the ramdisk
26	# when Emby starts, Emby searches all accessible paths for an existing one and finds
27	# the SSD fallback version — routing all sessions there until Emby restarts.
28	#
29	# Initialises /tmp/transcode_state.db with current target and flip counters.
30	# /tmp resets on reboot — correct, transcode state should not persist across boots.
31	#
32	# ==============================================================================================
33	# OPERATIONAL SAFEGUARDS
34	# ==============================================================================================
35	#
36	# Root Required
37	#   mount and symlink creation require root.
38	#
39	# Single Instance Lock
40	#   acquire_lock prevents duplicate runs at array start.
41	#
42	# Idempotent Mount Check
43	#   If RAMDISK_PATH is already a mountpoint, reports status and exits cleanly
44	#   without attempting to remount or changing anything.
45	#
46	# Notification Validated
47	#   platform_require_cmd confirms the notify script is present before use.
48	#
49	# Silent on Success
50	#   Startup script runs on every boot — no output when healthy.
51	#
52	# ==============================================================================================
53	# CONFIGURATION
54	# ==============================================================================================
55	#
56	# host*.conf
57	#
58	#   HOST*_RAMDISK_SIZE
59	#       tmpfs ceiling (e.g. 10G). Must change together with WARN_GB and LOW_GB.
60	#       Aliased by detect_hosts() → RAMDISK_SIZE.
61	#
62	#   HOST*_RAMDISK_WARN_GB
63	#       Usage level at which transcode_manager.sh flips symlink to SSD.
64	#
65	#   HOST*_RAMDISK_LOW_GB
66	#       Usage level at which transcode_manager.sh flips back to ramdisk.
67	#
68	#   HOST*_TRANSCODE_SSD
69	#       SSD fallback directory path.
70	#       Aliased by detect_hosts() → TRANSCODE_SSD.
71	#
72	# master.conf
73	#
74	#   TRANSCODE_LINK
75	#       Symlink path Emby uses as its transcode directory. Must match the path
76	#       configured in Emby's transcoding settings.
77	#
78	#   TRANSCODE_CHMOD / TRANSCODE_OWNER
79	#       Permissions applied to both ramdisk and SSD directories. (default: 755 / nobody:users)
80	#
81	# ==============================================================================================
82	# STATE FILES
83	# ==============================================================================================
84	#
85	#   /tmp/transcode_state.db    — current symlink target + flip count tracking
86	#                                Lives in /tmp (ephemeral — resets on reboot correctly)
87	#
88	# ==============================================================================================
89	# RUNTIME MODES
90	# ==============================================================================================
91	#
92	# ramdisk_setup.sh
93	#     Normal setup run. Called by array_started.sh at boot.
94	#
95	# ramdisk_setup.sh --dry-run
96	#     Show what would be created without creating anything.
97	#
98	# ramdisk_setup.sh --status
99	#     Show current ramdisk mount state, symlink target, and SSD directory state.
100	#
101	# ramdisk_setup.sh --log
102	#     Verbose output showing each creation step.
103	#
104	# ==============================================================================================
105
106	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
107
108	source "$SCRIPT_DIR/../load_config.sh"
109
110	parse_args "$@"
111
112	# ==============================================================================================
113	# ━━━ Setup ━━━
114	# ==============================================================================================
115	if [[ "$EUID" -ne 0 ]]; then
116	    error "Must be run as root — mount and symlink require root"
117	    exit 1
118	fi
119
120
121	acquire_lock
122
123	# detect_hosts() sets MY_ID and aliases RAMDISK_SIZE, TRANSCODE_SSD etc.
124	detect_hosts
125
126	log "Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
127	log "Ramdisk:   $RAMDISK_PATH ($RAMDISK_SIZE)"
128	log "Fallback:  $TRANSCODE_SSD"
129	log "Symlink:   $TRANSCODE_LINK"
130
131	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
132
133	# ==============================================================================================
134	# ━━━ Status ━━━
135	# ==============================================================================================
136	if [[ "$SHOW_STATUS" == true ]]; then
137	    echo ""
138	    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
139	    echo "$ICON_HOST   Identity:      $MY_ID ($LOCAL_SERVER_NAME)"
140	    echo "$ICON_RAM    Ramdisk path:  $RAMDISK_PATH"
141	    echo "$ICON_RAM    Ramdisk size:  $RAMDISK_SIZE"
142	    echo "$ICON_RAM    Warn at:       ${RAMDISK_WARN_GB}GB"
143	    echo "$ICON_RAM    Flip at:       ${RAMDISK_LOW_GB}GB"
144	    echo "$ICON_DISK   SSD fallback:  $TRANSCODE_SSD"
145	    echo "$ICON_LINK   Symlink:       $TRANSCODE_LINK"
146	    echo "$ICON_GEAR   Owner:         $TRANSCODE_OWNER"
147	    echo "$ICON_GEAR   Mode:          $TRANSCODE_CHMOD"
148	    echo ""
149
150	    if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
151	        USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
152	        AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
153	        echo "  $ICON_RAM  Ramdisk: mounted — $USAGE used / $AVAIL available ✅"
154	    else
155	        echo "  $ICON_RAM  Ramdisk: NOT mounted"
156	    fi
157
158	    if [[ -L "$TRANSCODE_LINK" ]]; then
159	        TARGET=$(readlink "$TRANSCODE_LINK")
160	        echo "  $ICON_LINK Symlink: $TRANSCODE_LINK → $TARGET"
161	    else
162	        echo "  $ICON_LINK Symlink: not set"
163	    fi
164
165	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
166	    exit 0
167	fi
168
169	# ==============================================================================================
170	# ━━━ Ramdisk ━━━
171	# ==============================================================================================
172	echo ""
173	echo "━━━ $ICON_RAM Ramdisk — $MY_ID ━━━"
174	log "$ICON_RAM  Path: $RAMDISK_PATH"
175	log "$ICON_RAM  Size: $RAMDISK_SIZE"
176	echo ""
177
178	START=$(date +%s)
179	SETUP_SUCCESS=true
180
181	if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
182	    USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
183	    AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
184	    log "Ramdisk already mounted — $USAGE used / $AVAIL available"
185	    log "Skipping mount — verifying symlink and permissions"
186	else
187	    if [[ "$DRY_RUN" == true ]]; then
188	        warn "DRY RUN — would create $RAMDISK_PATH"
189	        warn "DRY RUN — would mount tmpfs ${RAMDISK_SIZE} at $RAMDISK_PATH"
190	    else
191	        log "Creating ramdisk mount point: $RAMDISK_PATH"
192	        mkdir -p "$RAMDISK_PATH" || {
193	            error "Failed to create $RAMDISK_PATH"
194	            notify "Ramdisk setup failed on $(hostname) ($MY_ID) — could not create mount point" \
195	                "Ramdisk Setup" "warning"
196	            exit 1
197	        }
198
199	        log "Mounting tmpfs ${RAMDISK_SIZE} at $RAMDISK_PATH..."
200	        if mount -t tmpfs -o size="$RAMDISK_SIZE" tmpfs "$RAMDISK_PATH"; then
201	            warn "Ramdisk mounted — ${RAMDISK_SIZE} at $RAMDISK_PATH ✅"
202	        else
203	            error "Failed to mount ramdisk at $RAMDISK_PATH"
204	            notify "Ramdisk setup failed on $(hostname) ($MY_ID) — mount failed" \
205	                "Ramdisk Setup" "warning"
206	            exit 1
207	        fi
208	    fi
209	fi
210
211	# ==============================================================================================
212	# ━━━ SSD Fallback ━━━
213	# ==============================================================================================
214	echo ""
215	echo "━━━ $ICON_DISK SSD Fallback ━━━"
216
217	if [[ "$DRY_RUN" == true ]]; then
218	    warn "DRY RUN — would create SSD fallback: $TRANSCODE_SSD"
219	else
220	    if [[ -d "$TRANSCODE_SSD" ]]; then
221	        log "SSD fallback already exists: $TRANSCODE_SSD"
222	    else
223	        log "Creating SSD fallback directory: $TRANSCODE_SSD"
224	        if mkdir -p "$TRANSCODE_SSD"; then
225	            log "SSD fallback created: $TRANSCODE_SSD ✅"
226	        else
227	            error "Failed to create SSD fallback: $TRANSCODE_SSD"
228	            SETUP_SUCCESS=false
229	        fi
230	    fi
231	fi
232
233	# ==============================================================================================
234	# ━━━ Symlink ━━━
235	# ==============================================================================================
236	echo ""
237	echo "━━━ $ICON_LINK Symlink ━━━"
238
239	if [[ "$DRY_RUN" == true ]]; then
240	    warn "DRY RUN — would set $TRANSCODE_LINK → $RAMDISK_PATH"
241	else
242	    if [[ -L "$TRANSCODE_LINK" ]]; then
243	        CURRENT_TARGET=$(readlink "$TRANSCODE_LINK")
244	        if [[ "$CURRENT_TARGET" == "$RAMDISK_PATH" ]]; then
245	            log "Symlink already points to ramdisk — no change needed ✅"
246	        else
247	            log "Updating symlink: $CURRENT_TARGET → $RAMDISK_PATH"
248	            ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" || {
249	                error "Failed to update symlink"
250	                SETUP_SUCCESS=false
251	            }
252	        fi
253	    elif [[ -e "$TRANSCODE_LINK" ]]; then
254	        warn "$TRANSCODE_LINK exists but is not a symlink — removing and replacing"
255	        rm -rf "$TRANSCODE_LINK"
256	        ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" || {
257	            error "Failed to create symlink"
258	            SETUP_SUCCESS=false
259	        }
260	    else
261	        log "Creating symlink: $TRANSCODE_LINK → $RAMDISK_PATH"
262	        mkdir -p "$(dirname "$TRANSCODE_LINK")"
263	        ln -sfn "$RAMDISK_PATH" "$TRANSCODE_LINK" || {
264	            error "Failed to create symlink"
265	            SETUP_SUCCESS=false
266	        }
267	    fi
268
269	    [[ "$SETUP_SUCCESS" == true ]] && log "Symlink: $TRANSCODE_LINK → $RAMDISK_PATH ✅"
270	fi
271
272	# ==============================================================================================
273	# ━━━ Transcoding-temp Directory ━━━
274	# ==============================================================================================
275	# Pre-created inside ramdisk so Emby always finds it there at session start.
276	# Without this Emby creates it at its own first-writable path — which may be
277	# SSD even when the symlink points at the ramdisk — locking all sessions onto SSD.
278	echo ""
279	echo "━━━ $ICON_GEAR Transcoding Temp Directory ━━━"
280
281	TRANSCODE_TEMP_DIR="${RAMDISK_PATH}/transcoding-temp"
282
283	if [[ "$DRY_RUN" == true ]]; then
284	    warn "DRY RUN — would create $TRANSCODE_TEMP_DIR"
285	else
286	    if [[ -d "$TRANSCODE_TEMP_DIR" ]]; then
287	        log "transcoding-temp already exists on ramdisk"
288	    else
289	        if mkdir -p "$TRANSCODE_TEMP_DIR"; then
290	            log "Created transcoding-temp on ramdisk ✅"
291	        else
292	            error "Failed to create transcoding-temp on ramdisk"
293	            SETUP_SUCCESS=false
294	        fi
295	    fi
296
297	    if [[ -d "$TRANSCODE_TEMP_DIR" ]]; then
298	        chmod "$TRANSCODE_CHMOD" "$TRANSCODE_TEMP_DIR"
299	        chown "$TRANSCODE_OWNER" "$TRANSCODE_TEMP_DIR"
300	        log "Permissions set on transcoding-temp ($TRANSCODE_CHMOD $TRANSCODE_OWNER)"
301	    fi
302	fi
303
304	# ==============================================================================================
305	# ━━━ Permissions ━━━
306	# ==============================================================================================
307	echo ""
308	echo "━━━ $ICON_GEAR Permissions ━━━"
309
310	if [[ "$DRY_RUN" == true ]]; then
311	    warn "DRY RUN — would apply $TRANSCODE_CHMOD $TRANSCODE_OWNER to $RAMDISK_PATH and $TRANSCODE_SSD"
312	else
313	    for path in "$RAMDISK_PATH" "$TRANSCODE_SSD"; do
314	        if [[ -d "$path" ]]; then
315	            chmod "$TRANSCODE_CHMOD" "$path"
316	            chown "$TRANSCODE_OWNER" "$path"
317	            log "Permissions set: $path ($TRANSCODE_CHMOD $TRANSCODE_OWNER)"
318	        fi
319	    done
320	fi
321
322	# ==============================================================================================
323	# ━━━ Initialise State File ━━━
324	# ==============================================================================================
325	if [[ "$DRY_RUN" == false ]]; then
326	    STATE_FILE="${TRANSCODE_STATE_FILE:-${STATE_DIR:-/tmp}/transcode_state.db}"
327	    NOW=$(date +%s)
328	    cat > "$STATE_FILE" <<EOF
329	current_target=$RAMDISK_PATH
330	last_flip_time=$NOW
331	flip_count_hour=0
332	flip_hour_start=$NOW
333	EOF
334	    log "State file initialised: $STATE_FILE"
335	fi
336
337	END=$(date +%s)
338
339	# ==============================================================================================
340	# ━━━ Summary ━━━
341	# ==============================================================================================
342	echo ""
343	echo "━━━━━ $ICON_SUMMARY RAMDISK SETUP SUMMARY ━━━━━"
344	echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
345	echo "$ICON_RAM   Ramdisk:  $RAMDISK_PATH ($RAMDISK_SIZE)"
346	echo "$ICON_DISK  Fallback: $TRANSCODE_SSD"
347	echo "$ICON_LINK  Symlink:  $TRANSCODE_LINK"
348	echo "$ICON_TIME  Duration: $(format_duration $(( END - START )))"
349	echo ""
350
351	if [[ "$DRY_RUN" == true ]]; then
352	    warn "DRY RUN — no changes made"
353	elif [[ "$SETUP_SUCCESS" == true ]]; then
354	    echo "$ICON_DONE Status: done ✅"
355	else
356	    echo "$ICON_ERROR Status: SETUP HAD ERRORS"
357	    notify "Ramdisk setup errors on $(hostname) ($MY_ID) — check output" \
358	        "Ramdisk Setup" "warning"
359	    exit 1
360	fi
361	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Transcode Cleanup ==========================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Removes stale transcode files from both ramdisk and SSD fallback locations.
9	# Called by transcode_management.sh (Orchestrators/) before transcode_manager.sh —
10	# cleanup must run first so the manager sees real active-session usage, not
11	# inflated usage from stale files. Must be fast and non-blocking.
12	#
13	# A file is eligible for deletion only if ALL conditions are true:
14	#   1. Older than TRANSCODE_MAX_AGE minutes (mtime — last write time)
15	#   2. Not currently open by any process (checked via lsof pre-built map)
16	#
17	# ==============================================================================================
18	# DESIGN PRINCIPLES
19	# ==============================================================================================
20	#
21	# lsof Called Once, Not Per File
22	#   On a busy Live TV system the ramdisk contains thousands of HLS segment files.
23	#   Calling lsof once per file creates thousands of subprocess calls every 7 minutes.
24	#   lsof is called once per location to build a complete open-file map. All subsequent
25	#   checks are O(1) lookups against that map — thousands of files, one lsof call.
26	#
27	# No Session-Aware Cleanup
28	#   ffmpeg generates folder names independently of the media server API session IDs.
29	#   There is no reliable correlation between API session IDs and transcoding-temp
30	#   subfolder names. Attempting to correlate them would falsely treat active sessions
31	#   as ended. lsof is the correct check — if ffmpeg has a file open, it is active
32	#   regardless of folder naming or session state.
33	#
34	# transcoding-temp Is Never Deleted
35	#   If cleanup removes the empty transcoding-temp folder from the ramdisk, Emby
36	#   searches all accessible paths for an existing one, finds the SSD fallback version,
37	#   and routes all new sessions there until Emby restarts. The directory is excluded
38	#   from find by name — protected even when completely empty.
39	#
40	# Post-Cleanup Flip-Back
41	#   After removing stale files, checks whether ramdisk usage dropped below
42	#   RAMDISK_LOW_GB. If so — and symlink currently points at SSD — triggers a
43	#   flip back to ramdisk. This is the recovery path; the manager handles
44	#   the fill-up path.
45	#
46	# ==============================================================================================
47	# OPERATIONAL SAFEGUARDS
48	# ==============================================================================================
49	#
50	# Wait Lock
51	#   acquire_lock "wait" — waits if a previous cleanup run is still active rather
52	#   than exiting. The caller's 7-minute interval can overlap on a slow system.
53	#
54	# lsof Timeout
55	#   lsof call capped at 15 seconds per location — prevents blocking indefinitely
56	#   on a system with many open files.
57	#
58	# transcoding-temp Guard
59	#   `! -name "transcoding-temp"` in the find command — protected unconditionally.
60	#
61	# Silent by Default
62	#   Runs every 7 minutes — must not produce noise when healthy.
63	#
64	# ==============================================================================================
65	# CONFIGURATION
66	# ==============================================================================================
67	#
68	# host*.conf
69	#
70	#   HOST*_RAMDISK_PATH / HOST*_TRANSCODE_SSD / HOST*_RAMDISK_LOW_GB
71	#       Aliased by detect_hosts() → RAMDISK_PATH / TRANSCODE_SSD / RAMDISK_LOW_GB.
72	#
73	# master.conf
74	#
75	#   TRANSCODE_MAX_AGE
76	#       Minutes before an inactive transcode file is eligible for deletion. (default: 20)
77	#
78	#   TRANSCODE_ORPHAN_AGE
79	#       Minutes for orphan folder detection. (default: 30)
80	#
81	# ==============================================================================================
82	# RUNTIME MODES
83	# ==============================================================================================
84	#
85	# transcode_cleanup.sh
86	#     Remove stale files from ramdisk and SSD. Check for flip-back opportunity.
87	#
88	# transcode_cleanup.sh --dry-run
89	#     Show which files would be deleted. No deletions, no flip.
90	#
91	# transcode_cleanup.sh --status
92	#     Show current file counts, ages, and open-file status per location.
93	#
94	# transcode_cleanup.sh --log
95	#     Verbose per-file output including age, open status, and deletion result.
96	#
97	# ==============================================================================================
98
99	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
100
101	source "$SCRIPT_DIR/../load_config.sh"
102
103	parse_args "$@"
104
105	STATE_FILE="${TRANSCODE_STATE_FILE:-${STATE_DIR:-/tmp}/transcode_state.db}"
106
107	# ==============================================================================================
108	# ━━━ Setup ━━━
109	# ==============================================================================================
110	if [[ "$EUID" -ne 0 ]]; then
111	    error "Must be run as root"
112	    exit 1
113	fi
114
115
116	acquire_lock "wait"
117
118	# detect_hosts() sets MY_ID and aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_LOW_GB
119	detect_hosts
120
121	# lsof availability check
122	LSOF_AVAILABLE=false
123	if command -v lsof >/dev/null 2>&1; then
124	    LSOF_AVAILABLE=true
125	    log "lsof available — active file check enabled"
126	else
127	    warn "lsof not available — active file check skipped, all aged files eligible for deletion"
128	fi
129
130	log "Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
131	log "$ICON_GEAR Config:    max-age=${TRANSCODE_MAX_AGE}min orphan-age=${TRANSCODE_ORPHAN_AGE}min flip-back-below=${RAMDISK_LOW_GB}GB"
132	log "$ICON_RAM  Locations: ramdisk=$RAMDISK_PATH  ssd=$TRANSCODE_SSD"
133
134	# ==============================================================================================
135	# ━━━ Status ━━━
136	# ==============================================================================================
137	if [[ "$SHOW_STATUS" == true ]]; then
138	    echo ""
139	    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
140	    echo "$ICON_HOST   Identity:     $MY_ID ($LOCAL_SERVER_NAME)"
141	    echo "$ICON_RAM    Ramdisk:      $RAMDISK_PATH"
142	    echo "$ICON_DISK   SSD fallback: $TRANSCODE_SSD"
143	    echo "$ICON_TRASH  Max age:      ${TRANSCODE_MAX_AGE} minutes"
144	    echo "$ICON_TRASH  Orphan age:   ${TRANSCODE_ORPHAN_AGE} minutes"
145	    echo "$ICON_RAM    Flip at:      ${RAMDISK_LOW_GB}GB"
146	    echo "$ICON_GEAR   lsof check:   $LSOF_AVAILABLE"
147	    echo "$ICON_GEAR   Dry Run:      $DRY_RUN"
148	    echo ""
149
150	    if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
151	        USAGE=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $3}')
152	        AVAIL=$(df -BG "$RAMDISK_PATH" | awk 'NR==2 {print $4}')
153	        echo "  $ICON_RAM  Ramdisk: mounted — $USAGE used / $AVAIL available ✅"
154	    else
155	        echo "  $ICON_RAM  Ramdisk: not mounted"
156	    fi
157
158	    CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
159	    echo "  $ICON_LINK Current target: ${CURRENT_TARGET:-unknown}"
160	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
161	    exit 0
162	fi
163
164	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
165
166	# ==============================================================================================
167	# ── CLEANUP FUNCTION ──────────────────────────────────────────────────────────────────────────
168	# ==============================================================================================
169	# Scans a location and removes eligible files.
170	# Calls lsof ONCE per location — builds in-memory OPEN_FILES_MAP for O(1) lookup.
171	# Returns via LOCATION_REMOVED, LOCATION_FREED, LOCATION_SKIPPED, LOCATION_ACTIVE
172
173	cleanup_location() {
174	    local location="$1" label="$2" max_age="$3"
175	    local files_removed=0 bytes_freed=0 files_skipped=0 files_active=0 files_streaming=0
176
177	    if [[ ! -d "$location" ]]; then
178	        log "$label does not exist — skipping"
179	        LOCATION_REMOVED=0 LOCATION_FREED="0B" LOCATION_SKIPPED=0 LOCATION_ACTIVE=0 LOCATION_STREAMING=0
180	        return
181	    fi
182
183	    local file_count
184	    file_count=$(find "$location" -type f 2>/dev/null | wc -l)
185	    log "$label: $file_count files to scan (age threshold: ${max_age}min)"
186
187	    # Build in-memory open file map — O(1) lookup per file
188	    # One lsof call per location — never per file
189	    declare -A OPEN_FILES_MAP
190	    if [[ "$LSOF_AVAILABLE" == true ]]; then
191	        log "Building open file map for $label..."
192	        while IFS= read -r open_file; do
193	            [[ -n "$open_file" ]] && OPEN_FILES_MAP["$open_file"]=1
194	        done < <(timeout 15 lsof +D "$location" 2>/dev/null | awk 'NR>1 {print $9}' | sort -u)
195	        files_streaming=${#OPEN_FILES_MAP[@]}
196	        log "$files_streaming files currently open in $label"
197	    fi
198
199	    # Process aged files
200	    while IFS= read -r file; do
201	        [[ -z "$file" ]] && continue
202
203	        # O(1) open file check — in-memory map
204	        if [[ -n "${OPEN_FILES_MAP[$file]:-}" ]]; then
205	            (( files_active++ ))
206	            log "Skipping open file: $file"
207	            continue
208	        fi
209
210	        local file_size
211	        file_size=$(stat -c%s "$file" 2>/dev/null || echo 0)
212
213	        if [[ "$DRY_RUN" == true ]]; then
214	            log "DRY RUN — would delete: $(basename "$file")"
215	            (( files_skipped++ ))
216	        else
217	            if rm -f "$file" 2>/dev/null; then
218	                (( files_removed++ ))
219	                bytes_freed=$(( bytes_freed + file_size ))
220	                log "Deleted: $file"
221	            else
222	                warn "Could not delete: $file"
223	                (( files_skipped++ ))
224	            fi
225	        fi
226
227	    done < <(find "$location" -type f -mmin +"$max_age" 2>/dev/null)
228
229	    # Remove empty directories — but NEVER remove transcoding-temp
230	    # transcoding-temp must always exist on ramdisk so Emby finds it there first
231	    if [[ "$DRY_RUN" == false ]]; then
232	        find "$location" -mindepth 1 -type d -empty \
233	            ! -name "transcoding-temp" -delete 2>/dev/null
234	    fi
235
236	    # Format bytes freed
237	    local freed_human
238	    if (( bytes_freed > 1073741824 )); then
239	        freed_human=$(awk "BEGIN {printf \"%.1fGB\", $bytes_freed / 1073741824}")
240	    elif (( bytes_freed > 1048576 )); then
241	        freed_human=$(awk "BEGIN {printf \"%.1fMB\", $bytes_freed / 1048576}")
242	    elif (( bytes_freed > 0 )); then
243	        freed_human="${bytes_freed}B"
244	    else
245	        freed_human="0B"
246	    fi
247
248	    log "$label — removed $files_removed files ($freed_human freed) | streaming: $files_streaming | aged+open: $files_active | skipped: $files_skipped"
249
250	    LOCATION_REMOVED=$files_removed
251	    LOCATION_FREED=$freed_human
252	    LOCATION_SKIPPED=$files_skipped
253	    LOCATION_ACTIVE=$files_active
254	    LOCATION_STREAMING=$files_streaming
255	}
256
257	# ==============================================================================================
258	# ━━━ Transcode Cleanup ━━━
259	# ==============================================================================================
260	START=$(date +%s)
261	TOTAL_REMOVED=0 TOTAL_SKIPPED=0 TOTAL_ACTIVE=0 TOTAL_STREAMING=0
262	RAMDISK_FREED="0B" SSD_FREED="0B"
263
264	# Cleanup ramdisk
265	if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
266	    cleanup_location "$RAMDISK_PATH" "Ramdisk" "$TRANSCODE_MAX_AGE"
267	    TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED ))
268	    TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED ))
269	    TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE ))
270	    TOTAL_STREAMING=$(( TOTAL_STREAMING + LOCATION_STREAMING ))
271	    RAMDISK_FREED=$LOCATION_FREED
272	else
273	    log "Ramdisk not mounted — skipping ramdisk cleanup"
274	fi
275
276	# Cleanup SSD fallback
277	if [[ -d "$TRANSCODE_SSD" ]]; then
278	    cleanup_location "$TRANSCODE_SSD" "SSD fallback" "$TRANSCODE_MAX_AGE"
279	    TOTAL_REMOVED=$(( TOTAL_REMOVED + LOCATION_REMOVED ))
280	    TOTAL_SKIPPED=$(( TOTAL_SKIPPED + LOCATION_SKIPPED ))
281	    TOTAL_ACTIVE=$(( TOTAL_ACTIVE + LOCATION_ACTIVE ))
282	    TOTAL_STREAMING=$(( TOTAL_STREAMING + LOCATION_STREAMING ))
283	    SSD_FREED=$LOCATION_FREED
284	else
285	    log "SSD fallback not found — skipping SSD cleanup"
286	fi
287
288	# Post-cleanup — check if ramdisk recovered enough to flip symlink back to ramdisk
289	if [[ "$DRY_RUN" == false ]] && mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
290	    RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
291	    RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
292	    LOW_RECOVERED=$(awk "BEGIN {print ($RAMDISK_USED_GB < $RAMDISK_LOW_GB) ? 1 : 0}")
293	    CURRENT_TARGET=$(grep "^current_target=" "$STATE_FILE" 2>/dev/null | cut -d= -f2)
294
295	    if [[ "$LOW_RECOVERED" == "1" && "$CURRENT_TARGET" == "$TRANSCODE_SSD" ]]; then
296	        echo "Ramdisk has space after cleanup (${RAMDISK_USED_GB}GB < ${RAMDISK_LOW_GB}GB) — triggering manager to flip back"
297	        bash "$SCRIPT_DIR/transcode_manager.sh" --no-log
298	    fi
299	fi
300
301	END=$(date +%s)
302
303	# ==============================================================================================
304	# ━━━ Summary ━━━
305	# ==============================================================================================
306	echo "━━━━━ $ICON_SUMMARY TRANSCODE CLEANUP SUMMARY ━━━━━"
307	echo "$ICON_HOST   Identity: $MY_ID ($LOCAL_SERVER_NAME)"
308	echo "$ICON_RAM    Ramdisk freed: $RAMDISK_FREED"
309	echo "$ICON_DISK   SSD freed:     $SSD_FREED"
310	echo "$ICON_TRASH  Removed:       $TOTAL_REMOVED files"
311	echo "$ICON_RUNNING Streaming:    $TOTAL_STREAMING files (open by Emby — untouched)"
312	echo "$ICON_SHIELD  Protected:    $TOTAL_ACTIVE aged files saved by open-file check"
313	echo "$ICON_TRASH  Skipped:       $TOTAL_SKIPPED files"
314	echo "$ICON_TIME   Duration:      $(format_duration $(( END - START )))"
315
316	if [[ "$DRY_RUN" == true ]]; then
317	    warn "DRY RUN — no files deleted"
318	else
319	    echo "$ICON_DONE Status: done ✅"
320	fi
321	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"


Let me finish reading the remaining scripts.


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Transcode Manager ==========================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Monitors ramdisk usage and manages the transcode symlink direction. Called by
9	# transcode_management.sh (Orchestrators/) every 7 minutes — always after
10	# transcode_cleanup.sh runs first. Must be fast, non-blocking, and silent when
11	# nothing has changed.
12	#
13	# Emby's transcode path points at TRANSCODE_LINK (a symlink). ffmpeg resolves
14	# the symlink once at session start and holds a direct reference — existing
15	# sessions are completely unaffected by symlink flips. Only new sessions care
16	# where the symlink currently points.
17	#
18	# ==============================================================================================
19	# OPERATIONAL MODEL
20	# ==============================================================================================
21	#
22	# Three Modes (TRANSCODE_MANAGER_MODE)
23	#   smart   — auto-flips between ramdisk and SSD based on usage thresholds (default)
24	#             ramdisk above RAMDISK_WARN_GB → flip to SSD
25	#             ramdisk below RAMDISK_LOW_GB  → flip back to ramdisk
26	#   ramdisk — always uses ramdisk, warns if above threshold, never flips
27	#   ssd     — always uses SSD, never uses ramdisk (use during drain or maintenance)
28	#
29	# Safety Checks — Every Run Regardless of Mode
30	#   Symlink missing/broken    → recreate pointing at ramdisk, notify
31	#   Ramdisk disappeared       → flip to SSD immediately, notify warning
32	#   SSD path missing          → disable SSD fallback (error if mode=ssd)
33	#   transcoding-temp missing  → recreate on ramdisk silently
34	#   Permissions drift         → fix silently every run
35	#   Emby not running          → skip threshold checks, verify symlink only
36	#
37	# Session Display
38	#   Shows active streams from all configured TRANSCODE_SERVERS with user, title,
39	#   type (Live TV / TV Show / Movie), and play method (Transcode / Direct).
40	#   Split state shown when sessions exist on both ramdisk and SSD — normal during
41	#   a flip while ramdisk sessions drain.
42	#
43	# Daily Log
44	#   Appends one entry per run to TRANSCODE_DAILY_LOG, read by weekly_health_digest.sh.
45	#   Format: DATE|RAMDISK_USED_GB|FLIP_COUNT|RAM_SESSION_COUNT|SSD_SESSION_COUNT|FILES_CLEANED
46	#
47	# ==============================================================================================
48	# OPERATIONAL SAFEGUARDS
49	# ==============================================================================================
50	#
51	# Wait Lock
52	#   acquire_lock "wait" — waits if the previous run is still active. The 7-minute
53	#   interval can overlap on a system under heavy load.
54	#
55	# Docker Timeout
56	#   DOCKER_TIMEOUT caps all docker calls against a hung daemon.
57	#
58	# Notification Validated
59	#   platform_require_cmd confirms the notify script is present before use.
60	#
61	# Silent by Default
62	#   Runs every 7 minutes — only speaks when something changes or needs attention.
63	#
64	# ==============================================================================================
65	# CONFIGURATION
66	# ==============================================================================================
67	#
68	# host*.conf
69	#
70	#   HOST*_RAMDISK_PATH / HOST*_TRANSCODE_SSD
71	#       Ramdisk mount point and SSD fallback path.
72	#       Aliased by detect_hosts().
73	#
74	#   HOST*_RAMDISK_WARN_GB / HOST*_RAMDISK_LOW_GB / HOST*_RAMDISK_SIZE
75	#       Thresholds and ceiling. Change all three together.
76	#       Aliased by detect_hosts().
77	#
78	#   HOST*_TRANSCODE_SERVERS
79	#       Array of media server definitions: "ContainerName|URL|APIKey|Type"
80	#       Type: emby | jellyfin | plex
81	#       Aliased by detect_hosts() → TRANSCODE_SERVERS.
82	#
83	# master.conf
84	#
85	#   TRANSCODE_MANAGER_MODE
86	#       smart | ramdisk | ssd. (default: smart)
87	#
88	#   TRANSCODE_CHECK_EMBY
89	#       Skip threshold checks when Emby not running — prevents unnecessary flips
90	#       overnight when no sessions are active. (default: true)
91	#
92	#   TRANSCODE_FLIP_WARN
93	#       Notify if symlink flips this many times in one hour — indicates ramdisk
94	#       is undersized for the load. (default: 3)
95	#
96	# ==============================================================================================
97	# STATE FILES
98	# ==============================================================================================
99	#
100	#   /tmp/transcode_state.db    — current symlink target, flip count, last flip time


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Network Watchdog ===========================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Services-layer connectivity — checks that the outside world can actually reach
9	# what it needs to reach. Silent when everything is reachable. Only fires when
10	# something in the connectivity chain has broken.
11	#
12	# ==============================================================================================
13	# OPERATIONAL MODEL
14	# ==============================================================================================
15	#
16	# Check 1 — Internet Connectivity
17	#   curl external endpoint → fail = alert + skip all remaining checks.
18	#   Internet down means DDNS and NPM checks would false-positive — gating prevents noise.
19	#
20	# Check 2 — DDNS (Cloudflare)
21	#   Public IP via ifconfig.me vs DNS record via dig @1.1.1.1.
22	#   Match → pass (silent). Mismatch → restart DDNS container + notify.
23	#   Container restart triggers an immediate Cloudflare record update.
24	#
25	# Check 3 — Tailscale
26	#   tailscale status → Running → pass (silent). Not running → notify.
27	#   Notify only — no restart attempt. Tailscale state issues warrant human review.
28	#
29	# Check 4 — NPM Proxy (external check)
30	#   curl external URL → 2-strike system before restarting NginxProxyManager.
31	#   Strike 1: warn + notify. Strike 2: restart NPM + notify + clear strikes.
32	#   Strikes auto-clear when the external URL becomes reachable again.
33	#   External check — verifies the full stack (DNS → NPM → backend), not just NPM running.
34	#
35	# ==============================================================================================
36	# OPERATIONAL SAFEGUARDS
37	# ==============================================================================================
38	#
39	# Root Required
40	#   docker restart requires root.
41	#
42	# Single Instance Lock
43	#   acquire_lock prevents concurrent runs.
44	#
45	# Internet Gates All Checks
46	#   If internet is down, DDNS and NPM checks are skipped — no cascade of false positives.
47	#
48	# Strike Before Acting on NPM
49	#   Single curl failure could be transient DNS hiccup or CDN blip.
50	#   Two consecutive failures confirms NPM is the problem.
51	#
52	# ==============================================================================================
53	# CONFIGURATION
54	# ==============================================================================================
55	#
56	# master.conf
57	#
58	#   NETWORK_WATCHDOG_ENABLED          toggle entire watchdog (default: true)
59	#   NETWORK_WATCHDOG_INTERNET_URL     endpoint for internet reachability check
60	#   NETWORK_WATCHDOG_INTERNET_TIMEOUT curl timeout in seconds for internet check
61	#   NETWORK_WATCHDOG_CHECK_TAILSCALE  toggle tailscale check (default: true)
62	#   NETWORK_WATCHDOG_NPM_TIMEOUT      curl timeout for NPM external check
63	#   NETWORK_WATCHDOG_NPM_STRIKE_LIMIT consecutive failures before NPM restart
64	#   NETWORK_WATCHDOG_NPM_STATE_FILE   strike count persistence (/tmp — resets on reboot)
65	#
66	# host*.conf (host-specific)
67	#
68	#   HOST*_NETWORK_WATCHDOG_DDNS_DOMAIN     domain to resolve and compare to public IP
69	#   HOST*_NETWORK_WATCHDOG_DDNS_CONTAINER  container to restart on DDNS mismatch
70	#   HOST*_NETWORK_WATCHDOG_NPM_URL         external URL to test full proxy stack
71	#
72	# ==============================================================================================
73	# RUNTIME MODES
74	# ==============================================================================================
75	#
76	# network_watchdog.sh
77	#     Run all connectivity checks. Silent when healthy.
78	#
79	# network_watchdog.sh --dry-run
80	#     Run all checks without restarting any containers.
81	#
82	# network_watchdog.sh --status
83	#     Show configuration, current public IP, DNS record, NPM strike state.
84	#
85	# network_watchdog.sh --log
86	#     Verbose output — show each check result even when passing.
87	#
88	# ==============================================================================================
89
90	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
91
92	source "$SCRIPT_DIR/../../load_config.sh"
93
94	parse_args "$@"
95
96	# ==============================================================================================
97	# ━━━ Resolve host-specific config ━━━
98	# ==============================================================================================
99	_ddns_domain_var="${MY_ID}_NETWORK_WATCHDOG_DDNS_DOMAIN"
100	_ddns_container_var="${MY_ID}_NETWORK_WATCHDOG_DDNS_CONTAINER"


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Storage Watchdog ===========================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Pool and storage health monitoring — catches runaway data growth before it
9	# fills a pool. Runs as a single-pass script called by system_watchdog.sh each
10	# cycle. Sits between docker_watchdog.sh (container health) and
11	# stability_watchdog.sh (last line of defense). Never reboots — detects, alerts,
12	# and optionally remediates.
13	#
14	# ==============================================================================================
15	# OPERATIONAL MODEL
16	# ==============================================================================================
17	#
18	# Appdata Size Monitoring
19	#   Two complementary checks run every cycle:
20	#
21	#   Part 1 — Growth rate (zero-config catch-all):
22	#     Reads per-container dir totals via du, compares to previous cycle baseline.
23	#     Any container growing more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused
24	#     *.log scan inside that container's dir. No per-container config required —
25	#     new containers are covered automatically. Baseline built on first cycle after
26	#     boot; growth detection active from cycle 2.
27	#
28	#   Part 2 — Absolute log size:
29	#     Finds *.log / *.log.* files over WATCHDOG_APPDATA_LOG_MAX_GB anywhere in
30	#     WATCHDOG_APPDATA_PATHS. Catches logs already large but no longer actively
31	#     growing. Independent strike counter per file.
32	#
33	# Strike System
34	#   Reuses the same strike pattern as CPU/HTTP checks in docker_watchdog.sh:
35	#
36	#   Strike 1 — warn + notify: condition first detected this run
37	#   Strike 2 — warn + escalated notify: still present next cycle
38	#   Strike 3 (WATCHDOG_APPDATA_STRIKE_LIMIT) — action cycle:
39	#     WATCHDOG_APPDATA_TRUNCATE_LOGS=true  → truncate *.log in-place, clear strikes
40	#     WATCHDOG_APPDATA_TRUNCATE_LOGS=false → critical notify, hold strikes until resolved
41	#   Condition resolves (growth stops / log drops below threshold) → strikes auto-clear
42	#
43	# Suppress Ceiling (WATCHDOG_APPDATA_SIZES)
44	#   Containers in HOST*_WATCHDOG_APPDATA_SIZES suppress growth warnings while
45	#   under their configured ceiling MB. Use ONLY when a container legitimately
46	#   holds large stable data and would otherwise false-alarm (e.g. Tdarr cache).
47	#   Zero-config growth detection covers everything else automatically.
48	#
49	# ==============================================================================================
50	# DESIGN PRINCIPLES
51	# ==============================================================================================
52	#
53	# Zero-config for new containers
54	#   Growth rate detection requires no per-container configuration. Add a game
55	#   server, spin up a new arr, install anything — it is monitored automatically
56	#   from the second cycle after it appears. The suppress ceiling in conf is the
57	#   exception, not the rule.
58	#
59	# Alert-only for data, truncate-only for logs
60	#   Non-log growth (databases, game saves, caches) is detected and alerted but
61	#   never touched. Only *.log / *.log.* files are candidates for truncation —
62	#   and only when WATCHDOG_APPDATA_TRUNCATE_LOGS=true. Truncation zeroes the
63	#   file in-place; the container keeps its open file handle, space is reclaimed
64	#   immediately. Never deletes.
65	#
66	# Strike before acting
67	#   One cycle of growth could be a legitimate library scan or game save burst.
68	#   Three consecutive cycles of growth is a runaway. The strike system separates
69	#   transient activity from sustained problems before any action fires.
70	#
71	# Silent when healthy
72	#   Produces no output when all checks pass. Loud only when something needs
73	#   attention.
74	#
75	# ==============================================================================================
76	# CONFIGURATION
77	# ==============================================================================================
78	#
79	# master.conf
80	#
81	#   WATCHDOG_CHECK_APPDATA
82	#       Master toggle for all appdata checks (default: true)
83	#
84	#   WATCHDOG_APPDATA_PATHS
85	#       Array of paths to scan (e.g. "/mnt/docker-unraid/appdata")
86	#
87	#   WATCHDOG_APPDATA_GROWTH_GB
88	#       Per-cycle growth threshold in GB — flag containers growing more than this (default: 2)
89	#
90	#   WATCHDOG_APPDATA_LOG_MAX_GB
91	#       Absolute *.log file size alert threshold in GB (default: 2)
92	#
93	#   WATCHDOG_APPDATA_TRUNCATE_LOGS
94	#       Auto-truncate oversized *.log files on action cycle (default: false)
95	#
96	#   WATCHDOG_APPDATA_STRIKE_LIMIT
97	#       Consecutive cycles before action fires (default: 3)
98	#
99	#   WATCHDOG_APPDATA_GROWTH_FILE
100	#       Per-container size baseline — /tmp resets on reboot (correct: stale baseline