1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= conf_upgrade.sh ================================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Merges a new conf template into an existing user conf while preserving every
9	# value the user has already set. Run manually when the conf schema changes
10	# between versions — adds new keys, removes deprecated ones, and keeps the
11	# structure of the new template exactly.
12	#
13	#   Keys in template only  → ADDED   (placeholder/default — user fills in once)
14	#   Keys in target only    → REMOVED (deprecated in new version)
15	#   Keys in both           → KEPT    (target's value always wins, template ignored)
16	#   Comments / blank lines → always from template (structure follows new version)
17	#
18	# Supports all conf variable patterns: simple scalars, indexed arrays, and
19	# associative arrays (declare -A).
20	#
21	# ==============================================================================================
22	# OPERATIONAL SAFEGUARDS
23	# ==============================================================================================
24	#
25	# Dry-run Mode
26	#   --dry-run prints the full change report (ADDED / REMOVED / KEPT) then exits
27	#   without writing anything. Always preview before applying to production confs.
28	#
29	# Backup Option
30	#   --backup writes a .bak copy of the target before overwriting. Use when
31	#   applying to a conf that has never been upgraded before.
32	#
33	# File Existence Guards
34	#   Both --template and --target are validated before any parsing begins.
35	#   Missing files abort immediately with a clear error.
36	#
37	# Atomic Write
38	#   Merged output is written to a tempfile first, then copied to the target.
39	#   A partial write cannot corrupt the original.
40	#
41	# ==============================================================================================
42	# CONFIGURATION
43	# ==============================================================================================
44	#
45	# No conf vars. All inputs are CLI flags.
46	#
47	#   --template <file>   New version conf file (source of structure and defaults)
48	#   --target   <file>   Existing user conf (source of real values — always preserved)
49	#   --dry-run           Show what would change without writing
50	#   --backup            Write a .bak copy of target before modifying
51	#
52	# ==============================================================================================
53	# RUNTIME MODES
54	# ==============================================================================================
55	#
56	# conf_upgrade.sh --template master.conf.template --target Configurations/master.conf --dry-run
57	#     Preview what would be added, removed, and kept — no changes written.
58	#
59	# conf_upgrade.sh --template master.conf.template --target Configurations/master.conf --backup
60	#     Apply the upgrade, writing a .bak first.
61	#
62	# conf_upgrade.sh --template master.conf.template --target Configurations/master.conf
63	#     Apply the upgrade in-place with no backup.
64	#
65	# ==============================================================================================
66
67	set -uo pipefail
68
69	# ── Arguments ────────────────────────────────────────────────────────────────────────────────
70
71	TEMPLATE=""
72	TARGET=""
73	DRY_RUN=false
74	BACKUP=false
75
76	while [[ $# -gt 0 ]]; do
77	    case "$1" in
78	        --template) TEMPLATE="$2"; shift 2 ;;
79	        --target)   TARGET="$2";   shift 2 ;;
80	        --dry-run)  DRY_RUN=true;  shift   ;;
81	        --backup)   BACKUP=true;   shift   ;;
82	        *) echo "Unknown option: $1" >&2; exit 1 ;;
83	    esac
84	done
85
86	[[ -z "$TEMPLATE" ]] && { echo "Error: --template required" >&2; exit 1; }
87	[[ -z "$TARGET"   ]] && { echo "Error: --target required"   >&2; exit 1; }
88	[[ -f "$TEMPLATE" ]] || { echo "Error: template not found: $TEMPLATE" >&2; exit 1; }
89	[[ -f "$TARGET"   ]] || { echo "Error: target not found: $TARGET"     >&2; exit 1; }
90
91	# ── Parse target → KEY → full definition block ───────────────────────────────────────────────
92
93	declare -A HOST_MAP   # KEY → complete definition line(s) from user's conf
94
95	_parse_target() {
96	    local in_block=false cur_key="" cur_block="" line
97
98	    while IFS= read -r line || [[ -n "$line" ]]; do
99	        if [[ "$in_block" == true ]]; then
100	            cur_block+="$line"$'\n'
101	            # Closing ) — optional trailing whitespace and comment
102	            if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then
103	                HOST_MAP["$cur_key"]="$cur_block"
104	                in_block=false; cur_key=""; cur_block=""
105	            fi
106	        else
107	            # declare -A KEY=(
108	            if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then
109	                cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
110	            # KEY=(
111	            elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then
112	                cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
113	            # KEY=value  (simple scalar)
114	            elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then
115	                HOST_MAP["${BASH_REMATCH[1]}"]="$line"$'\n'
116	            fi
117	        fi
118	    done < "$TARGET"
119	}
120
121	# ── Walk template — collect stats (must run in current shell so arrays persist) ──────────────
122
123	declare -a ADDED=() KEPT=() REMOVED=()
124	declare -A TMPL_SEEN=()
125
126	_collect_stats() {
127	    local in_block=false cur_key="" line
128
129	    while IFS= read -r line || [[ -n "$line" ]]; do
130	        if [[ "$in_block" == true ]]; then
131	            if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then
132	                in_block=false
133	                TMPL_SEEN["$cur_key"]=1
134	                if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then KEPT+=("$cur_key")
135	                else                                         ADDED+=("$cur_key"); fi
136	                cur_key=""
137	            fi
138	        else
139	            if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then
140	                cur_key="${BASH_REMATCH[1]}"; in_block=true
141	            elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then
142	                cur_key="${BASH_REMATCH[1]}"; in_block=true
143	            elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then
144	                local k="${BASH_REMATCH[1]}"
145	                TMPL_SEEN["$k"]=1
146	                if [[ -n "${HOST_MAP[$k]+_}" ]]; then KEPT+=("$k")
147	                else                                   ADDED+=("$k"); fi
148	            fi
149	        fi
150	    done < "$TEMPLATE"
151
152	    for key in "${!HOST_MAP[@]}"; do
153	        [[ -z "${TMPL_SEEN[$key]+_}" ]] && REMOVED+=("$key")
154	    done
155	}
156
157	# ── Walk template — write merged output ──────────────────────────────────────────────────────
158	# Runs in a subshell (stdout redirected) — array mutations are intentionally discarded here.
159
160	_write_merged() {
161	    local in_block=false cur_key="" cur_block="" line
162
163	    while IFS= read -r line || [[ -n "$line" ]]; do
164	        if [[ "$in_block" == true ]]; then
165	            cur_block+="$line"$'\n'
166	            if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then
167	                in_block=false
168	                if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then printf '%s' "${HOST_MAP[$cur_key]}"
169	                else                                        printf '%s' "$cur_block"; fi
170	                cur_key=""; cur_block=""
171	            fi
172	        else
173	            if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then
174	                cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
175	            elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then
176	                cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
177	            elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then
178	                local k="${BASH_REMATCH[1]}"
179	                if [[ -n "${HOST_MAP[$k]+_}" ]]; then printf '%s' "${HOST_MAP[$k]}"
180	                else                                  printf '%s\n' "$line"; fi
181	            else
182	                printf '%s\n' "$line"
183	            fi
184	        fi
185	    done < "$TEMPLATE"
186	}
187
188	# ── Run ───────────────────────────────────────────────────────────────────────────────────────
189
190	_parse_target
191	_collect_stats
192
193	# ── Report ────────────────────────────────────────────────────────────────────────────────────
194
195	TARGET_NAME="$(basename "$TARGET")"
196	echo ""
197	echo "── conf_upgrade: $TARGET_NAME ──────────────────────────────────────────"
198
199	if [[ ${#ADDED[@]} -gt 0 ]]; then
200	    echo "  ADDED   (new — fill in your values where needed):"
201	    for k in "${ADDED[@]}"; do echo "    + $k"; done
202	fi
203
204	if [[ ${#REMOVED[@]} -gt 0 ]]; then
205	    echo "  REMOVED (deprecated — no longer in this version):"
206	    for k in "${REMOVED[@]}"; do echo "    - $k"; done
207	fi
208
209	echo "  KEPT    ${#KEPT[@]} existing vars — your values preserved"
210
211	if [[ ${#ADDED[@]} -eq 0 && ${#REMOVED[@]} -eq 0 ]]; then
212	    echo "  Already up to date — no changes needed."
213	    echo "────────────────────────────────────────────────────────────────────────"
214	    echo ""
215	    exit 0
216	fi
217
218	echo "────────────────────────────────────────────────────────────────────────"
219	echo ""
220
221	# ── Apply ─────────────────────────────────────────────────────────────────────────────────────
222
223	if [[ "$DRY_RUN" == true ]]; then
224	    echo "(dry-run — no changes written)"
225	    exit 0
226	fi
227
228	TMPOUT="$(mktemp)"
229	trap 'rm -f "$TMPOUT"' EXIT
230
231	_write_merged > "$TMPOUT"
232
233	if [[ "$BACKUP" == true ]]; then
234	    cp "$TARGET" "${TARGET}.bak"
235	    echo "Backup: ${TARGET}.bak"
236	fi
237
238	cp "$TMPOUT" "$TARGET"
239	echo "Updated: $TARGET"
240

#!/bin/bash
# ==============================================================================================
# ============================= conf_upgrade.sh ================================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Merges a new conf template into an existing user conf while preserving every
# value the user has already set. Run manually when the conf schema changes
# between versions — adds new keys, removes deprecated ones, and keeps the
# structure of the new template exactly.
#
#   Keys in template only  → ADDED   (placeholder/default — user fills in once)
#   Keys in target only    → REMOVED (deprecated in new version)
#   Keys in both           → KEPT    (target's value always wins, template ignored)
#   Comments / blank lines → always from template (structure follows new version)
#
# Supports all conf variable patterns: simple scalars, indexed arrays, and
# associative arrays (declare -A).
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Dry-run Mode
#   --dry-run prints the full change report (ADDED / REMOVED / KEPT) then exits
#   without writing anything. Always preview before applying to production confs.
#
# Backup Option
#   --backup writes a .bak copy of the target before overwriting. Use when
#   applying to a conf that has never been upgraded before.
#
# File Existence Guards
#   Both --template and --target are validated before any parsing begins.
#   Missing files abort immediately with a clear error.
#
# Atomic Write
#   Merged output is written to a tempfile first, then copied to the target.
#   A partial write cannot corrupt the original.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# No conf vars. All inputs are CLI flags.
#
#   --template <file>   New version conf file (source of structure and defaults)
#   --target   <file>   Existing user conf (source of real values — always preserved)
#   --dry-run           Show what would change without writing
#   --backup            Write a .bak copy of target before modifying
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# conf_upgrade.sh --template master.conf.template --target Configurations/master.conf --dry-run
#     Preview what would be added, removed, and kept — no changes written.
#
# conf_upgrade.sh --template master.conf.template --target Configurations/master.conf --backup
#     Apply the upgrade, writing a .bak first.
#
# conf_upgrade.sh --template master.conf.template --target Configurations/master.conf
#     Apply the upgrade in-place with no backup.
#
# ==============================================================================================

set -uo pipefail

# ── Arguments ────────────────────────────────────────────────────────────────────────────────

TEMPLATE=""
TARGET=""
DRY_RUN=false
BACKUP=false

while [[ $# -gt 0 ]]; do
    case "$1" in
        --template) TEMPLATE="$2"; shift 2 ;;
        --target)   TARGET="$2";   shift 2 ;;
        --dry-run)  DRY_RUN=true;  shift   ;;
        --backup)   BACKUP=true;   shift   ;;
        *) echo "Unknown option: $1" >&2; exit 1 ;;
    esac
done

[[ -z "$TEMPLATE" ]] && { echo "Error: --template required" >&2; exit 1; }
[[ -z "$TARGET"   ]] && { echo "Error: --target required"   >&2; exit 1; }
[[ -f "$TEMPLATE" ]] || { echo "Error: template not found: $TEMPLATE" >&2; exit 1; }
[[ -f "$TARGET"   ]] || { echo "Error: target not found: $TARGET"     >&2; exit 1; }

# ── Parse target → KEY → full definition block ───────────────────────────────────────────────

declare -A HOST_MAP   # KEY → complete definition line(s) from user's conf

_parse_target() {
    local in_block=false cur_key="" cur_block="" line

    while IFS= read -r line || [[ -n "$line" ]]; do
        if [[ "$in_block" == true ]]; then
            cur_block+="$line"$'\n'
            # Closing ) — optional trailing whitespace and comment
            if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then
                HOST_MAP["$cur_key"]="$cur_block"
                in_block=false; cur_key=""; cur_block=""
            fi
        else
            # declare -A KEY=(
            if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then
                cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
            # KEY=(
            elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then
                cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
            # KEY=value  (simple scalar)
            elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then
                HOST_MAP["${BASH_REMATCH[1]}"]="$line"$'\n'
            fi
        fi
    done < "$TARGET"
}

# ── Walk template — collect stats (must run in current shell so arrays persist) ──────────────

declare -a ADDED=() KEPT=() REMOVED=()
declare -A TMPL_SEEN=()

_collect_stats() {
    local in_block=false cur_key="" line

    while IFS= read -r line || [[ -n "$line" ]]; do
        if [[ "$in_block" == true ]]; then
            if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then
                in_block=false
                TMPL_SEEN["$cur_key"]=1
                if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then KEPT+=("$cur_key")
                else                                         ADDED+=("$cur_key"); fi
                cur_key=""
            fi
        else
            if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then
                cur_key="${BASH_REMATCH[1]}"; in_block=true
            elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then
                cur_key="${BASH_REMATCH[1]}"; in_block=true
            elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then
                local k="${BASH_REMATCH[1]}"
                TMPL_SEEN["$k"]=1
                if [[ -n "${HOST_MAP[$k]+_}" ]]; then KEPT+=("$k")
                else                                   ADDED+=("$k"); fi
            fi
        fi
    done < "$TEMPLATE"

    for key in "${!HOST_MAP[@]}"; do
        [[ -z "${TMPL_SEEN[$key]+_}" ]] && REMOVED+=("$key")
    done
}

# ── Walk template — write merged output ──────────────────────────────────────────────────────
# Runs in a subshell (stdout redirected) — array mutations are intentionally discarded here.

_write_merged() {
    local in_block=false cur_key="" cur_block="" line

    while IFS= read -r line || [[ -n "$line" ]]; do
        if [[ "$in_block" == true ]]; then
            cur_block+="$line"$'\n'
            if [[ "$line" =~ ^[[:space:]]*\)[[:space:]]*(#.*)?$ ]]; then
                in_block=false
                if [[ -n "${HOST_MAP[$cur_key]+_}" ]]; then printf '%s' "${HOST_MAP[$cur_key]}"
                else                                        printf '%s' "$cur_block"; fi
                cur_key=""; cur_block=""
            fi
        else
            if [[ "$line" =~ ^[[:space:]]*declare[[:space:]]+-[a-zA-Z]+[[:space:]]+([A-Z0-9_]+)[[:space:]]*=\( ]]; then
                cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
            elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*=\( ]]; then
                cur_key="${BASH_REMATCH[1]}"; in_block=true; cur_block="$line"$'\n'
            elif [[ "$line" =~ ^[[:space:]]*([A-Z0-9_]+)[[:space:]]*= ]]; then
                local k="${BASH_REMATCH[1]}"
                if [[ -n "${HOST_MAP[$k]+_}" ]]; then printf '%s' "${HOST_MAP[$k]}"
                else                                  printf '%s\n' "$line"; fi
            else
                printf '%s\n' "$line"
            fi
        fi
    done < "$TEMPLATE"
}

# ── Run ───────────────────────────────────────────────────────────────────────────────────────

_parse_target
_collect_stats

# ── Report ────────────────────────────────────────────────────────────────────────────────────

TARGET_NAME="$(basename "$TARGET")"
echo ""
echo "── conf_upgrade: $TARGET_NAME ──────────────────────────────────────────"

if [[ ${#ADDED[@]} -gt 0 ]]; then
    echo "  ADDED   (new — fill in your values where needed):"
    for k in "${ADDED[@]}"; do echo "    + $k"; done
fi

if [[ ${#REMOVED[@]} -gt 0 ]]; then
    echo "  REMOVED (deprecated — no longer in this version):"
    for k in "${REMOVED[@]}"; do echo "    - $k"; done
fi

echo "  KEPT    ${#KEPT[@]} existing vars — your values preserved"

if [[ ${#ADDED[@]} -eq 0 && ${#REMOVED[@]} -eq 0 ]]; then
    echo "  Already up to date — no changes needed."
    echo "────────────────────────────────────────────────────────────────────────"
    echo ""
    exit 0
fi

echo "────────────────────────────────────────────────────────────────────────"
echo ""

# ── Apply ─────────────────────────────────────────────────────────────────────────────────────

if [[ "$DRY_RUN" == true ]]; then
    echo "(dry-run — no changes written)"
    exit 0
fi

TMPOUT="$(mktemp)"
trap 'rm -f "$TMPOUT"' EXIT

_write_merged > "$TMPOUT"

if [[ "$BACKUP" == true ]]; then
    cp "$TARGET" "${TARGET}.bak"
    echo "Backup: ${TARGET}.bak"
fi

cp "$TMPOUT" "$TARGET"
echo "Updated: $TARGET"


Bash

ls /boot/config/plugins/varaverk/Configurations/ 2>/dev/null || echo "No Configurations dir"

List Configurations directory


host.conf.template
host1.conf
host1.conf.bak
host2.conf
master.conf
master.conf.bak
master.conf.template


find /boot/config/plugins/varaverk/Orchestrators -type f | sort

List all orchestrator scripts


lets double check our onboarding setup, and first launch setup... we may test it out on the second server and start setting it up


Review onboarding and first launch setup


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Array Start Orchestrator ===================================
4	# ==============================================================================================
5	# Single entry point for array start — fired by the Varaverk plugin's
6	# disks_mounted event hook (Plugin/unraid/event/disks_mounted/array_start_jobs).
7	# Launches everything configured in ARRAY_START_SCRIPTS in master.conf.
8	# This script exits after launching all scripts — the event hook sees it complete normally.
9	#
10	# ── WHAT IT LAUNCHES ──────────────────────────────────────────────────────────────────────────
11	# Configured in master.conf ARRAY_START_SCRIPTS — no changes to this script ever needed.
12	# Current order (order matters — see below):
13	#
14	#   ONE-SHOT (run and exit naturally):
15	#     System_Essentials/unraid_api_key_renew.sh — re-register Varaverk API key at boot
16	#     System_Essentials/inotify_tuning.sh       — raise inotify limits before containers start
17	#     System_Essentials/docker_syslog_filter.sh — suppress veth log noise before logs fill
18	#     System_Essentials/php_fpm_max_children.sh — WebGUI performance tuning
19	#     Transcodes/ramdisk_setup.sh               — create tmpfs + symlink before Emby starts
20	#     Docker_Essentials/docker_network_connect.sh — ensure networks + container connections
21	#
22	#   CONTINUOUS (run until array stops):
23	#     Fallback/fallback.sh                      — mutual fallback monitor
24	#
25	#   NOTE: watchdogs (docker, system, stability) are NOT launched here.
26	#   They run via watchdog_orchestrator.sh every 15 min (cron), not as daemons.
27	#
28	# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
29	#   unraid_api_key_renew.sh — before anything else — self-heals API registry at boot
30	#   inotify_tuning.sh     — must run BEFORE Code-Server and other containers start
31	#                           containers that start with low inotify limits keep them ✅
32	#   docker_syslog_filter  — must run BEFORE any container starts creating veth interfaces
33	#   ramdisk_setup.sh      — must run BEFORE Emby starts transcoding
34	#   docker_network_connect — must run BEFORE watchdogs check container states
35	#   fallback.sh           — last — needs everything else stable to make decisions
36	#
37	# ── ONE-SHOT vs CONTINUOUS DETECTION ─────────────────────────────────────────────────────────
38	# Script is launched in background with bash script.sh &
39	# After 1 second: if PID still alive → continuous (running in background)
40	#                  if PID dead + exit 0 → one-shot completed successfully
41	#                  if PID dead + exit N → failure
42	#
43	# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
44	#   Root check           — all launched scripts require root
45	#   acquire_lock         — prevents duplicate array start launches
46	#   detect_hosts()       — MY_ID in notifications
47	#   platform_require_cmd  — notify validated before use
48	#   chmod +x auto-fix    — non-executable scripts fixed before launch
49	#   Full path on failure — shows exact path for debugging
50	#   notify on failures   — alert if any script fails to launch
51	#
52	# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
53	#   ARRAY_START_SCRIPTS — ordered list of scripts to launch at array start
54	#
55	# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
56	#   array_started.sh              — normal launch (called by Varaverk disks_mounted event hook)
57	#   array_started.sh --dry-run    — show what would be launched without launching
58	#   array_started.sh --status     — show configured scripts and their current state
59	#   array_started.sh --log        — verbose output per script
60	# ==============================================================================================
61
62	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
63	ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
64
65	source "$ECOSYSTEM_ROOT/load_config.sh"
66
67	parse_args "$@"
68
69	# ==============================================================================================
70	# ━━━ Setup ━━━
71	# ==============================================================================================
72	if [[ "$EUID" -ne 0 ]]; then
73	    error "Must be run as root"
74	    exit 1
75	fi
76
77	platform_require_cmd \
78	    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
79	    "" "" \
80	    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
81
82	acquire_lock
83
84	detect_hosts
85
86	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — scripts will not be launched"
87
88	# ==============================================================================================
89	# ━━━ Status ━━━
90	# ==============================================================================================
91	if [[ "$SHOW_STATUS" == true ]]; then
92	    echo ""
93	    echo "━━━━━ $ICON_SUMMARY ARRAY START STATUS ━━━━━"
94	    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
95	    echo "$ICON_GEAR  Scripts:  ${#ARRAY_START_SCRIPTS[@]} configured"
96	    echo ""
97
98	    for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
99	        [[ -z "$relative_path" ]] && continue
100	        script_path="$ECOSYSTEM_ROOT/$relative_path"
101	        script_name=$(basename "$script_path")
102
103	        if [[ ! -f "$script_path" ]]; then
104	            echo "  $ICON_ERROR $script_name — FILE NOT FOUND"
105	            echo "    $script_path"
106	            continue
107	        fi
108
109	        [[ ! -x "$script_path" ]] && flag=" (not executable — will auto-fix)" || flag=""
110
111	        # Check if currently running
112	        if pgrep -f "$script_path" >/dev/null 2>&1; then
113	            RUN_PID=$(pgrep -f "$script_path" | head -1)
114	            echo "  $ICON_RUNNING $script_name — RUNNING (PID $RUN_PID)${flag}"
115	        else
116	            echo "  $ICON_NOT_RUNNING $script_name — not running${flag}"
117	        fi
118	    done
119	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
120	    exit 0
121	fi
122
123	# ==============================================================================================
124	# ━━━ Launch Scripts ━━━
125	# ==============================================================================================
126	if [[ ${#ARRAY_START_SCRIPTS[@]} -eq 0 ]]; then
127	    notify "Array started on $LOCAL_SERVER_NAME ($MY_ID) but ARRAY_START_SCRIPTS is empty — boot sequence skipped. Check master.conf." \
128	        "Array Start" "alert"
129	    error "ARRAY_START_SCRIPTS is empty — check master.conf"
130	    exit 1
131	fi
132
133	echo ""
134	echo "━━━ $ICON_GEAR Array Start — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
135	log "Ecosystem root: $ECOSYSTEM_ROOT"
136	log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..."
137	echo ""
138
139	START=$(date +%s)
140	LAUNCHED=0
141	FAILED=0
142	FAILED_SCRIPTS=()
143
144	for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
145	    [[ -z "$relative_path" ]] && continue
146
147	    SCRIPT_PATH="$ECOSYSTEM_ROOT/$relative_path"
148	    SCRIPT_NAME=$(basename "$SCRIPT_PATH")
149
150	    # File existence check
151	    if [[ ! -f "$SCRIPT_PATH" ]]; then
152	        error "$SCRIPT_NAME — not found"
153	        error "  Expected: $SCRIPT_PATH"
154	        (( FAILED++ ))
155	        FAILED_SCRIPTS+=("$SCRIPT_NAME")
156	        continue
157	    fi
158
159	    # Auto-fix permissions — chmod +x if needed
160	    if [[ ! -x "$SCRIPT_PATH" ]]; then
161	        warn "$SCRIPT_NAME — not executable, fixing..."
162	        chmod +x "$SCRIPT_PATH" || {
163	            error "$SCRIPT_NAME — chmod +x failed"
164	            (( FAILED++ ))
165	            FAILED_SCRIPTS+=("$SCRIPT_NAME")
166	            continue
167	        }
168	    fi
169
170	    if [[ "$DRY_RUN" == true ]]; then
171	        warn "DRY RUN — would launch: $SCRIPT_NAME"
172	        (( LAUNCHED++ ))
173	        continue
174	    fi
175
176	    log "$ICON_START Launching $SCRIPT_NAME..."
177	    bash "$SCRIPT_PATH" &
178	    PID=$!
179
180	    # Brief settle — 1s enough to detect immediate failures
181	    sleep 1
182
183	    if kill -0 "$PID" 2>/dev/null; then
184	        # Still running → continuous script
185	        warn "$SCRIPT_NAME — running (PID $PID) ✅"
186	        (( LAUNCHED++ ))
187	    else
188	        # Exited — check if one-shot success or failure
189	        wait "$PID"
190	        EXIT_CODE=$?
191	        if [[ "$EXIT_CODE" -eq 0 ]]; then
192	            log "$SCRIPT_NAME — completed (one-shot) ✅"
193	            (( LAUNCHED++ ))
194	        else
195	            error "$SCRIPT_NAME — exited with code $EXIT_CODE"
196	            error "  Path: $SCRIPT_PATH"
197	            (( FAILED++ ))
198	            FAILED_SCRIPTS+=("$SCRIPT_NAME")
199	        fi
200	    fi
201
202	done
203
204	END=$(date +%s)
205
206	# ==============================================================================================
207	# ━━━ Summary ━━━
208	# ==============================================================================================
209	echo ""
210	echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
211	echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
212	echo "$ICON_SUCCESS Launched: $LAUNCHED"
213	[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR  Failed:   $FAILED — ${FAILED_SCRIPTS[*]}"
214	echo "$ICON_TIME    Duration: $(format_duration $(( END - START )))"
215	echo ""
216
217	if [[ "$DRY_RUN" == true ]]; then
218	    warn "DRY RUN — no scripts launched"
219	elif [[ "$FAILED" -gt 0 ]]; then
220	    warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}"
221	    notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
222	        "Array Start" "warning"
223	else
224	    echo "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
225	fi
226	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

#!/bin/bash
# ==============================================================================================
# ================================= Array Start Orchestrator ===================================
# ==============================================================================================
# Single entry point for array start — fired by the Varaverk plugin's
# disks_mounted event hook (Plugin/unraid/event/disks_mounted/array_start_jobs).
# Launches everything configured in ARRAY_START_SCRIPTS in master.conf.
# This script exits after launching all scripts — the event hook sees it complete normally.
#
# ── WHAT IT LAUNCHES ──────────────────────────────────────────────────────────────────────────
# Configured in master.conf ARRAY_START_SCRIPTS — no changes to this script ever needed.
# Current order (order matters — see below):
#
#   ONE-SHOT (run and exit naturally):
#     System_Essentials/unraid_api_key_renew.sh — re-register Varaverk API key at boot
#     System_Essentials/inotify_tuning.sh       — raise inotify limits before containers start
#     System_Essentials/docker_syslog_filter.sh — suppress veth log noise before logs fill
#     System_Essentials/php_fpm_max_children.sh — WebGUI performance tuning
#     Transcodes/ramdisk_setup.sh               — create tmpfs + symlink before Emby starts
#     Docker_Essentials/docker_network_connect.sh — ensure networks + container connections
#
#   CONTINUOUS (run until array stops):
#     Fallback/fallback.sh                      — mutual fallback monitor
#
#   NOTE: watchdogs (docker, system, stability) are NOT launched here.
#   They run via watchdog_orchestrator.sh every 15 min (cron), not as daemons.
#
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
#   unraid_api_key_renew.sh — before anything else — self-heals API registry at boot
#   inotify_tuning.sh     — must run BEFORE Code-Server and other containers start
#                           containers that start with low inotify limits keep them ✅
#   docker_syslog_filter  — must run BEFORE any container starts creating veth interfaces
#   ramdisk_setup.sh      — must run BEFORE Emby starts transcoding
#   docker_network_connect — must run BEFORE watchdogs check container states
#   fallback.sh           — last — needs everything else stable to make decisions
#
# ── ONE-SHOT vs CONTINUOUS DETECTION ─────────────────────────────────────────────────────────
# Script is launched in background with bash script.sh &
# After 1 second: if PID still alive → continuous (running in background)
#                  if PID dead + exit 0 → one-shot completed successfully
#                  if PID dead + exit N → failure
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
#   Root check           — all launched scripts require root
#   acquire_lock         — prevents duplicate array start launches
#   detect_hosts()       — MY_ID in notifications
#   platform_require_cmd  — notify validated before use
#   chmod +x auto-fix    — non-executable scripts fixed before launch
#   Full path on failure — shows exact path for debugging
#   notify on failures   — alert if any script fails to launch
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   ARRAY_START_SCRIPTS — ordered list of scripts to launch at array start
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   array_started.sh              — normal launch (called by Varaverk disks_mounted event hook)
#   array_started.sh --dry-run    — show what would be launched without launching
#   array_started.sh --status     — show configured scripts and their current state
#   array_started.sh --log        — verbose output per script
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

source "$ECOSYSTEM_ROOT/load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

platform_require_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock

detect_hosts

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — scripts will not be launched"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY ARRAY START STATUS ━━━━━"
    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR  Scripts:  ${#ARRAY_START_SCRIPTS[@]} configured"
    echo ""

    for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
        [[ -z "$relative_path" ]] && continue
        script_path="$ECOSYSTEM_ROOT/$relative_path"
        script_name=$(basename "$script_path")

        if [[ ! -f "$script_path" ]]; then
            echo "  $ICON_ERROR $script_name — FILE NOT FOUND"
            echo "    $script_path"
            continue
        fi

        [[ ! -x "$script_path" ]] && flag=" (not executable — will auto-fix)" || flag=""

        # Check if currently running
        if pgrep -f "$script_path" >/dev/null 2>&1; then
            RUN_PID=$(pgrep -f "$script_path" | head -1)
            echo "  $ICON_RUNNING $script_name — RUNNING (PID $RUN_PID)${flag}"
        else
            echo "  $ICON_NOT_RUNNING $script_name — not running${flag}"
        fi
    done
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Launch Scripts ━━━
# ==============================================================================================
if [[ ${#ARRAY_START_SCRIPTS[@]} -eq 0 ]]; then
    notify "Array started on $LOCAL_SERVER_NAME ($MY_ID) but ARRAY_START_SCRIPTS is empty — boot sequence skipped. Check master.conf." \
        "Array Start" "alert"
    error "ARRAY_START_SCRIPTS is empty — check master.conf"
    exit 1
fi

echo ""
echo "━━━ $ICON_GEAR Array Start — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
log "Ecosystem root: $ECOSYSTEM_ROOT"
log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..."
echo ""

START=$(date +%s)
LAUNCHED=0
FAILED=0
FAILED_SCRIPTS=()

for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
    [[ -z "$relative_path" ]] && continue

    SCRIPT_PATH="$ECOSYSTEM_ROOT/$relative_path"
    SCRIPT_NAME=$(basename "$SCRIPT_PATH")

    # File existence check
    if [[ ! -f "$SCRIPT_PATH" ]]; then
        error "$SCRIPT_NAME — not found"
        error "  Expected: $SCRIPT_PATH"
        (( FAILED++ ))
        FAILED_SCRIPTS+=("$SCRIPT_NAME")
        continue
    fi

    # Auto-fix permissions — chmod +x if needed
    if [[ ! -x "$SCRIPT_PATH" ]]; then
        warn "$SCRIPT_NAME — not executable, fixing..."
        chmod +x "$SCRIPT_PATH" || {
            error "$SCRIPT_NAME — chmod +x failed"
            (( FAILED++ ))
            FAILED_SCRIPTS+=("$SCRIPT_NAME")
            continue
        }
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would launch: $SCRIPT_NAME"
        (( LAUNCHED++ ))
        continue
    fi

    log "$ICON_START Launching $SCRIPT_NAME..."
    bash "$SCRIPT_PATH" &
    PID=$!

    # Brief settle — 1s enough to detect immediate failures
    sleep 1

    if kill -0 "$PID" 2>/dev/null; then
        # Still running → continuous script
        warn "$SCRIPT_NAME — running (PID $PID) ✅"
        (( LAUNCHED++ ))
    else
        # Exited — check if one-shot success or failure
        wait "$PID"
        EXIT_CODE=$?
        if [[ "$EXIT_CODE" -eq 0 ]]; then
            log "$SCRIPT_NAME — completed (one-shot) ✅"
            (( LAUNCHED++ ))
        else
            error "$SCRIPT_NAME — exited with code $EXIT_CODE"
            error "  Path: $SCRIPT_PATH"
            (( FAILED++ ))
            FAILED_SCRIPTS+=("$SCRIPT_NAME")
        fi
    fi

done

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_SUCCESS Launched: $LAUNCHED"
[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR  Failed:   $FAILED — ${FAILED_SCRIPTS[*]}"
echo "$ICON_TIME    Duration: $(format_duration $(( END - START )))"
echo ""

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no scripts launched"
elif [[ "$FAILED" -gt 0 ]]; then
    warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}"
    notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
        "Array Start" "warning"
else
    echo "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"