1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Unraid API Key Renewal ====================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Creates/overwrites the Varaverk API key in the unraid-api service registry at
9	# array start. The registry is ephemeral — OS updates and service restarts clear
10	# it. This script re-registers the key every boot so Varaverk's enhanced
11	# monitoring self-heals without manual intervention.
12	#
13	# Also updates HOST*_UNRAID_API_KEY in the local host conf so the partnership
14	# page always reflects the live key value.
15	#
16	# ==============================================================================================
17	# RUNTIME MODES
18	# ==============================================================================================
19	#
20	# unraid_api_key_renew.sh
21	#     Renew the key. Silent on success.
22	#
23	# unraid_api_key_renew.sh --dry-run
24	#     Show what would happen — no changes made.
25	#
26	# unraid_api_key_renew.sh --log
27	#     Verbose output.
28	#
29	# ==============================================================================================
30
31	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
32
33	source "$SCRIPT_DIR/../../../load_config.sh"
34
35	parse_args "$@"
36	acquire_lock
37	detect_hosts
38
39	# ──────────────────────────────────────────────────────────────────────────────
40	CONF_FILE="$SCRIPT_DIR/../Configurations/${MY_ID,,}.conf"
41	VAR_NAME="${MY_ID}_UNRAID_API_KEY"
42
43	# Key name: "Varaverk <hostname>" stripping any unraid- prefix
44	# Space separator — unRAID API only allows letters, numbers, and spaces
45	HOSTNAME_SUFFIX=$(hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//' || hostname -s)
46	KEY_NAME="Varaverk ${HOSTNAME_SUFFIX}"
47
48	log "$ICON_GEAR Conf file: $CONF_FILE"
49	log "$ICON_GEAR Key var:   $VAR_NAME"
50	log "$ICON_GEAR Key name:  $KEY_NAME"
51
52	if [[ ! -f "$CONF_FILE" ]]; then
53	    error "Conf file not found: $CONF_FILE"
54	    exit 1
55	fi
56
57	if [[ "$DRY_RUN" == true ]]; then
58	    warn "DRY RUN — would check registry for $KEY_NAME, renew only if missing"
59	    exit 0
60	fi
61
62	# ──────────────────────────────────────────────────────────────────────────────
63	# Check if key already exists in the unraid-api registry before creating.
64	# --overwrite generates a new key value every time, invalidating the old one.
65	# Only renew if the registry has lost it.
66	log "Checking unraid-api registry for $KEY_NAME..."
67	EXISTING=$(timeout 5 /usr/local/sbin/unraid-api apikey --name "$KEY_NAME" --json </dev/null 2>/dev/null)
68	KEY=$(echo "$EXISTING" | jq -r '.key // empty' 2>/dev/null)
69
70	if [[ -n "$KEY" ]]; then
71	    PREVIEW="${KEY:0:8}...${KEY: -4}"
72	    echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
73	    log "Key found in registry — no renewal needed"
74	    exit 0
75	fi
76
77	log "Key not found in registry — creating new key..."
78
79	RAW=$(timeout 10 /usr/local/sbin/unraid-api apikey \
80	    --name "$KEY_NAME" --create --overwrite \
81	    --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1)
82
83	if [[ -z "$RAW" ]]; then
84	    error "unraid-api returned no output"
85	    exit 1
86	fi
87
88	KEY=$(echo "$RAW" | jq -r '.key // empty' 2>/dev/null)
89	if [[ -z "$KEY" ]]; then
90	    error "No key in unraid-api response: ${RAW:0:200}"
91	    exit 1
92	fi
93
94	# ──────────────────────────────────────────────────────────────────────────────
95	if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
96	    sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
97	else
98	    echo "    ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
99	fi
100
101	PREVIEW="${KEY:0:8}...${KEY: -4}"
102	log "Writing new key to: $CONF_FILE"
103	warn "API key renewed ✅ — $VAR_NAME = $PREVIEW  (registry had lost it)"
104
105	# ── Push renewed key into each partner's OWN conf ─────────────────────────────
106	# Each host's conf is its complete keychest — no cross-host conf files needed.
107	# SSH_KEY is set by detect_hosts() — this server's outbound private key.
108	if [[ -z "$SSH_KEY" ]]; then
109	    log "No SSH key configured — skipping partner push"
110	    exit 0
111	fi
112
113	for host_var in $(compgen -v | grep -E '^HOST[0-9]+$'); do
114	    partner_host="${!host_var}"
115	    [[ -z "$partner_host" ]] && continue
116	    [[ "${host_var,,}" == "${MY_ID,,}" ]] && continue
117
118	    partner_slot="${host_var,,}"   # e.g. host2
119	    partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
120	    [[ -z "$partner_ip" ]] && { log "Cannot resolve IP for $partner_host — skipping"; continue; }
121
122	    # Target is the partner's OWN conf on their machine
123	    partner_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
124	    tmp=$(mktemp /tmp/vv_kp_XXXXXX.sh)
125	    remote="/tmp/vv_kp_${RANDOM}.sh"
126	    chmod 700 "$tmp"
127
128	    # Key stays in the temp file — never appears in SSH command args
129	    cat > "$tmp" <<PUSHSCRIPT
130	#!/bin/sh
131	target='${partner_conf}'
132	if grep -q "\b${VAR_NAME}\b" "\$target" 2>/dev/null; then
133	  sed -i 's|^\(\\s*${VAR_NAME}\\s*=\\s*\)"[^"]*"|\1"${KEY}"|' "\$target"
134	else
135	  printf '    ${VAR_NAME}="%s"\n' '${KEY}' >> "\$target"
136	fi
137	echo ok
138	PUSHSCRIPT
139
140	    if timeout 10 scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
141	        -o StrictHostKeyChecking=no "$tmp" "root@${partner_ip}:${remote}" 2>/dev/null; then
142	        if timeout 10 ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
143	            -o StrictHostKeyChecking=no "root@${partner_ip}" \
144	            "bash '${remote}'; rc=\$?; rm -f '${remote}'; exit \$rc" 2>/dev/null | grep -q ok; then
145	            log "Key pushed to $partner_host ✅"
146	        else
147	            warn "Key push to $partner_host failed — they can create their own copy"
148	        fi
149	    else
150	        warn "SCP to $partner_host failed — skipping"
151	    fi
152	    rm -f "$tmp"
153	done
154

#!/bin/bash
# ==============================================================================================
# ================================= Unraid API Key Renewal ====================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Creates/overwrites the Varaverk API key in the unraid-api service registry at
# array start. The registry is ephemeral — OS updates and service restarts clear
# it. This script re-registers the key every boot so Varaverk's enhanced
# monitoring self-heals without manual intervention.
#
# Also updates HOST*_UNRAID_API_KEY in the local host conf so the partnership
# page always reflects the live key value.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# unraid_api_key_renew.sh
#     Renew the key. Silent on success.
#
# unraid_api_key_renew.sh --dry-run
#     Show what would happen — no changes made.
#
# unraid_api_key_renew.sh --log
#     Verbose output.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../../../load_config.sh"

parse_args "$@"
acquire_lock
detect_hosts

# ──────────────────────────────────────────────────────────────────────────────
CONF_FILE="$SCRIPT_DIR/../Configurations/${MY_ID,,}.conf"
VAR_NAME="${MY_ID}_UNRAID_API_KEY"

# Key name: "Varaverk <hostname>" stripping any unraid- prefix
# Space separator — unRAID API only allows letters, numbers, and spaces
HOSTNAME_SUFFIX=$(hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//' || hostname -s)
KEY_NAME="Varaverk ${HOSTNAME_SUFFIX}"

log "$ICON_GEAR Conf file: $CONF_FILE"
log "$ICON_GEAR Key var:   $VAR_NAME"
log "$ICON_GEAR Key name:  $KEY_NAME"

if [[ ! -f "$CONF_FILE" ]]; then
    error "Conf file not found: $CONF_FILE"
    exit 1
fi

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would check registry for $KEY_NAME, renew only if missing"
    exit 0
fi

# ──────────────────────────────────────────────────────────────────────────────
# Check if key already exists in the unraid-api registry before creating.
# --overwrite generates a new key value every time, invalidating the old one.
# Only renew if the registry has lost it.
log "Checking unraid-api registry for $KEY_NAME..."
EXISTING=$(timeout 5 /usr/local/sbin/unraid-api apikey --name "$KEY_NAME" --json </dev/null 2>/dev/null)
KEY=$(echo "$EXISTING" | jq -r '.key // empty' 2>/dev/null)

if [[ -n "$KEY" ]]; then
    PREVIEW="${KEY:0:8}...${KEY: -4}"
    echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
    log "Key found in registry — no renewal needed"
    exit 0
fi

log "Key not found in registry — creating new key..."

RAW=$(timeout 10 /usr/local/sbin/unraid-api apikey \
    --name "$KEY_NAME" --create --overwrite \
    --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1)

if [[ -z "$RAW" ]]; then
    error "unraid-api returned no output"
    exit 1
fi

KEY=$(echo "$RAW" | jq -r '.key // empty' 2>/dev/null)
if [[ -z "$KEY" ]]; then
    error "No key in unraid-api response: ${RAW:0:200}"
    exit 1
fi

# ──────────────────────────────────────────────────────────────────────────────
if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
    sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
else
    echo "    ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
fi

PREVIEW="${KEY:0:8}...${KEY: -4}"
log "Writing new key to: $CONF_FILE"
warn "API key renewed ✅ — $VAR_NAME = $PREVIEW  (registry had lost it)"

# ── Push renewed key into each partner's OWN conf ─────────────────────────────
# Each host's conf is its complete keychest — no cross-host conf files needed.
# SSH_KEY is set by detect_hosts() — this server's outbound private key.
if [[ -z "$SSH_KEY" ]]; then
    log "No SSH key configured — skipping partner push"
    exit 0
fi

for host_var in $(compgen -v | grep -E '^HOST[0-9]+$'); do
    partner_host="${!host_var}"
    [[ -z "$partner_host" ]] && continue
    [[ "${host_var,,}" == "${MY_ID,,}" ]] && continue

    partner_slot="${host_var,,}"   # e.g. host2
    partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
    [[ -z "$partner_ip" ]] && { log "Cannot resolve IP for $partner_host — skipping"; continue; }

    # Target is the partner's OWN conf on their machine
    partner_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
    tmp=$(mktemp /tmp/vv_kp_XXXXXX.sh)
    remote="/tmp/vv_kp_${RANDOM}.sh"
    chmod 700 "$tmp"

    # Key stays in the temp file — never appears in SSH command args
    cat > "$tmp" <<PUSHSCRIPT
#!/bin/sh
target='${partner_conf}'
if grep -q "\b${VAR_NAME}\b" "\$target" 2>/dev/null; then
  sed -i 's|^\(\\s*${VAR_NAME}\\s*=\\s*\)"[^"]*"|\1"${KEY}"|' "\$target"
else
  printf '    ${VAR_NAME}="%s"\n' '${KEY}' >> "\$target"
fi
echo ok
PUSHSCRIPT

    if timeout 10 scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
        -o StrictHostKeyChecking=no "$tmp" "root@${partner_ip}:${remote}" 2>/dev/null; then
        if timeout 10 ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
            -o StrictHostKeyChecking=no "root@${partner_ip}" \
            "bash '${remote}'; rc=\$?; rm -f '${remote}'; exit \$rc" 2>/dev/null | grep -q ok; then
            log "Key pushed to $partner_host ✅"
        else
            warn "Key push to $partner_host failed — they can create their own copy"
        fi
    else
        warn "SCP to $partner_host failed — skipping"
    fi
    rm -f "$tmp"
done


Bash

grep -n "UNRAID_API_KEY\|create_api_key\|api_key" /boot/config/plugins/varaverk/Plugin/unraid/api/create_api_key.php | head -30

Find API key creation logic


12:echo json_encode(vv_auto_create_api_key($host, $host . '.conf'));


API key not configured — add HOST1_UNRAID_API_KEY in Scheduler → host conf to enable enhanced monitoring.i have to fight this every day or more..


Let me check whether the renewal script is wired into array start, and what the current state of the key is.


grep -rn "unraid_api_key_renew\|api_key_renew" /boot/config/plugins/varaverk/ --include="*.sh" --include="*.php" --include="*.json" 2>/dev/null | grep -v ".git"

Find all references to the API key renewal script


/boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh:20:# unraid_api_key_renew.sh
/boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh:23:# unraid_api_key_renew.sh --dry-run
/boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh:26:# unraid_api_key_renew.sh --log
/boot/config/plugins/varaverk/Plugin/unraid/api/storage.php:144:    $script = SCRIPTS_DIR . '/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh';
/boot/config/plugins/varaverk/Plugin/unraid/api/storage.php:146:        echo json_encode(['ok' => false, 'error' => 'unraid_api_key_renew.sh not found']); exit;
/boot/config/plugins/varaverk/Orchestrators/watchdog_orchestrator.sh:10:#   Default: resource_watchdog → docker_watchdog → system_watchdog → unraid_api_key_renew → stability_watchdog
/boot/config/plugins/varaverk/Orchestrators/array_started.sh:15:#     System_Essentials/unraid_api_key_renew.sh — re-register Varaverk API key at boot
/boot/config/plugins/varaverk/Orchestrators/array_started.sh:29:#   unraid_api_key_renew.sh — before anything else — self-heals API registry at boot


timeout 5 /usr/local/sbin/unraid-api apikey --name "Varaverk Gmer4Lfe" --json </dev/null 2>/dev/null || echo "FAILED or no such key"

Check if Varaverk API key exists in Unraid registry


The key exists in the Unraid registry but hasn't been written to host1.conf. Let me check the array_started script and why the write isn't happening.


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Array Start Orchestrator ===================================
4	# ==============================================================================================
5	# Single entry point for array start — fired by the Varaverk plugin's
6	# disks_mounted event hook (Plugin/unraid/event/disks_mounted/array_start_jobs).
7	# Launches everything configured in ARRAY_START_SCRIPTS in master.conf.
8	# This script exits after launching all scripts — the event hook sees it complete normally.
9	#
10	# ── WHAT IT LAUNCHES ──────────────────────────────────────────────────────────────────────────
11	# Configured in master.conf ARRAY_START_SCRIPTS — no changes to this script ever needed.
12	# Current order (order matters — see below):
13	#
14	#   ONE-SHOT (run and exit naturally):
15	#     System_Essentials/unraid_api_key_renew.sh — re-register Varaverk API key at boot
16	#     System_Essentials/inotify_tuning.sh       — raise inotify limits before containers start
17	#     System_Essentials/docker_syslog_filter.sh — suppress veth log noise before logs fill
18	#     System_Essentials/php_fpm_max_children.sh — WebGUI performance tuning
19	#     Transcodes/ramdisk_setup.sh               — create tmpfs + symlink before Emby starts
20	#     Docker_Essentials/docker_network_connect.sh — ensure networks + container connections
21	#
22	#   CONTINUOUS (run until array stops):
23	#     Fallback/fallback.sh                      — mutual fallback monitor
24	#
25	#   NOTE: watchdogs (docker, system, stability) are NOT launched here.
26	#   They run via watchdog_orchestrator.sh every 15 min (cron), not as daemons.
27	#
28	# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
29	#   unraid_api_key_renew.sh — before anything else — self-heals API registry at boot
30	#   inotify_tuning.sh     — must run BEFORE Code-Server and other containers start
31	#                           containers that start with low inotify limits keep them ✅
32	#   docker_syslog_filter  — must run BEFORE any container starts creating veth interfaces
33	#   ramdisk_setup.sh      — must run BEFORE Emby starts transcoding
34	#   docker_network_connect — must run BEFORE watchdogs check container states
35	#   fallback.sh           — last — needs everything else stable to make decisions
36	#
37	# ── ONE-SHOT vs CONTINUOUS DETECTION ─────────────────────────────────────────────────────────
38	# Script is launched in background with bash script.sh &
39	# After 1 second: if PID still alive → continuous (running in background)
40	#                  if PID dead + exit 0 → one-shot completed successfully
41	#                  if PID dead + exit N → failure
42	#
43	# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
44	#   Root check           — all launched scripts require root
45	#   acquire_lock         — prevents duplicate array start launches
46	#   detect_hosts()       — MY_ID in notifications
47	#   platform_require_cmd  — notify validated before use
48	#   chmod +x auto-fix    — non-executable scripts fixed before launch
49	#   Full path on failure — shows exact path for debugging
50	#   notify on failures   — alert if any script fails to launch
51	#
52	# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
53	#   ARRAY_START_SCRIPTS — ordered list of scripts to launch at array start
54	#
55	# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
56	#   array_started.sh              — normal launch (called by Varaverk disks_mounted event hook)
57	#   array_started.sh --dry-run    — show what would be launched without launching
58	#   array_started.sh --status     — show configured scripts and their current state
59	#   array_started.sh --log        — verbose output per script
60	# ==============================================================================================
61
62	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
63	ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
64
65	source "$ECOSYSTEM_ROOT/load_config.sh"
66
67	parse_args "$@"
68
69	# ==============================================================================================
70	# ━━━ Setup ━━━
71	# ==============================================================================================
72	if [[ "$EUID" -ne 0 ]]; then
73	    error "Must be run as root"
74	    exit 1
75	fi
76
77	platform_require_cmd \
78	    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
79	    "" "" \
80	    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
81
82	acquire_lock
83
84	detect_hosts
85
86	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — scripts will not be launched"
87
88	# ==============================================================================================
89	# ━━━ Status ━━━
90	# ==============================================================================================
91	if [[ "$SHOW_STATUS" == true ]]; then
92	    echo ""
93	    echo "━━━━━ $ICON_SUMMARY ARRAY START STATUS ━━━━━"
94	    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
95	    echo "$ICON_GEAR  Scripts:  ${#ARRAY_START_SCRIPTS[@]} configured"
96	    echo ""
97
98	    for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
99	        [[ -z "$relative_path" ]] && continue
100	        script_path="$ECOSYSTEM_ROOT/$relative_path"
101	        script_name=$(basename "$script_path")
102
103	        if [[ ! -f "$script_path" ]]; then
104	            echo "  $ICON_ERROR $script_name — FILE NOT FOUND"
105	            echo "    $script_path"
106	            continue
107	        fi
108
109	        [[ ! -x "$script_path" ]] && flag=" (not executable — will auto-fix)" || flag=""
110
111	        # Check if currently running
112	        if pgrep -f "$script_path" >/dev/null 2>&1; then
113	            RUN_PID=$(pgrep -f "$script_path" | head -1)
114	            echo "  $ICON_RUNNING $script_name — RUNNING (PID $RUN_PID)${flag}"
115	        else
116	            echo "  $ICON_NOT_RUNNING $script_name — not running${flag}"
117	        fi
118	    done
119	    echo "━━━━━━━━━━━━━━━━━━━━━━━"
120	    exit 0
121	fi
122
123	# ==============================================================================================
124	# ━━━ Launch Scripts ━━━
125	# ==============================================================================================
126	echo ""
127	echo "━━━ $ICON_GEAR Array Start — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
128	log "Ecosystem root: $ECOSYSTEM_ROOT"
129	log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..."
130	echo ""
131
132	START=$(date +%s)
133	LAUNCHED=0
134	FAILED=0
135	FAILED_SCRIPTS=()
136
137	for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
138	    [[ -z "$relative_path" ]] && continue
139
140	    SCRIPT_PATH="$ECOSYSTEM_ROOT/$relative_path"
141	    SCRIPT_NAME=$(basename "$SCRIPT_PATH")
142
143	    # File existence check
144	    if [[ ! -f "$SCRIPT_PATH" ]]; then
145	        error "$SCRIPT_NAME — not found"
146	        error "  Expected: $SCRIPT_PATH"
147	        (( FAILED++ ))
148	        FAILED_SCRIPTS+=("$SCRIPT_NAME")
149	        continue
150	    fi
151
152	    # Auto-fix permissions — chmod +x if needed
153	    if [[ ! -x "$SCRIPT_PATH" ]]; then
154	        warn "$SCRIPT_NAME — not executable, fixing..."
155	        chmod +x "$SCRIPT_PATH" || {
156	            error "$SCRIPT_NAME — chmod +x failed"
157	            (( FAILED++ ))
158	            FAILED_SCRIPTS+=("$SCRIPT_NAME")
159	            continue
160	        }
161	    fi
162
163	    if [[ "$DRY_RUN" == true ]]; then
164	        warn "DRY RUN — would launch: $SCRIPT_NAME"
165	        (( LAUNCHED++ ))
166	        continue
167	    fi
168
169	    log "$ICON_START Launching $SCRIPT_NAME..."
170	    bash "$SCRIPT_PATH" &
171	    PID=$!
172
173	    # Brief settle — 1s enough to detect immediate failures
174	    sleep 1
175
176	    if kill -0 "$PID" 2>/dev/null; then
177	        # Still running → continuous script
178	        warn "$SCRIPT_NAME — running (PID $PID) ✅"
179	        (( LAUNCHED++ ))
180	    else
181	        # Exited — check if one-shot success or failure
182	        wait "$PID"
183	        EXIT_CODE=$?
184	        if [[ "$EXIT_CODE" -eq 0 ]]; then
185	            log "$SCRIPT_NAME — completed (one-shot) ✅"
186	            (( LAUNCHED++ ))
187	        else
188	            error "$SCRIPT_NAME — exited with code $EXIT_CODE"
189	            error "  Path: $SCRIPT_PATH"
190	            (( FAILED++ ))
191	            FAILED_SCRIPTS+=("$SCRIPT_NAME")
192	        fi
193	    fi
194
195	done
196
197	END=$(date +%s)
198
199	# ==============================================================================================
200	# ━━━ Summary ━━━
201	# ==============================================================================================
202	echo ""
203	echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
204	echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
205	echo "$ICON_SUCCESS Launched: $LAUNCHED"
206	[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR  Failed:   $FAILED — ${FAILED_SCRIPTS[*]}"
207	echo "$ICON_TIME    Duration: $(format_duration $(( END - START )))"
208	echo ""
209
210	if [[ "$DRY_RUN" == true ]]; then
211	    warn "DRY RUN — no scripts launched"
212	elif [[ "$FAILED" -gt 0 ]]; then
213	    warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}"
214	    notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
215	        "Array Start" "warning"
216	else
217	    echo "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
218	fi
219	echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

#!/bin/bash
# ==============================================================================================
# ================================= Array Start Orchestrator ===================================
# ==============================================================================================
# Single entry point for array start — fired by the Varaverk plugin's
# disks_mounted event hook (Plugin/unraid/event/disks_mounted/array_start_jobs).
# Launches everything configured in ARRAY_START_SCRIPTS in master.conf.
# This script exits after launching all scripts — the event hook sees it complete normally.
#
# ── WHAT IT LAUNCHES ──────────────────────────────────────────────────────────────────────────
# Configured in master.conf ARRAY_START_SCRIPTS — no changes to this script ever needed.
# Current order (order matters — see below):
#
#   ONE-SHOT (run and exit naturally):
#     System_Essentials/unraid_api_key_renew.sh — re-register Varaverk API key at boot
#     System_Essentials/inotify_tuning.sh       — raise inotify limits before containers start
#     System_Essentials/docker_syslog_filter.sh — suppress veth log noise before logs fill
#     System_Essentials/php_fpm_max_children.sh — WebGUI performance tuning
#     Transcodes/ramdisk_setup.sh               — create tmpfs + symlink before Emby starts
#     Docker_Essentials/docker_network_connect.sh — ensure networks + container connections
#
#   CONTINUOUS (run until array stops):
#     Fallback/fallback.sh                      — mutual fallback monitor
#
#   NOTE: watchdogs (docker, system, stability) are NOT launched here.
#   They run via watchdog_orchestrator.sh every 15 min (cron), not as daemons.
#
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
#   unraid_api_key_renew.sh — before anything else — self-heals API registry at boot
#   inotify_tuning.sh     — must run BEFORE Code-Server and other containers start
#                           containers that start with low inotify limits keep them ✅
#   docker_syslog_filter  — must run BEFORE any container starts creating veth interfaces
#   ramdisk_setup.sh      — must run BEFORE Emby starts transcoding
#   docker_network_connect — must run BEFORE watchdogs check container states
#   fallback.sh           — last — needs everything else stable to make decisions
#
# ── ONE-SHOT vs CONTINUOUS DETECTION ─────────────────────────────────────────────────────────
# Script is launched in background with bash script.sh &
# After 1 second: if PID still alive → continuous (running in background)
#                  if PID dead + exit 0 → one-shot completed successfully
#                  if PID dead + exit N → failure
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
#   Root check           — all launched scripts require root
#   acquire_lock         — prevents duplicate array start launches
#   detect_hosts()       — MY_ID in notifications
#   platform_require_cmd  — notify validated before use
#   chmod +x auto-fix    — non-executable scripts fixed before launch
#   Full path on failure — shows exact path for debugging
#   notify on failures   — alert if any script fails to launch
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   ARRAY_START_SCRIPTS — ordered list of scripts to launch at array start
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   array_started.sh              — normal launch (called by Varaverk disks_mounted event hook)
#   array_started.sh --dry-run    — show what would be launched without launching
#   array_started.sh --status     — show configured scripts and their current state
#   array_started.sh --log        — verbose output per script
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

source "$ECOSYSTEM_ROOT/load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

platform_require_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock

detect_hosts

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — scripts will not be launched"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY ARRAY START STATUS ━━━━━"
    echo "$ICON_HOST  Identity: $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR  Scripts:  ${#ARRAY_START_SCRIPTS[@]} configured"
    echo ""

    for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
        [[ -z "$relative_path" ]] && continue
        script_path="$ECOSYSTEM_ROOT/$relative_path"
        script_name=$(basename "$script_path")

        if [[ ! -f "$script_path" ]]; then
            echo "  $ICON_ERROR $script_name — FILE NOT FOUND"
            echo "    $script_path"
            continue
        fi

        [[ ! -x "$script_path" ]] && flag=" (not executable — will auto-fix)" || flag=""

        # Check if currently running
        if pgrep -f "$script_path" >/dev/null 2>&1; then
            RUN_PID=$(pgrep -f "$script_path" | head -1)
            echo "  $ICON_RUNNING $script_name — RUNNING (PID $RUN_PID)${flag}"
        else
            echo "  $ICON_NOT_RUNNING $script_name — not running${flag}"
        fi
    done
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Launch Scripts ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Array Start — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
log "Ecosystem root: $ECOSYSTEM_ROOT"
log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..."
echo ""

START=$(date +%s)
LAUNCHED=0
FAILED=0
FAILED_SCRIPTS=()

for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
    [[ -z "$relative_path" ]] && continue

    SCRIPT_PATH="$ECOSYSTEM_ROOT/$relative_path"
    SCRIPT_NAME=$(basename "$SCRIPT_PATH")

    # File existence check
    if [[ ! -f "$SCRIPT_PATH" ]]; then
        error "$SCRIPT_NAME — not found"
        error "  Expected: $SCRIPT_PATH"
        (( FAILED++ ))
        FAILED_SCRIPTS+=("$SCRIPT_NAME")
        continue
    fi

    # Auto-fix permissions — chmod +x if needed
    if [[ ! -x "$SCRIPT_PATH" ]]; then
        warn "$SCRIPT_NAME — not executable, fixing..."
        chmod +x "$SCRIPT_PATH" || {
            error "$SCRIPT_NAME — chmod +x failed"
            (( FAILED++ ))
            FAILED_SCRIPTS+=("$SCRIPT_NAME")
            continue
        }
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would launch: $SCRIPT_NAME"
        (( LAUNCHED++ ))
        continue
    fi

    log "$ICON_START Launching $SCRIPT_NAME..."
    bash "$SCRIPT_PATH" &
    PID=$!

    # Brief settle — 1s enough to detect immediate failures
    sleep 1

    if kill -0 "$PID" 2>/dev/null; then
        # Still running → continuous script
        warn "$SCRIPT_NAME — running (PID $PID) ✅"
        (( LAUNCHED++ ))
    else
        # Exited — check if one-shot success or failure
        wait "$PID"
        EXIT_CODE=$?
        if [[ "$EXIT_CODE" -eq 0 ]]; then
            log "$SCRIPT_NAME — completed (one-shot) ✅"
            (( LAUNCHED++ ))
        else
            error "$SCRIPT_NAME — exited with code $EXIT_CODE"
            error "  Path: $SCRIPT_PATH"
            (( FAILED++ ))
            FAILED_SCRIPTS+=("$SCRIPT_NAME")
        fi
    fi

done

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
echo "$ICON_HOST    Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_SUCCESS Launched: $LAUNCHED"
[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR  Failed:   $FAILED — ${FAILED_SCRIPTS[*]}"
echo "$ICON_TIME    Duration: $(format_duration $(( END - START )))"
echo ""

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no scripts launched"
elif [[ "$FAILED" -gt 0 ]]; then
    warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}"
    notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
        "Array Start" "warning"
else
    echo "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"


Found the bug. The renewal script exits early when the key is already in the registry — without checking if `host1.conf` still has it. The key IS in the registry right now (`1825c3a2e0...`), but host1.conf is empty. Every time you reboot or the conf gets refreshed, the script finds the key in registry, says "no renewal needed," and exits without writing it.

The fix is one line — write to conf regardless of whether the key was newly created or already existed:


66	log "Checking unraid-api registry for $KEY_NAME..."
67	EXISTING=$(timeout 5 /usr/local/sbin/unraid-api apikey --name "$KEY_NAME" --json </dev/null 2>/dev/null)
68	KEY=$(echo "$EXISTING" | jq -r '.key // empty' 2>/dev/null)
69
70	if [[ -n "$KEY" ]]; then
71	    PREVIEW="${KEY:0:8}...${KEY: -4}"
72	    echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
73	    log "Key found in registry — no renewal needed"
74	    exit 0
75	fi
76
77	log "Key not found in registry — creating new key..."
78
79	RAW=$(timeout 10 /usr/local/sbin/unraid-api apikey \
80	    --name "$KEY_NAME" --create --overwrite \
81	    --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1)
82
83	if [[ -z "$RAW" ]]; then
84	    error "unraid-api returned no output"
85	    exit 1
86	fi
87
88	KEY=$(echo "$RAW" | jq -r '.key // empty' 2>/dev/null)
89	if [[ -z "$KEY" ]]; then
90	    error "No key in unraid-api response: ${RAW:0:200}"
91	    exit 1
92	fi
93
94	# ──────────────────────────────────────────────────────────────────────────────
95	if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
96	    sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
97	else
98	    echo "    ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
99	fi
100
101	PREVIEW="${KEY:0:8}...${KEY: -4}"
102	log "Writing new key to: $CONF_FILE"
103	warn "API key renewed ✅ — $VAR_NAME = $PREVIEW  (registry had lost it)"
104
105	# ── Push renewed key into each partner's OWN conf ─────────────────────────────

log "Checking unraid-api registry for $KEY_NAME..."
EXISTING=$(timeout 5 /usr/local/sbin/unraid-api apikey --name "$KEY_NAME" --json </dev/null 2>/dev/null)
KEY=$(echo "$EXISTING" | jq -r '.key // empty' 2>/dev/null)

if [[ -n "$KEY" ]]; then
    PREVIEW="${KEY:0:8}...${KEY: -4}"
    echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
    log "Key found in registry — no renewal needed"
    exit 0
fi

log "Key not found in registry — creating new key..."

RAW=$(timeout 10 /usr/local/sbin/unraid-api apikey \
    --name "$KEY_NAME" --create --overwrite \
    --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1)

if [[ -z "$RAW" ]]; then
    error "unraid-api returned no output"
    exit 1
fi

KEY=$(echo "$RAW" | jq -r '.key // empty' 2>/dev/null)
if [[ -z "$KEY" ]]; then
    error "No key in unraid-api response: ${RAW:0:200}"
    exit 1
fi

# ──────────────────────────────────────────────────────────────────────────────
if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
    sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
else
    echo "    ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
fi

PREVIEW="${KEY:0:8}...${KEY: -4}"
log "Writing new key to: $CONF_FILE"
warn "API key renewed ✅ — $VAR_NAME = $PREVIEW  (registry had lost it)"

# ── Push renewed key into each partner's OWN conf ─────────────────────────────


Edit

if [[ -n "$KEY" ]]; then
    PREVIEW="${KEY:0:8}...${KEY: -4}"
    echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
    log "Key found in registry — no renewal needed"
    exit 0
fi

if [[ -n "$KEY" ]]; then
    PREVIEW="${KEY:0:8}...${KEY: -4}"
    # Always sync registry key → conf, even if the key was already there.
    # Conf gets wiped on git pull / conf regeneration without touching the registry.
    CONF_HAS_KEY=$(grep -oP "(?<=^\s*${VAR_NAME}=\")[^\"]*" "$CONF_FILE" 2>/dev/null || true)
    if [[ "$CONF_HAS_KEY" == "$KEY" ]]; then
        echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
        log "Key in registry and conf — no action needed"
        exit 0
    fi
    log "Key in registry but conf is stale — syncing..."
    if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
        sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
    else
        echo "    ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
    fi
    echo "API key synced to conf ✅ — $VAR_NAME = $PREVIEW"
    exit 0
fi


The file /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh has been updated successfully. (file state is current in your context — no need to Read it back)

if [[ -n "$KEY" ]]; then
    PREVIEW="${KEY:0:8}...${KEY: -4}"
    # Always sync registry key → conf, even if the key was already there.
    # Conf gets wiped on git pull / conf regeneration without touching the registry.
    CONF_HAS_KEY=$(grep -oP "(?<=^\s*${VAR_NAME}=\")[^\"]*" "$CONF_FILE" 2>/dev/null || true)
    if [[ "$CONF_HAS_KEY" == "$KEY" ]]; then
        echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
        log "Key in registry and conf — no action needed"
        exit 0
    fi
    log "Key in registry but conf is stale — syncing..."
    if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
        sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
    else
        echo "    ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
    fi
    echo "API key synced to conf ✅ — $VAR_NAME = $PREVIEW"
    exit 0
fi

#!/bin/bash
# ==============================================================================================
# ================================= Unraid API Key Renewal ====================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Creates/overwrites the Varaverk API key in the unraid-api service registry at
# array start. The registry is ephemeral — OS updates and service restarts clear
# it. This script re-registers the key every boot so Varaverk's enhanced
# monitoring self-heals without manual intervention.
#
# Also updates HOST*_UNRAID_API_KEY in the local host conf so the partnership
# page always reflects the live key value.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# unraid_api_key_renew.sh
#     Renew the key. Silent on success.
#
# unraid_api_key_renew.sh --dry-run
#     Show what would happen — no changes made.
#
# unraid_api_key_renew.sh --log
#     Verbose output.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../../../load_config.sh"

parse_args "$@"
acquire_lock
detect_hosts

# ──────────────────────────────────────────────────────────────────────────────
CONF_FILE="$SCRIPT_DIR/../Configurations/${MY_ID,,}.conf"
VAR_NAME="${MY_ID}_UNRAID_API_KEY"

# Key name: "Varaverk <hostname>" stripping any unraid- prefix
# Space separator — unRAID API only allows letters, numbers, and spaces
HOSTNAME_SUFFIX=$(hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//' || hostname -s)
KEY_NAME="Varaverk ${HOSTNAME_SUFFIX}"

log "$ICON_GEAR Conf file: $CONF_FILE"
log "$ICON_GEAR Key var:   $VAR_NAME"
log "$ICON_GEAR Key name:  $KEY_NAME"

if [[ ! -f "$CONF_FILE" ]]; then
    error "Conf file not found: $CONF_FILE"
    exit 1
fi

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would check registry for $KEY_NAME, renew only if missing"
    exit 0
fi

# ──────────────────────────────────────────────────────────────────────────────
# Check if key already exists in the unraid-api registry before creating.
# --overwrite generates a new key value every time, invalidating the old one.
# Only renew if the registry has lost it.
log "Checking unraid-api registry for $KEY_NAME..."
EXISTING=$(timeout 5 /usr/local/sbin/unraid-api apikey --name "$KEY_NAME" --json </dev/null 2>/dev/null)
KEY=$(echo "$EXISTING" | jq -r '.key // empty' 2>/dev/null)

if [[ -n "$KEY" ]]; then
    PREVIEW="${KEY:0:8}...${KEY: -4}"
    echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
    log "Key found in registry — no renewal needed"
    exit 0
fi

log "Key not found in registry — creating new key..."

RAW=$(timeout 10 /usr/local/sbin/unraid-api apikey \
    --name "$KEY_NAME" --create --overwrite \
    --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1)

if [[ -z "$RAW" ]]; then
    error "unraid-api returned no output"
    exit 1
fi

KEY=$(echo "$RAW" | jq -r '.key // empty' 2>/dev/null)
if [[ -z "$KEY" ]]; then
    error "No key in unraid-api response: ${RAW:0:200}"
    exit 1
fi

# ──────────────────────────────────────────────────────────────────────────────
if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
    sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
else
    echo "    ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
fi

PREVIEW="${KEY:0:8}...${KEY: -4}"
log "Writing new key to: $CONF_FILE"
warn "API key renewed ✅ — $VAR_NAME = $PREVIEW  (registry had lost it)"

# ── Push renewed key into each partner's OWN conf ─────────────────────────────
# Each host's conf is its complete keychest — no cross-host conf files needed.
# SSH_KEY is set by detect_hosts() — this server's outbound private key.
if [[ -z "$SSH_KEY" ]]; then
    log "No SSH key configured — skipping partner push"
    exit 0
fi

for host_var in $(compgen -v | grep -E '^HOST[0-9]+$'); do
    partner_host="${!host_var}"
    [[ -z "$partner_host" ]] && continue
    [[ "${host_var,,}" == "${MY_ID,,}" ]] && continue

    partner_slot="${host_var,,}"   # e.g. host2
    partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
    [[ -z "$partner_ip" ]] && { log "Cannot resolve IP for $partner_host — skipping"; continue; }

    # Target is the partner's OWN conf on their machine
    partner_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
    tmp=$(mktemp /tmp/vv_kp_XXXXXX.sh)
    remote="/tmp/vv_kp_${RANDOM}.sh"
    chmod 700 "$tmp"

    # Key stays in the temp file — never appears in SSH command args
    cat > "$tmp" <<PUSHSCRIPT
#!/bin/sh
target='${partner_conf}'
if grep -q "\b${VAR_NAME}\b" "\$target" 2>/dev/null; then
  sed -i 's|^\(\\s*${VAR_NAME}\\s*=\\s*\)"[^"]*"|\1"${KEY}"|' "\$target"
else
  printf '    ${VAR_NAME}="%s"\n' '${KEY}' >> "\$target"
fi
echo ok
PUSHSCRIPT

    if timeout 10 scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
        -o StrictHostKeyChecking=no "$tmp" "root@${partner_ip}:${remote}" 2>/dev/null; then
        if timeout 10 ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
            -o StrictHostKeyChecking=no "root@${partner_ip}" \
            "bash '${remote}'; rc=\$?; rm -f '${remote}'; exit \$rc" 2>/dev/null | grep -q ok; then
            log "Key pushed to $partner_host ✅"
        else
            warn "Key push to $partner_host failed — they can create their own copy"
        fi
    else
        warn "SCP to $partner_host failed — skipping"
    fi
    rm -f "$tmp"
done

-    echo "API key valid ✅ — $VAR_NAME = $PREVIEW"

-    log "Key found in registry — no renewal needed"

+    # Always sync registry key → conf, even if the key was already there.

+    # Conf gets wiped on git pull / conf regeneration without touching the registry.

+    CONF_HAS_KEY=$(grep -oP "(?<=^\s*${VAR_NAME}=\")[^\"]*" "$CONF_FILE" 2>/dev/null || true)

+    if [[ "$CONF_HAS_KEY" == "$KEY" ]]; then

+        echo "API key valid ✅ — $VAR_NAME = $PREVIEW"

+        log "Key in registry and conf — no action needed"

+    log "Key in registry but conf is stale — syncing..."

+    if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then

+        echo "    ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"

+    echo "API key synced to conf ✅ — $VAR_NAME = $PREVIEW"


Now let's run it to fix the conf right now:


Bash

Run the renewal script to sync the key into host1.conf


Error: Exit code 1
[LOG] 🔏 Lock acquired: unraid_api_key_renew (PID 2988046)
[LOG] 🖥 Host: unRAID-Gmer4Lfe → unRAID-Jayred365
[LOG] ⚙ Conf file: /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/../Configurations/host1.conf
[LOG] ⚙ Key var:   HOST1_UNRAID_API_KEY
[LOG] ⚙ Key name:  Varaverk Gmer4Lfe
❌ [ERROR] Conf file not found: /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/../Configurations/host1.conf


The conf path in the script has the wrong number of `../` traversals. Let me fix that too:


38
39	# ──────────────────────────────────────────────────────────────────────────────
40	CONF_FILE="$SCRIPT_DIR/../Configurations/${MY_ID,,}.conf"
41	VAR_NAME="${MY_ID}_UNRAID_API_KEY"
42
43	# Key name: "Varaverk <hostname>" stripping any unraid- prefix
44	# Space separator — unRAID API only allows letters, numbers, and spaces
45	HOSTNAME_SUFFIX=$(hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//' || hostname -s)
46	KEY_NAME="Varaverk ${HOSTNAME_SUFFIX}"
47

# ──────────────────────────────────────────────────────────────────────────────
CONF_FILE="$SCRIPT_DIR/../Configurations/${MY_ID,,}.conf"
VAR_NAME="${MY_ID}_UNRAID_API_KEY"

# Key name: "Varaverk <hostname>" stripping any unraid- prefix
# Space separator — unRAID API only allows letters, numbers, and spaces
HOSTNAME_SUFFIX=$(hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//' || hostname -s)
KEY_NAME="Varaverk ${HOSTNAME_SUFFIX}"


#!/bin/bash
# ==============================================================================================
# ================================= Unraid API Key Renewal ====================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Creates/overwrites the Varaverk API key in the unraid-api service registry at
# array start. The registry is ephemeral — OS updates and service restarts clear
# it. This script re-registers the key every boot so Varaverk's enhanced
# monitoring self-heals without manual intervention.
#
# Also updates HOST*_UNRAID_API_KEY in the local host conf so the partnership
# page always reflects the live key value.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# unraid_api_key_renew.sh
#     Renew the key. Silent on success.
#
# unraid_api_key_renew.sh --dry-run
#     Show what would happen — no changes made.
#
# unraid_api_key_renew.sh --log
#     Verbose output.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../../../load_config.sh"

parse_args "$@"
acquire_lock
detect_hosts

# ──────────────────────────────────────────────────────────────────────────────
CONF_FILE="$SCRIPT_DIR/../Configurations/${MY_ID,,}.conf"
VAR_NAME="${MY_ID}_UNRAID_API_KEY"

# Key name: "Varaverk <hostname>" stripping any unraid- prefix
# Space separator — unRAID API only allows letters, numbers, and spaces
HOSTNAME_SUFFIX=$(hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//' || hostname -s)
KEY_NAME="Varaverk ${HOSTNAME_SUFFIX}"

log "$ICON_GEAR Conf file: $CONF_FILE"
log "$ICON_GEAR Key var:   $VAR_NAME"
log "$ICON_GEAR Key name:  $KEY_NAME"

if [[ ! -f "$CONF_FILE" ]]; then
    error "Conf file not found: $CONF_FILE"
    exit 1
fi

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would check registry for $KEY_NAME, renew only if missing"
    exit 0
fi

# ──────────────────────────────────────────────────────────────────────────────
# Check if key already exists in the unraid-api registry before creating.
# --overwrite generates a new key value every time, invalidating the old one.
# Only renew if the registry has lost it.
log "Checking unraid-api registry for $KEY_NAME..."
EXISTING=$(timeout 5 /usr/local/sbin/unraid-api apikey --name "$KEY_NAME" --json </dev/null 2>/dev/null)
KEY=$(echo "$EXISTING" | jq -r '.key // empty' 2>/dev/null)

if [[ -n "$KEY" ]]; then
    PREVIEW="${KEY:0:8}...${KEY: -4}"
    # Always sync registry key → conf, even if the key was already there.
    # Conf gets wiped on git pull / conf regeneration without touching the registry.
    CONF_HAS_KEY=$(grep -oP "(?<=^\s*${VAR_NAME}=\")[^\"]*" "$CONF_FILE" 2>/dev/null || true)
    if [[ "$CONF_HAS_KEY" == "$KEY" ]]; then
        echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
        log "Key in registry and conf — no action needed"
        exit 0
    fi
    log "Key in registry but conf is stale — syncing..."
    if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
        sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
    else
        echo "    ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
    fi
    echo "API key synced to conf ✅ — $VAR_NAME = $PREVIEW"
    exit 0
fi

log "Key not found in registry — creating new key..."

RAW=$(timeout 10 /usr/local/sbin/unraid-api apikey \
    --name "$KEY_NAME" --create --overwrite \
    --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1)

if [[ -z "$RAW" ]]; then
    error "unraid-api returned no output"
    exit 1
fi

KEY=$(echo "$RAW" | jq -r '.key // empty' 2>/dev/null)
if [[ -z "$KEY" ]]; then
    error "No key in unraid-api response: ${RAW:0:200}"
    exit 1
fi

# ──────────────────────────────────────────────────────────────────────────────
if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
    sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
else
    echo "    ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
fi

PREVIEW="${KEY:0:8}...${KEY: -4}"
log "Writing new key to: $CONF_FILE"
warn "API key renewed ✅ — $VAR_NAME = $PREVIEW  (registry had lost it)"

# ── Push renewed key into each partner's OWN conf ─────────────────────────────
# Each host's conf is its complete keychest — no cross-host conf files needed.
# SSH_KEY is set by detect_hosts() — this server's outbound private key.
if [[ -z "$SSH_KEY" ]]; then
    log "No SSH key configured — skipping partner push"
    exit 0
fi

for host_var in $(compgen -v | grep -E '^HOST[0-9]+$'); do
    partner_host="${!host_var}"
    [[ -z "$partner_host" ]] && continue
    [[ "${host_var,,}" == "${MY_ID,,}" ]] && continue

    partner_slot="${host_var,,}"   # e.g. host2
    partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
    [[ -z "$partner_ip" ]] && { log "Cannot resolve IP for $partner_host — skipping"; continue; }

    # Target is the partner's OWN conf on their machine
    partner_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
    tmp=$(mktemp /tmp/vv_kp_XXXXXX.sh)
    remote="/tmp/vv_kp_${RANDOM}.sh"
    chmod 700 "$tmp"

    # Key stays in the temp file — never appears in SSH command args
    cat > "$tmp" <<PUSHSCRIPT
#!/bin/sh
target='${partner_conf}'
if grep -q "\b${VAR_NAME}\b" "\$target" 2>/dev/null; then
  sed -i 's|^\(\\s*${VAR_NAME}\\s*=\\s*\)"[^"]*"|\1"${KEY}"|' "\$target"
else
  printf '    ${VAR_NAME}="%s"\n' '${KEY}' >> "\$target"
fi
echo ok
PUSHSCRIPT

    if timeout 10 scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
        -o StrictHostKeyChecking=no "$tmp" "root@${partner_ip}:${remote}" 2>/dev/null; then
        if timeout 10 ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
            -o StrictHostKeyChecking=no "root@${partner_ip}" \
            "bash '${remote}'; rc=\$?; rm -f '${remote}'; exit \$rc" 2>/dev/null | grep -q ok; then
            log "Key pushed to $partner_host ✅"
        else
            warn "Key push to $partner_host failed — they can create their own copy"
        fi
    else
        warn "SCP to $partner_host failed — skipping"
    fi
    rm -f "$tmp"
done


Run the renewal script again with the fixed path


[LOG] 🔏 Lock acquired: unraid_api_key_renew (PID 3038551)
[LOG] 🖥 Host: unRAID-Gmer4Lfe → unRAID-Jayred365
[LOG] ⚙ Conf file: /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/../../../Configurations/host1.conf
[LOG] ⚙ Key var:   HOST1_UNRAID_API_KEY
[LOG] ⚙ Key name:  Varaverk Gmer4Lfe
[LOG] Checking unraid-api registry for Varaverk Gmer4Lfe...
[LOG] Key in registry but conf is stale — syncing...
API key synced to conf ✅ — HOST1_UNRAID_API_KEY = 1825c3a2...4dbb


Let's verify it landed in host1.conf:


Confirm key was written to host1.conf


Two bugs fixed:

1. **Path bug** (`../` → `../../../`) — the script was resolving to `Plugin/unraid/Configurations/` instead of the actual `Configurations/` dir, so it always hit "conf file not found" and exited before reaching the write step. This means the script has probably **never worked** when called automatically at array start.

2. **Early-exit bug** — even with the correct path, when the key already existed in the Unraid registry, the script would exit without checking whether host1.conf still had the key. Now it compares the registry key to the conf value and only skips if they match.

The banner should disappear on next page load. On future reboots, `array_started.sh` will run the renewal script and it'll self-heal the conf correctly.


any other issues you see


Let me look for similar issues — same path pattern, same early-exit logic — across the codebase.


grep -rn 'SCRIPT_DIR/\.\.' /boot/config/plugins/varaverk/Plugin/unraid/ --include="*.sh" | grep -v ".git" | grep -v "load_config"

Find all SCRIPT_DIR relative paths in Plugin/unraid scripts


/boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh:40:CONF_FILE="$SCRIPT_DIR/../../../Configurations/${MY_ID,,}.conf"


grep -rn 'source.*load_config\|SCRIPT_DIR.*load_config' /boot/config/plugins/varaverk/Plugin/unraid/ --include="*.sh" | grep -v ".git"

Find how scripts in Plugin/unraid source load_config


/boot/config/plugins/varaverk/Plugin/unraid/Tools/remote_arr_cache_writer.sh:29:source "$SCRIPT_DIR/../../../load_config.sh"
/boot/config/plugins/varaverk/Plugin/unraid/Tools/recreate_shares.sh:80:source "$SCRIPT_DIR/../../../load_config.sh"
/boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/unraid_api_key_renew.sh:33:source "$SCRIPT_DIR/../../../load_config.sh"
/boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/php_fpm_max_children.sh:94:source "$SCRIPT_DIR/../../../load_config.sh"
/boot/config/plugins/varaverk/Plugin/unraid/Watchdogs/System/webgui_watchdog.sh:95:source "$SCRIPT_DIR/../../../../load_config.sh"
/boot/config/plugins/varaverk/Plugin/unraid/Tools/conf_populate.sh:56:source "$SCRIPT_DIR/../../../load_config.sh"
/boot/config/plugins/varaverk/Plugin/unraid/Partnership/containers.sh:284:            "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
/boot/config/plugins/varaverk/Plugin/unraid/Partnership/containers.sh:289:            "source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
/boot/config/plugins/varaverk/Plugin/unraid/Tools/storage_migrate.sh:45:source "$SCRIPT_DIR/../../../load_config.sh"
/boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/user_scripts_stop.sh:73:source "$SCRIPT_DIR/../../../load_config.sh"
/boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/mover_stop.sh:77:source "$SCRIPT_DIR/../../../load_config.sh"


# Check PHP files for logic issues in the monitor page - specifically the banner condition
# Also check create_api_key.php and the watchdog_orchestrator reference
grep -n "UNRAID_API_KEY\|key_missing\|key_found\|api_status\|api_banner" /boot/config/plugins/varaverk/Plugin/unraid/pages/scheduler.php | head -20

Check scheduler page for API key references


(Bash completed with no output)


grep -n "unraid_api_key_renew" /boot/config/plugins/varaverk/Orchestrators/watchdog_orchestrator.sh | head -10

Check if watchdog orchestrator also calls the renewal script


10:#   Default: resource_watchdog → docker_watchdog → system_watchdog → unraid_api_key_renew → stability_watchdog


grep -n "unraid_api_key_renew\|UNRAID_API_KEY" /boot/config/plugins/varaverk/Orchestrators/watchdog_orchestrator.sh

Check watchdog orchestrator for API key renewal calls


1	<?php
2	header('Content-Type: application/json');
3	header('Cache-Control: no-store, no-cache');
4	require_once dirname(__DIR__) . '/include/config.php';
5
6	$host = vv_detect_host();
7	if (!preg_match('/^host\d+$/', $host)) {
8	    echo json_encode(['ok' => false, 'error' => 'Cannot detect local host']);
9	    exit;
10	}
11
12	echo json_encode(vv_auto_create_api_key($host, $host . '.conf'));
13

<?php
header('Content-Type: application/json');
header('Cache-Control: no-store, no-cache');
require_once dirname(__DIR__) . '/include/config.php';

$host = vv_detect_host();
if (!preg_match('/^host\d+$/', $host)) {
    echo json_encode(['ok' => false, 'error' => 'Cannot detect local host']);
    exit;
}

echo json_encode(vv_auto_create_api_key($host, $host . '.conf'));


grep -n "vv_auto_create_api_key\|UNRAID_API_KEY" /boot/config/plugins/varaverk/Plugin/unraid/include/config.php | head -30

Find PHP API key creation function


220:    $key  = $apiKey ?: ($vars[strtoupper($hostId) . '_UNRAID_API_KEY'] ?? '');
362:function vv_auto_create_api_key(string $hostId, string $confFile): array {
363:    $varName = strtoupper($hostId) . '_UNRAID_API_KEY';


grep -rn "unraid_api_key_renew" /boot/config/plugins/varaverk/Configurations/master.conf 2>/dev/null | head -10

Check if renewal script is in master.conf ARRAY_START_SCRIPTS


/boot/config/plugins/varaverk/Configurations/master.conf:284:        "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register Varaverk API key — registry is ephemeral
/boot/config/plugins/varaverk/Configurations/master.conf:317:        "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register API key if registry lost it


275
276	# ━━━ Array Start ━━━
277	# Scripts launched by array_started.sh when the array comes online.
278	# Launched in order — each as a background process.
279	# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
280	# Continuous scripts (failover) run until array stops.
281	# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
282	# watchdog_orchestrator.sh — NOT launched here.
283	    ARRAY_START_SCRIPTS=(
284	        "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register Varaverk API key — registry is ephemeral
285	        "System_Essentials/conf_sync.sh"              # pull partner confs + push own conf into /tmp/.vv/ RAM cache
286	        "Transcodes/ramdisk_setup.sh"                 # creates ramdisk + symlink before Emby starts
287	        "System_Essentials/docker_syslog_filter.sh"   # suppress veth noise before logs fill
288	        "Plugin/unraid/System_Essentials/php_fpm_max_children.sh"   # WebGUI performance tuning
289	        "System_Essentials/inotify_tuning.sh"         # bump inotify limits — containers miss events if exhausted
290	        "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
291	        "Tools/claude_startup.sh"                     # persist Claude data + binary to appdata; re-symlink on boot
292	        "Fallback/fallback.sh"                        # mutual failover — continuous
293	    )
294
295	# ━━━ Array Stop ━━━
296	# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order.
297	# Run sequentially (foreground) — each must complete before the next starts.
298	# Order matters: user scripts first (prevents new ops), then data movement, then containers.
299	    ARRAY_STOP_SCRIPTS=(
300	        "Plugin/unraid/System_Essentials/user_scripts_stop.sh"        # stop background scripts before they start new ops
301	        "Fallback/fallback.sh --stop"                   # gracefully stop fallback (not caught by user_scripts_stop)
302	        "System_Essentials/rsync_stop.sh --rsync-only"  # kill rsync; skip container recovery (handled below)
303	        "Plugin/unraid/System_Essentials/mover_stop.sh"               # stop mover after rsync (they conflict on same files)
304	        "Docker_Essentials/docker_container_stop.sh"    # stop all containers last
305	    )
306
307	# ━━━ Watchdog Orchestrator ━━━
308	# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
309	# Schedule: * * * * * (every minute)
310	# NOT in ARRAY_START_SCRIPTS — has its own cron entry.
311	# Order matters — resource first (frees pressure), docker second (heals with freed resources),
312	# system third (storage + webgui component health), stability last (last line of defense).
313	    WATCHDOG_ORCHESTRATOR_SCRIPTS=(
314	        "Watchdogs/resource_watchdog.sh"              # reduce system pressure before healing attempts
315	        "Watchdogs/docker_watchdog.sh"                # heal containers with freed resources
316	        "Watchdogs/system_watchdog.sh"                # system component health — storage + webgui
317	        "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register API key if registry lost it
318	        "Watchdogs/stability_watchdog.sh"             # reboot if all else fails — last line of defense
319	    )
320
321	# ━━━ System Watchdog ━━━
322	# system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle.
323	# Called by watchdog_orchestrator.sh — not scheduled directly.
324	    SYSTEM_WATCHDOG_SCRIPTS=(
325	        "Watchdogs/System/storage_watchdog.sh"   # pool growth + runaway log detection
326	        "Plugin/unraid/Watchdogs/System/webgui_watchdog.sh"    # WebGUI availability — nginx → php-fpm → emhttp
327	        "Watchdogs/System/network_watchdog.sh"   # internet, DDNS, Tailscale, NPM proxy
328	    )
329
330	# ━━━ Critical Sync Maintenance ━━━
331	# critical_sync_maintenance.sh runs every 30 minutes.
332	# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check
333	# partnership --check always runs last regardless of rsync gate.
334	# Comment out entries to disable without removing.

# ━━━ Array Start ━━━
# Scripts launched by array_started.sh when the array comes online.
# Launched in order — each as a background process.
# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
# Continuous scripts (failover) run until array stops.
# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
# watchdog_orchestrator.sh — NOT launched here.
    ARRAY_START_SCRIPTS=(
        "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register Varaverk API key — registry is ephemeral
        "System_Essentials/conf_sync.sh"              # pull partner confs + push own conf into /tmp/.vv/ RAM cache
        "Transcodes/ramdisk_setup.sh"                 # creates ramdisk + symlink before Emby starts
        "System_Essentials/docker_syslog_filter.sh"   # suppress veth noise before logs fill
        "Plugin/unraid/System_Essentials/php_fpm_max_children.sh"   # WebGUI performance tuning
        "System_Essentials/inotify_tuning.sh"         # bump inotify limits — containers miss events if exhausted
        "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
        "Tools/claude_startup.sh"                     # persist Claude data + binary to appdata; re-symlink on boot
        "Fallback/fallback.sh"                        # mutual failover — continuous
    )

# ━━━ Array Stop ━━━
# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order.
# Run sequentially (foreground) — each must complete before the next starts.
# Order matters: user scripts first (prevents new ops), then data movement, then containers.
    ARRAY_STOP_SCRIPTS=(
        "Plugin/unraid/System_Essentials/user_scripts_stop.sh"        # stop background scripts before they start new ops
        "Fallback/fallback.sh --stop"                   # gracefully stop fallback (not caught by user_scripts_stop)
        "System_Essentials/rsync_stop.sh --rsync-only"  # kill rsync; skip container recovery (handled below)
        "Plugin/unraid/System_Essentials/mover_stop.sh"               # stop mover after rsync (they conflict on same files)
        "Docker_Essentials/docker_container_stop.sh"    # stop all containers last
    )

# ━━━ Watchdog Orchestrator ━━━
# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
# Schedule: * * * * * (every minute)
# NOT in ARRAY_START_SCRIPTS — has its own cron entry.
# Order matters — resource first (frees pressure), docker second (heals with freed resources),
# system third (storage + webgui component health), stability last (last line of defense).
    WATCHDOG_ORCHESTRATOR_SCRIPTS=(
        "Watchdogs/resource_watchdog.sh"              # reduce system pressure before healing attempts
        "Watchdogs/docker_watchdog.sh"                # heal containers with freed resources
        "Watchdogs/system_watchdog.sh"                # system component health — storage + webgui
        "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register API key if registry lost it
        "Watchdogs/stability_watchdog.sh"             # reboot if all else fails — last line of defense
    )

# ━━━ System Watchdog ━━━
# system_watchdog.sh runs SYSTEM_WATCHDOG_SCRIPTS sequentially each cycle.
# Called by watchdog_orchestrator.sh — not scheduled directly.
    SYSTEM_WATCHDOG_SCRIPTS=(
        "Watchdogs/System/storage_watchdog.sh"   # pool growth + runaway log detection
        "Plugin/unraid/Watchdogs/System/webgui_watchdog.sh"    # WebGUI availability — nginx → php-fpm → emhttp
        "Watchdogs/System/network_watchdog.sh"   # internet, DDNS, Tailscale, NPM proxy
    )

# ━━━ Critical Sync Maintenance ━━━
# critical_sync_maintenance.sh runs every 30 minutes.
# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check
# partnership --check always runs last regardless of rsync gate.
# Comment out entries to disable without removing.


355	    }
356	    ksort($hosts);
357	    return $hosts ?: ['host1' => 'HOST1'];
358	}
359
360	// Create (or overwrite) the Varaverk Unraid API key and write it into host conf.
361	// Returns ['ok'=>true,'key_preview'=>'...'] or ['ok'=>false,'error'=>'...'].
362	function vv_auto_create_api_key(string $hostId, string $confFile): array {
363	    $varName = strtoupper($hostId) . '_UNRAID_API_KEY';
364	    $output  = shell_exec('timeout 10 /usr/local/sbin/unraid-api apikey --name "Varaverk" --create --overwrite --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1');
365	    if (!$output) {
366	        return ['ok' => false, 'error' => 'unraid-api returned no output'];
367	    }
368	    $data = json_decode(trim($output), true);
369	    $key  = $data['key'] ?? null;
370	    if (!$key) {
371	        return ['ok' => false, 'error' => 'No key in response'];
372	    }
373	    $raw = vv_read_conf_raw($confFile);
374	    if ($raw === '') {
375	        return ['ok' => false, 'error' => 'Cannot read ' . $confFile];
376	    }
377	    if (!str_contains($raw, $varName)) {
378	        foreach ([strtoupper($hostId) . '_OWNER_EMAIL', strtoupper($hostId) . '_SSH_KEY'] as $anchor) {
379	            if (str_contains($raw, $anchor)) {
380	                $raw = preg_replace('/^(\s*' . preg_quote($anchor, '/') . '\s*=.*$)/m',
381	                    '$1' . "\n    " . $varName . '=""', $raw, 1);
382	                break;
383	            }
384	        }
385	    }
386	    $raw = preg_replace('/^(\s*' . preg_quote($varName, '/') . '\s*=\s*)"[^"]*"/m',
387	        '${1}"' . $key . '"', $raw);
388	    vv_write_conf_raw($confFile, $raw);
389	    return ['ok' => true, 'key_preview' => substr($key, 0, 8) . '...' . substr($key, -4)];
390	}
391
392	// Local LAN IP via routing table — static-cached per request.
393	// Previously duplicated in include/docker_folders.php and inline in include/docker.php.
394	function vv_local_ip(): string {
395	    static $ip = null;
396	    if ($ip !== null) return $ip;
397	    $ip = trim(shell_exec("ip route get 8.8.8.8 2>/dev/null | awk '/src/{for(i=1;i<=NF;i++)if(\$i==\"src\")print \$(i+1)}'") ?? '');
398	    return $ip;
399	}
400

    }
    ksort($hosts);
    return $hosts ?: ['host1' => 'HOST1'];
}

// Create (or overwrite) the Varaverk Unraid API key and write it into host conf.
// Returns ['ok'=>true,'key_preview'=>'...'] or ['ok'=>false,'error'=>'...'].
function vv_auto_create_api_key(string $hostId, string $confFile): array {
    $varName = strtoupper($hostId) . '_UNRAID_API_KEY';
    $output  = shell_exec('timeout 10 /usr/local/sbin/unraid-api apikey --name "Varaverk" --create --overwrite --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1');
    if (!$output) {
        return ['ok' => false, 'error' => 'unraid-api returned no output'];
    }
    $data = json_decode(trim($output), true);
    $key  = $data['key'] ?? null;
    if (!$key) {
        return ['ok' => false, 'error' => 'No key in response'];
    }
    $raw = vv_read_conf_raw($confFile);
    if ($raw === '') {
        return ['ok' => false, 'error' => 'Cannot read ' . $confFile];
    }
    if (!str_contains($raw, $varName)) {
        foreach ([strtoupper($hostId) . '_OWNER_EMAIL', strtoupper($hostId) . '_SSH_KEY'] as $anchor) {
            if (str_contains($raw, $anchor)) {
                $raw = preg_replace('/^(\s*' . preg_quote($anchor, '/') . '\s*=.*$)/m',
                    '$1' . "\n    " . $varName . '=""', $raw, 1);
                break;
            }
        }
    }
    $raw = preg_replace('/^(\s*' . preg_quote($varName, '/') . '\s*=\s*)"[^"]*"/m',
        '${1}"' . $key . '"', $raw);
    vv_write_conf_raw($confFile, $raw);
    return ['ok' => true, 'key_preview' => substr($key, 0, 8) . '...' . substr($key, -4)];
}

// Local LAN IP via routing table — static-cached per request.
// Previously duplicated in include/docker_folders.php and inline in include/docker.php.
function vv_local_ip(): string {
    static $ip = null;
    if ($ip !== null) return $ip;
    $ip = trim(shell_exec("ip route get 8.8.8.8 2>/dev/null | awk '/src/{for(i=1;i<=NF;i++)if(\$i==\"src\")print \$(i+1)}'") ?? '');
    return $ip;
}


grep -n "vv_read_conf_raw\|vv_write_conf_raw\|CONF_DIR\|conf_dir" /boot/config/plugins/varaverk/Plugin/unraid/include/config.php | head -20

Find conf file path resolution in PHP


9:define('CONF_DIR',    SCRIPTS_DIR . '/Configurations');
45:    $master = vv_read_conf_raw('master.conf');
76:    $localPath = CONF_DIR . '/master.conf';
77:    $master    = vv_read_conf_raw('master.conf');
149:    $master = vv_read_conf_raw('master.conf');
162:function vv_read_conf_raw(string $filename): string {
163:    $path = CONF_DIR . '/' . $filename;
167:function vv_write_conf_raw(string $filename, string $content): bool {
168:    $path = CONF_DIR . '/' . $filename;
187:        foreach (glob(CONF_DIR . '/*.conf') as $f) {
202:        $raw = vv_read_conf_raw($f);
373:    $raw = vv_read_conf_raw($confFile);
388:    vv_write_conf_raw($confFile, $raw);


# Check what the watchdog_orchestrator actually does with the scripts list
grep -A 30 "WATCHDOG_ORCHESTRATOR_SCRIPTS\|run.*script\|for.*script" /boot/config/plugins/varaverk/Orchestrators/watchdog_orchestrator.sh | head -50

Check how watchdog orchestrator runs its scripts


# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
# Schedule: */15 * * * * (every 15 minutes)
#
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
#   Driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf — add, remove, or reorder there.
#   Default: resource_watchdog → docker_watchdog → system_watchdog → unraid_api_key_renew → stability_watchdog
#
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
#   Resource Watchdog first — frees RAM and CPU before healing attempts container restarts.
#                            Containers restarted into a resource-pressured system just fail again.
#   Docker Watchdog second — restarts with pressure already reduced, more likely to stabilise.
#   System Watchdog third  — system component health after containers are healed.
#   API key renew fourth   — self-heals unraid-api registry loss; check-first, silent when valid.
#   Stability Watchdog last — only reboots when all prior layers could not resolve the issue.
#
# ── ARRAY CHECK ───────────────────────────────────────────────────────────────────────────────
#   Exits immediately if /mnt/user is not mounted as shfs (array not started).
#   Watchdogs check Docker containers and storage — meaningless without the array.
#   Prevents false positives and unnecessary reboots when array is stopped or stopping.
#
# ── STARTUP GRACE ─────────────────────────────────────────────────────────────────────────────
#   No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds.
#   Prevents false positives from containers still starting at array launch.
#   Each sub-script enforces this independently — orchestrator exits early to avoid log noise.
#
# ── OVERLAP PROTECTION ────────────────────────────────────────────────────────────────────────
#   acquire_lock() — exits immediately if a prior cycle is still in progress.
#   Prevents pile-up when a cycle runs long (daemon restart attempt = 30s, etc.).
#
# ── REPLACES ──────────────────────────────────────────────────────────────────────────────────
#   Continuous loops previously in system_watchdog.sh and docker_watchdog.sh.
#   Those scripts are now single-pass — this orchestrator provides the cadence.
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   WATCHDOG_ORCHESTRATOR_SCRIPTS         — watchdogs to run, in order
#   WATCHDOG_STARTUP_GRACE                — seconds after boot before checks activate
#   WATCHDOG_ORCHESTRATOR_HEARTBEAT       — periodic heartbeat log toggle
#   WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   watchdog_orchestrator.sh           — normal run (called by cron every 15 minutes)
#   watchdog_orchestrator.sh --dry-run — pass --dry-run to all sub-scripts
#   watchdog_orchestrator.sh --status  — show script paths and current grace state
#   watchdog_orchestrator.sh --log     — verbose output from all sub-scripts
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

source "$ECOSYSTEM_ROOT/load_config.sh"


# Check the PHP create_api_key function - it uses "--name Varaverk" but the shell uses "Varaverk Gmer4Lfe"
# Also check if it always overwrites (generates new key) even when one exists
grep -n "Varaverk\|key_name\|--name\|--create\|--overwrite" /boot/config/plugins/varaverk/Plugin/unraid/include/config.php | head -20

Check PHP API key name mismatch


360:// Create (or overwrite) the Varaverk Unraid API key and write it into host conf.
364:    $output  = shell_exec('timeout 10 /usr/local/sbin/unraid-api apikey --name "Varaverk" --create --overwrite --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1');


# Check where the UI calls create_api_key - is it a deliberate "regenerate" action or routine?
grep -rn "create_api_key\|createApiKey" /boot/config/plugins/varaverk/Plugin/unraid/js/varaverk.js | head -10
grep -rn "create_api_key\|createApiKey" /boot/config/plugins/varaverk/Plugin/unraid/pages/ | head -10

Check how create_api_key is called from UI


/boot/config/plugins/varaverk/Plugin/unraid/pages/setup.php:341:  fetch('/plugins/varaverk/api/create_api_key.php?_=' + Date.now())
/boot/config/plugins/varaverk/Plugin/unraid/pages/partnership.php:167:  fetch('/plugins/varaverk/api/create_api_key.php?_=' + Date.now())


325	    body: new URLSearchParams({action: 'pull'})
326	  }).then(r => r.json()).then(d => {
327	    if (d.ok) {
328	      btn.textContent = '✓ Done';
329	      setTimeout(vvLoadChecklist, 600);
330	    } else {
331	      if (errEl) errEl.textContent = d.error || 'Pull failed';
332	      btn.disabled = false; btn.textContent = 'Retry';
333	    }
334	  }).catch(() => { btn.disabled = false; btn.textContent = 'Retry'; });
335	}
336
337	// ── API key ───────────────────────────────────────────────────────────────────
338	function vvCreateKey(btn) {
339	  const status = document.getElementById('vv-key-status');
340	  btn.disabled = true; btn.textContent = '⟳ Creating…';
341	  fetch('/plugins/varaverk/api/create_api_key.php?_=' + Date.now())
342	    .then(r => r.json()).then(d => {
343	      if (d.ok) {
344	        status.textContent = '✓ Key created — ' + d.key_preview;
345	        status.style.color = '#4a8';
346	        btn.textContent = 'Created ✓'; btn.style.opacity = '.6';
347	        vvLoadChecklist();
348	      } else {
349	        status.textContent = '✗ ' + (d.error || 'Failed');
350	        status.style.color = '#a44';
351	        btn.disabled = false; btn.textContent = 'Retry';
352	      }
353	    }).catch(e => {
354	      status.textContent = '✗ ' + e; status.style.color = '#a44';
355	      btn.disabled = false; btn.textContent = 'Retry';
356	    });
357	}
358
359	// ── Save ──────────────────────────────────────────────────────────────────────

    body: new URLSearchParams({action: 'pull'})
  }).then(r => r.json()).then(d => {
    if (d.ok) {
      btn.textContent = '✓ Done';
      setTimeout(vvLoadChecklist, 600);
    } else {
      if (errEl) errEl.textContent = d.error || 'Pull failed';
      btn.disabled = false; btn.textContent = 'Retry';
    }
  }).catch(() => { btn.disabled = false; btn.textContent = 'Retry'; });
}

// ── API key ───────────────────────────────────────────────────────────────────
function vvCreateKey(btn) {
  const status = document.getElementById('vv-key-status');
  btn.disabled = true; btn.textContent = '⟳ Creating…';
  fetch('/plugins/varaverk/api/create_api_key.php?_=' + Date.now())
    .then(r => r.json()).then(d => {
      if (d.ok) {
        status.textContent = '✓ Key created — ' + d.key_preview;
        status.style.color = '#4a8';
        btn.textContent = 'Created ✓'; btn.style.opacity = '.6';
        vvLoadChecklist();
      } else {
        status.textContent = '✗ ' + (d.error || 'Failed');
        status.style.color = '#a44';
        btn.disabled = false; btn.textContent = 'Retry';
      }
    }).catch(e => {
      status.textContent = '✗ ' + e; status.style.color = '#a44';
      btn.disabled = false; btn.textContent = 'Retry';
    });
}

// ── Save ──────────────────────────────────────────────────────────────────────


155	  delete _vvOnboarding[hostId];
156	  if (_vvPtReload) _vvPtReload();
157	}
158	function vvPtHideDeleteKeys(hostId) {
159	  delete _vvDeleteKeys[hostId];
160	  if (_vvPtReload) _vvPtReload();
161	}
162
163	function vvApiKey(btn) {
164	  const origText = btn.textContent;
165	  btn.disabled = true;
166	  btn.textContent = '⟳ Working…';
167	  fetch('/plugins/varaverk/api/create_api_key.php?_=' + Date.now())
168	    .then(r => {
169	      if (!r.ok || r.status === 0) throw new Error('HTTP ' + r.status + ' ' + r.statusText);
170	      return r.text();
171	    })
172	    .then(text => {
173	      if (!text.trim()) throw new Error('Empty response from server');
174	      return JSON.parse(text);
175	    })
176	    .then(d => {
177	      if (d.ok) {
178	        if (_vvPtReload) _vvPtReload();
179	      } else {
180	        alert('Failed: ' + (d.error ?? 'Unknown error'));
181	        btn.disabled = false; btn.textContent = origText;
182	      }
183	    })
184	    .catch(e => {

  delete _vvOnboarding[hostId];
  if (_vvPtReload) _vvPtReload();
}
function vvPtHideDeleteKeys(hostId) {
  delete _vvDeleteKeys[hostId];
  if (_vvPtReload) _vvPtReload();
}

function vvApiKey(btn) {
  const origText = btn.textContent;
  btn.disabled = true;
  btn.textContent = '⟳ Working…';
  fetch('/plugins/varaverk/api/create_api_key.php?_=' + Date.now())
    .then(r => {
      if (!r.ok || r.status === 0) throw new Error('HTTP ' + r.status + ' ' + r.statusText);
      return r.text();
    })
    .then(text => {
      if (!text.trim()) throw new Error('Empty response from server');
      return JSON.parse(text);
    })
    .then(d => {
      if (d.ok) {
        if (_vvPtReload) _vvPtReload();
      } else {
        alert('Failed: ' + (d.error ?? 'Unknown error'));
        btn.disabled = false; btn.textContent = origText;
      }
    })
    .catch(e => {


# Check if there's a "Varaverk" (no hostname) key in the registry too
timeout 5 /usr/local/sbin/unraid-api apikey --name "Varaverk" --json </dev/null 2>/dev/null || echo "not found"

Check if PHP-created key with different name exists


grep -n "SCRIPT_DIR\|CONF_FILE\|Configurations" /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/php_fpm_max_children.sh | head -15
grep -n "SCRIPT_DIR\|CONF_FILE\|Configurations" /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/mover_stop.sh | head -10
grep -n "SCRIPT_DIR\|CONF_FILE\|Configurations" /boot/config/plugins/varaverk/Plugin/unraid/System_Essentials/user_scripts_stop.sh | head -10

Check other Plugin/unraid scripts for path issues


92:SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
94:source "$SCRIPT_DIR/../../../load_config.sh"
75:SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
77:source "$SCRIPT_DIR/../../../load_config.sh"
71:SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
73:source "$SCRIPT_DIR/../../../load_config.sh"


There's another real bug — two conflicting API keys in the registry. Let me check the checklist and then assess all issues together.


grep -n "api_key\|UNRAID_API_KEY\|unraid_api" /boot/config/plugins/varaverk/Plugin/unraid/api/checklist.php | head -20

Check how the checklist validates the API key


34:$apiKey = trim(vv_parse_conf_scalar($confRaw, $hostIdUp . '_UNRAID_API_KEY'));
36:    'id'     => 'api_key',


360	// Create (or overwrite) the Varaverk Unraid API key and write it into host conf.
361	// Returns ['ok'=>true,'key_preview'=>'...'] or ['ok'=>false,'error'=>'...'].
362	function vv_auto_create_api_key(string $hostId, string $confFile): array {
363	    $varName = strtoupper($hostId) . '_UNRAID_API_KEY';
364	    $output  = shell_exec('timeout 10 /usr/local/sbin/unraid-api apikey --name "Varaverk" --create --overwrite --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1');
365	    if (!$output) {
366	        return ['ok' => false, 'error' => 'unraid-api returned no output'];
367	    }
368	    $data = json_decode(trim($output), true);
369	    $key  = $data['key'] ?? null;
370	    if (!$key) {
371	        return ['ok' => false, 'error' => 'No key in response'];
372	    }
373	    $raw = vv_read_conf_raw($confFile);
374	    if ($raw === '') {
375	        return ['ok' => false, 'error' => 'Cannot read ' . $confFile];
376	    }
377	    if (!str_contains($raw, $varName)) {
378	        foreach ([strtoupper($hostId) . '_OWNER_EMAIL', strtoupper($hostId) . '_SSH_KEY'] as $anchor) {
379	            if (str_contains($raw, $anchor)) {
380	                $raw = preg_replace('/^(\s*' . preg_quote($anchor, '/') . '\s*=.*$)/m',
381	                    '$1' . "\n    " . $varName . '=""', $raw, 1);
382	                break;
383	            }
384	        }
385	    }
386	    $raw = preg_replace('/^(\s*' . preg_quote($varName, '/') . '\s*=\s*)"[^"]*"/m',
387	        '${1}"' . $key . '"', $raw);
388	    vv_write_conf_raw($confFile, $raw);
389	    return ['ok' => true, 'key_preview' => substr($key, 0, 8) . '...' . substr($key, -4)];
390	}
391

// Create (or overwrite) the Varaverk Unraid API key and write it into host conf.
// Returns ['ok'=>true,'key_preview'=>'...'] or ['ok'=>false,'error'=>'...'].
function vv_auto_create_api_key(string $hostId, string $confFile): array {
    $varName = strtoupper($hostId) . '_UNRAID_API_KEY';
    $output  = shell_exec('timeout 10 /usr/local/sbin/unraid-api apikey --name "Varaverk" --create --overwrite --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1');
    if (!$output) {
        return ['ok' => false, 'error' => 'unraid-api returned no output'];
    }
    $data = json_decode(trim($output), true);
    $key  = $data['key'] ?? null;
    if (!$key) {
        return ['ok' => false, 'error' => 'No key in response'];
    }
    $raw = vv_read_conf_raw($confFile);
    if ($raw === '') {
        return ['ok' => false, 'error' => 'Cannot read ' . $confFile];
    }
    if (!str_contains($raw, $varName)) {
        foreach ([strtoupper($hostId) . '_OWNER_EMAIL', strtoupper($hostId) . '_SSH_KEY'] as $anchor) {
            if (str_contains($raw, $anchor)) {
                $raw = preg_replace('/^(\s*' . preg_quote($anchor, '/') . '\s*=.*$)/m',
                    '$1' . "\n    " . $varName . '=""', $raw, 1);
                break;
            }
        }
    }
    $raw = preg_replace('/^(\s*' . preg_quote($varName, '/') . '\s*=\s*)"[^"]*"/m',
        '${1}"' . $key . '"', $raw);
    vv_write_conf_raw($confFile, $raw);
    return ['ok' => true, 'key_preview' => substr($key, 0, 8) . '...' . substr($key, -4)];
}


# Get the hostname suffix the shell script uses
hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//'

Get hostname suffix used in key name


Here's the full picture of what I found:

---

**Bug 1 & 2 (fixed):** Already done — path bug and early-exit-without-write in `unraid_api_key_renew.sh`.

---

**Bug 3: Two conflicting API keys in the registry**

The PHP `vv_auto_create_api_key` (called by the Setup page "Create API Key" button) registers under the name `"Varaverk"`. The shell script registers under `"Varaverk Gmer4Lfe"`. They're tracked as separate registry entries. Right now both exist:

- `"Varaverk Gmer4Lfe"` → `1825c3a2...4dbb` (shell's key, currently in conf)
- `"Varaverk"` → `e6acb2df...ef4b9` (PHP's key, orphaned)

If anyone clicks the Setup/Partnership "Create API Key" button, the PHP writes its key to conf — and then the watchdog (every 15 min) sees a mismatch and overwrites it back to the shell's key. So the UI button is silently broken.

Fix: align the PHP to use the same hostname-suffixed name:


Wasted call — file unchanged since your last Read. Refer to that earlier tool_result instead.


Edit

$output  = shell_exec('timeout 10 /usr/local/sbin/unraid-api apikey --name "Varaverk" --create --overwrite --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1');

    $hostname = trim((string)shell_exec("hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//'")) ?: 'Varaverk';
    $keyName  = 'Varaverk ' . $hostname;
    $output   = shell_exec('timeout 10 /usr/local/sbin/unraid-api apikey --name ' . escapeshellarg($keyName) . ' --create --overwrite --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1');


The file /boot/config/plugins/varaverk/Plugin/unraid/include/config.php has been updated successfully. (file state is current in your context — no need to Read it back)

    $hostname = trim((string)shell_exec("hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//'")) ?: 'Varaverk';
    $keyName  = 'Varaverk ' . $hostname;
    $output   = shell_exec('timeout 10 /usr/local/sbin/unraid-api apikey --name ' . escapeshellarg($keyName) . ' --create --overwrite --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1');

// Returns ['ok'=>true,'key_preview'=>'...'] or ['ok'=>false,'error'=>'...'].

function vv_auto_create_api_key(string $hostId, string $confFile): array {

$varName = strtoupper($hostId) . '_UNRAID_API_KEY';

-    $output  = shell_exec('timeout 10 /usr/local/sbin/unraid-api apikey --name "Varaverk" --create --overwrite --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1');

+    $hostname = trim((string)shell_exec("hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//'")) ?: 'Varaverk';

+    $keyName  = 'Varaverk ' . $hostname;

+    $output   = shell_exec('timeout 10 /usr/local/sbin/unraid-api apikey --name ' . escapeshellarg($keyName) . ' --create --overwrite --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1');

return ['ok' => false, 'error' => 'unraid-api returned no output'];


Now the PHP button and the shell script track the same registry entry. The stale `"Varaverk"` orphan key in the registry is harmless but can be left alone (it's read-only from our side).

---

**Bug 4: PHP button always regenerates (`--overwrite`)**

Every click on "Create API Key" in the UI issues `--create --overwrite`, which generates a brand-new key value and invalidates the old one. This means any active API calls using the old key break mid-cycle. Since the button label is "Create" not "Regenerate", it should check first.

But looking at `unraid_api_key_renew.sh` — the shell already does the idempotent check-first pattern, and runs every 15 min. The PHP button is really only used for initial setup or manual recovery, so the overwrite behaviour is arguably intentional there. I'd leave this one as-is unless you want to change the UX.

---

**No other path bugs** in the other `Plugin/unraid/System_Essentials/` scripts — `php_fpm_max_children.sh`, `mover_stop.sh`, `user_scripts_stop.sh` all have the correct `../../../` traversal.

**Summary of changes made:** 3 fixes — wrong `../` depth in `unraid_api_key_renew.sh`, early-exit skipping the conf write, and PHP key name not matching shell key name.


lets just scan and look for bug


Agent

Bug scan across varaverk plugin

Prompt:
Do a thorough bug scan of the Varaverk Unraid plugin at /boot/config/plugins/varaverk/. This is a self-healing two-server Unraid home media ecosystem plugin, written in bash and PHP.

I'm looking for real bugs — logic errors, incorrect paths, race conditions, broken self-healing flows, missing variable checks, silent failures, auth gaps, and anything else that would cause something to not work correctly or silently do the wrong thing. Not style issues, not hypothetical future problems — actual defects.

Focus areas (in priority order):

1. **Bash scripts** — wrong SCRIPT_DIR relative paths (depth mismatch), unquoted variables in conditions/loops, variables used before being set, exit codes ignored when they matter, lock files not cleaned up, subshell failures swallowed silently, array handling bugs (word splitting), declare -A used in subshells.

2. **PHP API files** in Plugin/unraid/api/ and Plugin/unraid/include/ — auth checks missing or bypassable, shell_exec with unescaped user input, logic bugs in conf read/write, any place a function can return empty/null and the caller doesn't check.

3. **Orchestrator/watchdog scripts** — scripts that should be one-shot behaving as continuous, wrong loop conditions, heartbeat/lock file logic, timing assumptions that can break.

4. **Cross-cutting** — conf variables read before load_config.sh is sourced, functions called with wrong arg count, files assumed to exist without checking.

For each bug: file path, line number(s), what the bug is, and why it matters. Skip anything minor or purely stylistic. Group by file. Report up to ~20 highest-confidence bugs.