API key: use space separator, fix error-capture bug; add conf_sync and missing files from dev

unRAID API rejects underscores and hyphens in key names — only letters, numbers,
and spaces allowed. Varaverk_HOST1 / Varaverk_Gmer4Lfe both fail; now uses
"Varaverk <hostname>" (space). Also adds monitor_remote.php, conf_populate.sh,
and conf_sync.sh from dev branch that were missing from production.
This commit is contained in:
Gmer4Lfe
2026-06-04 16:17:47 -04:00
parent a41adfe020
commit b7894681e6
4 changed files with 495 additions and 185 deletions
+138
View File
@@ -0,0 +1,138 @@
#!/bin/bash
# ==============================================================================================
# ============================= Conf Cache Sync ================================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Maintains a RAM-resident conf cache at /tmp/.vv/config/cached/.confs/.
# Credentials and partner keys live in RAM only — never on disk across hosts.
#
# On array start (default / --array-start):
# 1. Copy own conf to local cache
# 2. Pull each available partner's conf from their disk → local cache
# 3. Push own conf to each available partner's /tmp/.vv/ cache
#
# On conf save (--push-only):
# Fast path — push updated own conf to all partners' /tmp/.vv/ cache only.
# No pulls, no local cache rebuild.
#
# Cache is /tmp (tmpfs) — cleared every reboot, repopulated by this script
# on next array start. Scripts source from cache for partner vars; own vars
# always come from disk (load_config.sh skips cached copy of own conf).
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# conf_sync.sh Full sync: pull from all partners + push to all partners
# conf_sync.sh --push-only Push own conf to all partners (fast, for conf-save hook)
# conf_sync.sh --dry-run Show what would happen, no changes
# conf_sync.sh --log Verbose output
#
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
PUSH_ONLY=false
FILTERED_ARGS=()
for arg in "$@"; do
case "$arg" in
--push-only) PUSH_ONLY=true ;;
*) FILTERED_ARGS+=("$arg") ;;
esac
done
parse_args "${FILTERED_ARGS[@]}"
detect_hosts
CACHE_DIR="/tmp/.vv/config/cached/.confs"
MY_CONF="$SCRIPTS_ROOT/Configurations/${MY_ID,,}.conf"
SSH_TIMEOUT=10
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# ── Ensure cache dir exists ───────────────────────────────────────────────────
if [[ "$DRY_RUN" == false ]]; then
mkdir -p "$CACHE_DIR"
fi
# ── Copy own conf into local cache ───────────────────────────────────────────
if [[ "$PUSH_ONLY" == false ]]; then
if [[ -f "$MY_CONF" ]]; then
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would copy $(basename "$MY_CONF")$CACHE_DIR/"
else
cp "$MY_CONF" "$CACHE_DIR/${MY_ID,,}.conf" && \
log "Own conf cached ✅" || warn "Failed to cache own conf"
fi
else
warn "Own conf not found: $MY_CONF"
fi
fi
# ── Per-partner sync ──────────────────────────────────────────────────────────
PUSHED=0
PULLED=0
FAILED=0
for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
partner_host="${!host_var}"
[[ -z "$partner_host" ]] && continue
[[ "${host_var,,}" == "${MY_ID,,}" ]] && continue
partner_slot="${host_var,,}" # e.g. host2
partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
if [[ -z "$partner_ip" ]]; then
warn "$partner_host — cannot resolve Tailscale IP, skipping"
(( FAILED++ ))
continue
fi
# ── Pull: grab partner's conf from their disk → our local cache ──────────
if [[ "$PUSH_ONLY" == false ]]; then
remote_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would pull $partner_host:$remote_conf$CACHE_DIR/${partner_slot}.conf"
elif timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
"root@${partner_ip}:${remote_conf}" \
"$CACHE_DIR/${partner_slot}.conf" 2>/dev/null; then
log "Pulled ${partner_slot}.conf from $partner_host"
(( PULLED++ ))
else
warn "Could not pull ${partner_slot}.conf from $partner_host"
(( FAILED++ ))
fi
fi
# ── Push: send own conf to partner's /tmp/.vv/ cache ────────────────────
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would push ${MY_ID,,}.conf → $partner_host:/tmp/.vv/config/cached/.confs/"
continue
fi
# Ensure partner's cache dir exists, then SCP own conf into it
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
"root@${partner_ip}" "mkdir -p '$CACHE_DIR'" 2>/dev/null
if timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
"$MY_CONF" \
"root@${partner_ip}:${CACHE_DIR}/${MY_ID,,}.conf" 2>/dev/null; then
log "Pushed ${MY_ID,,}.conf to $partner_host"
(( PUSHED++ ))
else
warn "Could not push to $partner_host"
(( FAILED++ ))
fi
done
# ── Summary ───────────────────────────────────────────────────────────────────
if [[ "$PUSH_ONLY" == true ]]; then
info "Conf push complete — pushed to $PUSHED host(s)${FAILED:+, $FAILED failed}"
else
info "Conf sync complete — pulled $PULLED, pushed $PUSHED${FAILED:+, $FAILED failed}"
fi
+111 -185
View File
@@ -5,223 +5,149 @@
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Manages per-host named API keys in each machine's Unraid registry so every
# host can call every other host's GraphQL API directly for real-time monitoring.
# Creates/overwrites the Varaverk API key in the unraid-api service registry at
# array start. The registry is ephemeral — OS updates and service restarts clear
# it. This script re-registers the key every boot so Varaverk's enhanced
# monitoring self-heals without manual intervention.
#
# KEY MODEL
# ─────────────────────────────────────────────────────────────────────────────
# HOST1's Unraid registry holds:
# Varaverk_HOST1 — HOST1's own key (for HOST1 to call its own API)
# Varaverk_HOST2 — HOST2's access key to HOST1's API (removed on offboard)
#
# HOST2's Unraid registry holds:
# Varaverk_HOST2 — HOST2's own key
# Varaverk_HOST1 — HOST1's access key to HOST2's API (removed on offboard)
#
# host1.conf holds:
# HOST1_UNRAID_API_KEY — value of Varaverk_HOST1 from HOST1's registry
# HOST2_UNRAID_API_KEY — value of Varaverk_HOST1 from HOST2's registry
# (HOST1 uses this to call HOST2's GraphQL directly)
#
# host2.conf holds the mirror:
# HOST2_UNRAID_API_KEY — value of Varaverk_HOST2 from HOST2's registry
# HOST1_UNRAID_API_KEY — value of Varaverk_HOST2 from HOST1's registry
# (HOST2 uses this to call HOST1's GraphQL directly)
#
# ── vv_remote_hosts_stats() picks up HOST2_UNRAID_API_KEY from host1.conf
# automatically — direct GraphQL, no SSH needed for remote monitoring.
# Also updates HOST*_UNRAID_API_KEY in the local host conf so the partnership
# page always reflects the live key value.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# unraid_api_key_renew.sh
# Local only — runs at array start. Re-registers Varaverk_HOST1 if the
# ephemeral registry lost it. Fast, no SSH.
# Renew the key. Silent on success.
#
# unraid_api_key_renew.sh --all-hosts
# Full cross-host key setup. Run once from Settings → API Keys → Setup.
# For each partner:
# • Creates Varaverk_HOST1 on PARTNER's registry → stores value in local host*.conf
# • Creates Varaverk_HOST2 on LOCAL registry → SSHes value to partner's host*.conf
# unraid_api_key_renew.sh --dry-run
# Show what would happen — no changes made.
#
# unraid_api_key_renew.sh --dry-run — show what would happen
# unraid_api_key_renew.sh --log — verbose output
# unraid_api_key_renew.sh --log
# Verbose output.
#
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@"
acquire_lock
detect_hosts
# Parse --all-hosts
ALL_HOSTS=false
for _arg in "$@"; do [[ "$_arg" == "--all-hosts" ]] && ALL_HOSTS=true; done
unset _arg
LOCAL_CONF="$SCRIPT_DIR/../Configurations/${MY_ID,,}.conf"
LOCAL_KEY_NAME="Varaverk_${MY_ID}"
LOCAL_VAR="${MY_ID}_UNRAID_API_KEY"
log "$ICON_GEAR Local conf: $LOCAL_CONF"
log "$ICON_GEAR Key name: $LOCAL_KEY_NAME"
log "$ICON_GEAR All hosts: $ALL_HOSTS"
if [[ ! -f "$LOCAL_CONF" ]]; then
error "Conf file not found: $LOCAL_CONF"; exit 1
fi
[[ "$DRY_RUN" == true ]] && {
warn "DRY RUN — would check/create $LOCAL_KEY_NAME in registry and sync to host conf"
[[ "$ALL_HOSTS" == true ]] && warn "DRY RUN — would also SSH all partners for cross-host key setup"
exit 0
}
# ── Helper: ensure a named key exists in the LOCAL registry, return its value ──
_ensure_local_key() {
local name="$1"
local existing key
existing=$(timeout 5 /usr/local/sbin/unraid-api apikey --name "$name" --json </dev/null 2>/dev/null)
key=$(echo "$existing" | jq -r '.key // empty' 2>/dev/null)
if [[ -n "$key" ]]; then
echo "$key"; return 0
fi
local raw
raw=$(timeout 10 /usr/local/sbin/unraid-api apikey \
--name "$name" --create --overwrite \
--description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1)
key=$(echo "$raw" | jq -r '.key // empty' 2>/dev/null)
[[ -z "$key" ]] && { error "Failed to create $name: ${raw:0:200}"; return 1; }
echo "$key"
}
# ── Helper: write HOST*_UNRAID_API_KEY into a conf file ────────────────────────
_write_key() {
local conf="$1" var="$2" key="$3"
[[ ! -f "$conf" ]] && return 1
if grep -q "^\s*${var}\s*=" "$conf"; then
sed -i "s|^\(\s*${var}\s*=\s*\)\"[^\"]*\"|\1\"${key}\"|" "$conf"
else
printf ' %s="%s"\n' "$var" "$key" >> "$conf"
fi
}
# ── Helper: ensure a named key exists on a REMOTE registry via SSH ─────────────
_ensure_remote_key() {
local ssh_key="$1" remote_ip="$2" key_name="$3"
ssh -i "$ssh_key" \
-o ConnectTimeout=10 -o StrictHostKeyChecking=no -o BatchMode=yes \
"root@${remote_ip}" "
EXISTING=\$(timeout 5 /usr/local/sbin/unraid-api apikey --name '${key_name}' --json </dev/null 2>/dev/null)
KEY=\$(echo \"\$EXISTING\" | jq -r '.key // empty' 2>/dev/null)
if [[ -n \"\$KEY\" ]]; then
echo \"\$KEY\"
else
timeout 10 /usr/local/sbin/unraid-api apikey \\
--name '${key_name}' --create --overwrite \\
--description 'Varaverk plugin' --roles ADMIN --json </dev/null 2>/dev/null \\
| jq -r '.key // empty' 2>/dev/null
fi
" 2>/dev/null | tr -d '[:space:]'
}
# ──────────────────────────────────────────────────────────────────────────────
# Step 1 — Local key (always)
# ──────────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_GEAR Local — $LOCAL_KEY_NAME ━━━"
CONF_FILE="$SCRIPT_DIR/../Configurations/${MY_ID,,}.conf"
VAR_NAME="${MY_ID}_UNRAID_API_KEY"
LOCAL_KEY=$(_ensure_local_key "$LOCAL_KEY_NAME")
if [[ -z "$LOCAL_KEY" ]]; then
error "Could not obtain $LOCAL_KEY_NAME from registry"
# Key name: "Varaverk <hostname>" stripping any unraid- prefix
# Space separator — unRAID API only allows letters, numbers, and spaces
HOSTNAME_SUFFIX=$(hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//' || hostname -s)
KEY_NAME="Varaverk ${HOSTNAME_SUFFIX}"
log "$ICON_GEAR Conf file: $CONF_FILE"
log "$ICON_GEAR Key var: $VAR_NAME"
log "$ICON_GEAR Key name: $KEY_NAME"
if [[ ! -f "$CONF_FILE" ]]; then
error "Conf file not found: $CONF_FILE"
exit 1
fi
# Sync to conf only if changed
CONF_KEY=$(grep "^\s*${LOCAL_VAR}\s*=" "$LOCAL_CONF" 2>/dev/null | sed 's/.*="\(.*\)".*/\1/' | tr -d '[:space:]')
if [[ "$CONF_KEY" != "$LOCAL_KEY" ]]; then
_write_key "$LOCAL_CONF" "$LOCAL_VAR" "$LOCAL_KEY"
warn "Synced $LOCAL_VAR${LOCAL_KEY:0:8}...${LOCAL_KEY: -4}"
else
echo " $LOCAL_VAR valid ✅ — ${LOCAL_KEY:0:8}...${LOCAL_KEY: -4}"
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would check registry for $KEY_NAME, renew only if missing"
exit 0
fi
[[ "$ALL_HOSTS" != true ]] && exit 0
# ──────────────────────────────────────────────────────────────────────────────
# Check if key already exists in the unraid-api registry before creating.
# --overwrite generates a new key value every time, invalidating the old one.
# Only renew if the registry has lost it.
log "Checking unraid-api registry for $KEY_NAME..."
EXISTING=$(timeout 5 /usr/local/sbin/unraid-api apikey --name "$KEY_NAME" --json </dev/null 2>/dev/null)
KEY=$(echo "$EXISTING" | jq -r '.key // empty' 2>/dev/null)
if [[ -n "$KEY" ]]; then
PREVIEW="${KEY:0:8}...${KEY: -4}"
echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
log "Key found in registry — no renewal needed"
exit 0
fi
log "Key not found in registry — creating new key..."
RAW=$(timeout 10 /usr/local/sbin/unraid-api apikey \
--name "$KEY_NAME" --create --overwrite \
--description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1)
if [[ -z "$RAW" ]]; then
error "unraid-api returned no output"
exit 1
fi
KEY=$(echo "$RAW" | jq -r '.key // empty' 2>/dev/null)
if [[ -z "$KEY" ]]; then
error "No key in unraid-api response: ${RAW:0:200}"
exit 1
fi
# ──────────────────────────────────────────────────────────────────────────────
# Step 2 — Cross-host key setup (--all-hosts)
# ──────────────────────────────────────────────────────────────────────────────
echo ""
echo "━━━ $ICON_SYNC Cross-host key setup ━━━"
if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
else
echo " ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
fi
PARTNER_OK=0; PARTNER_FAIL=0
PREVIEW="${KEY:0:8}...${KEY: -4}"
log "Writing new key to: $CONF_FILE"
warn "API key renewed ✅ — $VAR_NAME = $PREVIEW (registry had lost it)"
for host_var in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
[[ "$host_var" == "$MY_ID" ]] && continue
hostname="${!host_var:-}"; [[ -z "$hostname" ]] && continue
# ── Push renewed key into each partner's OWN conf ─────────────────────────────
# Each host's conf is its complete keychest — no cross-host conf files needed.
# SSH_KEY is set by detect_hosts() — this server's outbound private key.
if [[ -z "$SSH_KEY" ]]; then
log "No SSH key configured — skipping partner push"
exit 0
fi
r_var="${host_var}_UNRAID_API_KEY" # e.g. HOST2_UNRAID_API_KEY
r_my_key_name="Varaverk_${MY_ID}" # key HOST1 uses on HOST2's registry
r_their_key_name="Varaverk_${host_var}" # key HOST2 uses on HOST1's registry
r_conf="/boot/config/plugins/varaverk/Configurations/${host_var,,}.conf"
for host_var in $(compgen -v | grep -E '^HOST[0-9]+$'); do
partner_host="${!host_var}"
[[ -z "$partner_host" ]] && continue
[[ "${host_var,,}" == "${MY_ID,,}" ]] && continue
echo ""
echo " $host_var ($hostname)"
partner_slot="${host_var,,}" # e.g. host2
partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
[[ -z "$partner_ip" ]] && { log "Cannot resolve IP for $partner_host — skipping"; continue; }
REMOTE_IP=$(resolve_tailscale_ip "$hostname")
if [[ -z "$REMOTE_IP" ]]; then
warn " $host_var: cannot resolve Tailscale IP — skipping"
(( PARTNER_FAIL++ )); continue
# Target is the partner's OWN conf on their machine
partner_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
tmp=$(mktemp /tmp/vv_kp_XXXXXX.sh)
remote="/tmp/vv_kp_${RANDOM}.sh"
chmod 700 "$tmp"
# Key stays in the temp file — never appears in SSH command args
cat > "$tmp" <<PUSHSCRIPT
#!/bin/sh
target='${partner_conf}'
if grep -q "\b${VAR_NAME}\b" "\$target" 2>/dev/null; then
sed -i 's|^\(\\s*${VAR_NAME}\\s*=\\s*\)"[^"]*"|\1"${KEY}"|' "\$target"
else
printf ' ${VAR_NAME}="%s"\n' '${KEY}' >> "\$target"
fi
echo ok
PUSHSCRIPT
if timeout 10 scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
-o StrictHostKeyChecking=no "$tmp" "root@${partner_ip}:${remote}" 2>/dev/null; then
if timeout 10 ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
-o StrictHostKeyChecking=no "root@${partner_ip}" \
"bash '${remote}'; rc=\$?; rm -f '${remote}'; exit \$rc" 2>/dev/null | grep -q ok; then
log "Key pushed to $partner_host"
else
warn "Key push to $partner_host failed — they can create their own copy"
fi
else
warn "SCP to $partner_host failed — skipping"
fi
# ── A: Get HOST1's access key from HOST2's registry ────────────────────────
# Creates Varaverk_HOST1 on HOST2 → store in host1.conf as HOST2_UNRAID_API_KEY
log " Creating $r_my_key_name on $host_var registry…"
MY_KEY_ON_REMOTE=$(_ensure_remote_key "$SSH_KEY" "$REMOTE_IP" "$r_my_key_name")
if [[ -z "$MY_KEY_ON_REMOTE" ]]; then
warn " $host_var: could not create $r_my_key_name on their registry"
(( PARTNER_FAIL++ )); continue
fi
_write_key "$LOCAL_CONF" "$r_var" "$MY_KEY_ON_REMOTE"
echo " A ✅ $r_my_key_name on $host_var → stored as $r_var in ${MY_ID,,}.conf"
# ── B: Create HOST2's access key on HOST1's registry ──────────────────────
# Creates Varaverk_HOST2 on HOST1 → store in host2.conf as HOST1_UNRAID_API_KEY
log " Creating $r_their_key_name on local registry…"
THEIR_KEY_ON_LOCAL=$(_ensure_local_key "$r_their_key_name")
if [[ -z "$THEIR_KEY_ON_LOCAL" ]]; then
warn " Could not create $r_their_key_name on local registry"
(( PARTNER_FAIL++ )); continue
fi
# Write to partner's conf via SSH
ssh -i "$SSH_KEY" \
-o ConnectTimeout=10 -o StrictHostKeyChecking=no -o BatchMode=yes \
"root@${REMOTE_IP}" "
CONF='$r_conf'
VAR='${MY_ID}_UNRAID_API_KEY'
KEY='$THEIR_KEY_ON_LOCAL'
[[ ! -f \"\$CONF\" ]] && exit 1
if grep -q \"^\s*\${VAR}\s*=\" \"\$CONF\"; then
sed -i \"s|\(\s*\${VAR}\s*=\s*\)\\\"[^\\\"]*\\\"|\1\\\"\${KEY}\\\"|\" \"\$CONF\"
else
printf ' %s=\"%s\"\n' \"\$VAR\" \"\$KEY\" >> \"\$CONF\"
fi
" 2>/dev/null \
&& echo " B ✅ $r_their_key_name on local → stored as ${MY_ID}_UNRAID_API_KEY in ${host_var,,}.conf" \
|| warn " B: could not write ${MY_ID}_UNRAID_API_KEY to ${host_var,,}.conf"
(( PARTNER_OK++ ))
rm -f "$tmp"
done
echo ""
echo "━━━━━ $ICON_SUMMARY Key Setup Summary ━━━━━"
echo " Local key: $LOCAL_VAR"
echo " Partner setup: $PARTNER_OK ok · $PARTNER_FAIL failed"
echo ""
echo " HOST1 can now call HOST2's GraphQL directly using HOST2_UNRAID_API_KEY"
echo " HOST2 can now call HOST1's GraphQL directly using HOST1_UNRAID_API_KEY"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"