#!/bin/bash
# ==============================================================================================
# ============================= Media Shares Permissions =======================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Apply nobody:users ownership and correct permissions to all media shares.
# Runs daily as the first job in the maintenance window — arr cleanup depends
# on correct ownership to rename and delete files.
#
# Now a proper daily failsafe: files arrive with wrong ownership from rsync
# without --chown, manual admin copies, containers with unconfigured PUID/PGID,
# or unRAID environment resets after updates.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
#   acquire_lock "wait"  — wait if previous run still active (large share scans)
#   detect_hosts()       — correct share list per host via MY_ID aliases
#   Empty array guard    — warns and exits cleanly if no shares configured
#   Folder existence     — skips missing shares with warning, continues others
#   Separate passes      — directories and files chmod'd separately for correctness
#   platform_require_cmd  — notify script validated before use
#   Silent by default    — only failures produce output, success is silent
#
# Diagnostic — high corrected count on every run means a container has wrong PUID/PGID:
#   Correct values on unRAID: PUID=99 (nobody) PGID=100 (users)
#   Common culprits: SABnzbd, qBittorrent, slskd — check these first
#   Once fixed, this script should correct 0 files per run (pure failsafe)
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# host*.conf
#
#   HOST*_MEDIA_PERMISSION_SHARES — shares this host applies permissions to
#   Aliased by detect_hosts() — script uses MEDIA_PERMISSION_SHARES
#
# master.conf
#
#   PERMISSIONS_DIR_MODE   — directory permissions (default 755)
#   PERMISSIONS_FILE_MODE  — file permissions (default 664)
#   PERMISSIONS_OWNER      — ownership applied to all files (default nobody:users)
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
#   media_shares_permissions.sh              — normal run
#   media_shares_permissions.sh --dry-run    — preview without making changes
#   media_shares_permissions.sh --log        — verbose output
#   media_shares_permissions.sh --status     — show config and exit
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

platform_require_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

acquire_lock "wait"

# detect_hosts() sets MY_ID and aliases HOST*_MEDIA_PERMISSION_SHARES
detect_hosts

# Empty array guard
if [[ ${#MEDIA_PERMISSION_SHARES[@]} -eq 0 ]]; then
    warn "MEDIA_PERMISSION_SHARES is empty for $MY_ID — nothing to do"
    warn "Check HOST*_MEDIA_PERMISSION_SHARES in host*.conf"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST    Identity:   $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_PERMS   Dir mode:   ${PERMISSIONS_DIR_MODE:-755}"
    echo "$ICON_PERMS   File mode:  ${PERMISSIONS_FILE_MODE:-664}"
    echo "$ICON_PERMS   Owner:      $PERMISSIONS_OWNER"
    echo "$ICON_PERMS   Shares:     ${#MEDIA_PERMISSION_SHARES[@]}"
    echo ""
    for share in "${MEDIA_PERMISSION_SHARES[@]}"; do
        local_status="missing"
        [[ -d "$share" ]] && local_status="exists"
        echo "    $share — $local_status"
    done
    echo "$ICON_GEAR    Dry Run:    $DRY_RUN"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Apply Permissions ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_PERMS Media Permissions — $MY_ID ━━━"
log "Dir mode:  ${PERMISSIONS_DIR_MODE:-755}"
log "File mode: ${PERMISSIONS_FILE_MODE:-664}"
log "Owner:     $PERMISSIONS_OWNER"
log "Shares:    ${#MEDIA_PERMISSION_SHARES[@]}"
echo ""

START=$(date +%s)
FAILED=()
UPDATED=()
SKIPPED=()
TOTAL_DIRS_FIXED=0
TOTAL_FILES_FIXED=0

for SHARE in "${MEDIA_PERMISSION_SHARES[@]}"; do
    SHARE_NAME=$(basename "$SHARE")

    if [[ ! -d "$SHARE" ]]; then
        warn "$SHARE_NAME not found — skipping"
        SKIPPED+=("$SHARE_NAME")
        continue
    fi

    if [[ "$DRY_RUN" == true ]]; then
        # Count what would be changed without making changes
        DIR_COUNT=$(find "$SHARE" -type d ! -perm "${PERMISSIONS_DIR_MODE:-755}" \
            2>/dev/null | wc -l)
        FILE_COUNT=$(find "$SHARE" -type f ! -perm "${PERMISSIONS_FILE_MODE:-664}" \
            2>/dev/null | wc -l)
        OWNER_COUNT=$(find "$SHARE" ! -user nobody -o ! -group users \
            2>/dev/null | wc -l)
        warn "DRY RUN — $SHARE_NAME: $DIR_COUNT dirs, $FILE_COUNT files, $OWNER_COUNT ownership fixes needed"
        continue
    fi

    log "Updating $SHARE_NAME..."

    CHMOD_DIR_OK=true
    CHMOD_FILE_OK=true
    CHOWN_OK=true

    # Count files with wrong ownership before fixing (diagnostic)
    WRONG_OWNER=$(find "$SHARE" \( ! -user nobody -o ! -group users \) \
        2>/dev/null | wc -l)

    # Apply ownership first — affects all files and directories
    chown -R "$PERMISSIONS_OWNER" "$SHARE" 2>/dev/null || CHOWN_OK=false

    # Apply directory permissions — separate pass for correctness
    # Directories need execute bit — different from files
    find "$SHARE" -type d -exec chmod "${PERMISSIONS_DIR_MODE:-755}" {} + \
        2>/dev/null || CHMOD_DIR_OK=false

    # Apply file permissions — no execute bit on media files
    # Ignore "No such file" errors: race condition with volatile dirs (e.g. Emby transcodes)
    _chmod_errs=$(find "$SHARE" -type f -exec chmod "${PERMISSIONS_FILE_MODE:-664}" {} + 2>&1 | \
        grep -v "No such file or directory" | grep -c "chmod:" || true)
    [[ "$_chmod_errs" -gt 0 ]] && CHMOD_FILE_OK=false

    if [[ "$CHMOD_DIR_OK" == true && \
          "$CHMOD_FILE_OK" == true && \
          "$CHOWN_OK" == true ]]; then
        log "$ICON_UNLOCKED $SHARE_NAME — permissions applied"
        UPDATED+=("$SHARE_NAME")
        # Log diagnostic if many files had wrong ownership
        if [[ "$WRONG_OWNER" -gt 0 ]]; then
            warn "$SHARE_NAME — corrected $WRONG_OWNER file(s) with wrong ownership"
            warn "If this is high, check container PUID/PGID settings (should be PUID=99 PGID=100)"
        fi
        TOTAL_DIRS_FIXED=$(( TOTAL_DIRS_FIXED + 1 ))
        TOTAL_FILES_FIXED=$(( TOTAL_FILES_FIXED + WRONG_OWNER ))
    else
        error "$SHARE_NAME — permissions failed"
        error "  chown: $CHOWN_OK  chmod dirs: $CHMOD_DIR_OK  chmod files: $CHMOD_FILE_OK"
        FAILED+=("$SHARE_NAME")
    fi
done

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY MEDIA PERMISSIONS SUMMARY ━━━━━"
echo "$ICON_HOST    Identity:   $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_PERMS   Dir mode:   ${PERMISSIONS_DIR_MODE:-755}"
echo "$ICON_PERMS   File mode:  ${PERMISSIONS_FILE_MODE:-664}"
echo "$ICON_PERMS   Owner:      $PERMISSIONS_OWNER"
echo "$ICON_TIME    Duration:   $(format_duration $(( END - START )))"
echo ""
[[ ${#UPDATED[@]} -gt 0 ]] && log "Updated: ${#UPDATED[@]} shares"
[[ ${#SKIPPED[@]} -gt 0 ]] && warn "Skipped: ${SKIPPED[*]} (not found)"
[[ ${#FAILED[@]} -gt 0 ]]  && echo "$ICON_ERROR    Failed:  ${FAILED[*]}"

# Diagnostic — high correction count indicates container PUID/PGID issue
if [[ "$TOTAL_FILES_FIXED" -gt 50 ]]; then
    warn "$TOTAL_FILES_FIXED files had wrong ownership this run"
    warn "High count suggests a container is not set to PUID=99 PGID=100"
    warn "Common culprits: SABnzbd, qBittorrent, slskd — check container env vars"
fi

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ ${#FAILED[@]} -gt 0 ]]; then
    echo "$ICON_ERROR    Status: SOME SHARES FAILED"
    notify "Media permissions failed on $(hostname) — ${FAILED[*]}" \
        "Media Permissions" "warning"
else
    echo "$ICON_DONE Status: done — ${#UPDATED[@]} shares updated"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ ${#FAILED[@]} -gt 0 ]] && exit 1
exit 0


<?php
header('Content-Type: application/json');
require_once dirname(__DIR__) . '/include/config.php';
require_once dirname(__DIR__) . '/include/monitor.php';

// ── Live sync log ─────────────────────────────────────────────────────────────
$action  = $_GET['action'] ?? '';
if ($action === 'rsync_log') {
    $lockDir = '/tmp/unraid_locks';
    $lines   = [];
    $profile = null;
    $live    = false;
    $elapsed = 0;

    // Active sync — read live log
    foreach (glob("$lockDir/rsync_*.lock") ?: [] as $lf) {
        $content = trim(@file_get_contents($lf) ?: '');
        [$pid, $locked_name] = array_pad(explode(':', $content, 2), 2, '');
        if (!$pid || !file_exists("/proc/$pid")) continue;
        $profile = preg_replace('/^rsync_/', '', $locked_name ?: basename($lf, '.lock'));
        $elapsed = time() - (int)filemtime($lf);
        $liveLog = "$lockDir/rsync_{$profile}.log";
        $raw     = file_exists($liveLog) ? (file($liveLog, FILE_IGNORE_NEW_LINES) ?: []) : [];
        $live    = true;
        $lines   = $raw;
        break;
    }

    // No active sync — use most recent last.log
    if (!$live) {
        $lastLogs = glob("$lockDir/rsync_*.last.log") ?: [];
        if ($lastLogs) {
            usort($lastLogs, fn($a, $b) => filemtime($b) <=> filemtime($a));
            $lastLog = $lastLogs[0];
            $profile = preg_replace('/^rsync_(.+)\.last\.log$/', '$1', basename($lastLog));
            $lines   = file($lastLog, FILE_IGNORE_NEW_LINES) ?: [];
        }
    }

    // Strip ANSI escape codes, keep last 200 lines
    $lines = array_map(fn($l) => preg_replace('/\x1b\[[0-9;]*[mGKHF]/', '', $l), $lines);
    $lines = array_slice($lines, -200);

    echo json_encode(['ok' => true, 'live' => $live, 'profile' => $profile, 'elapsed' => $elapsed, 'lines' => array_values($lines)]);
    exit;
}

$base = vv_rsync_status();
$vars = vv_conf_vars();

$base['windows']['fallback'] = ($vars['FALLBACK_RSYNC_ENABLED'] ?? 'true') !== 'false';

// Bandwidth history — last 30 days
$bwLog    = DATA_DIR . '/bandwidth_history.db';
$warnGb   = (float)($vars['BANDWIDTH_WARN_GB'] ?? 50);
$cutoff   = date('Y-m-d', strtotime('-30 days'));
$history  = [];

if (file_exists($bwLog)) {
    foreach (file($bwLog, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES) ?: [] as $line) {
        $p = explode('|', $line);
        if (count($p) < 5) continue;
        if ($p[0] < $cutoff) continue;
        $history[] = [
            'date'     => $p[0],
            'time'     => $p[1],
            'profile'  => $p[2],
            'duration' => (int)$p[3],
            'status'   => trim($p[4]),
            'bytes'    => isset($p[5]) ? (int)$p[5] : 0,
        ];
    }
}

// Per-window orch arrays (scripts + sync shares)
$masterRaw = vv_read_conf_raw('master.conf');
$myId      = strtoupper(vv_detect_host());
$hostRaw   = vv_read_conf_raw(vv_detect_host() . '.conf');

$winArrayDefs = [
    'critical'     => ['CRITICAL_MAINTENANCE_SCRIPTS',     "{$myId}_CRITICAL_SYNC_SHARES"],
    'intermediate' => ['INTERMEDIATE_MAINTENANCE_SCRIPTS', "{$myId}_INTERMEDIATE_SYNC_SHARES"],
    'daily'        => ['DAILY_MAINTENANCE_SCRIPTS',        "{$myId}_DAILY_SYNC_SHARES"],
    'weekly'       => ['WEEKLY_MAINTENANCE_SCRIPTS',       "{$myId}_WEEKLY_SYNC_SHARES"],
    'fallback'     => [null,                               null],
];
$winArrays = [];
foreach ($winArrayDefs as $win => [$sv, $shv]) {
    $winArrays[$win] = [
        'scripts' => $sv  ? vv_parse_bash_array($masterRaw, $sv)  : [],
        'shares'  => $shv ? vv_parse_bash_array($hostRaw,   $shv) : [],
    ];
}

echo json_encode([
    'enabled'    => $base['enabled'],
    'windows'    => $base['windows'],
    'active'     => $base['active'],
    'last_sync'  => $base['last_sync'],
    'bw_history' => $history,
    'bw_warn_gb' => $warnGb,
    'win_arrays' => $winArrays,
    'settings'   => [
        'bw_limit'     => (int)($vars['BW_LIMIT']          ?? 0),
        'retry_count'  => (int)($vars['RETRY_COUNT']        ?? 3),
        'sleep'        => (int)($vars['SLEEP']              ?? 300),
        'bw_warn_gb'   => (float)($vars['BANDWIDTH_WARN_GB'] ?? 50),
    ],
    'ts'         => time(),
]);


━━ 📋 INTERMEDIATE SYNC SUMMARY ━━━━━ 🖥   Identity: HOST1 (unRAID-Gmer4Lfe) ⏱   Window:   2026-06-06 16:00:06 → 16:00:46 ⏱   Duration: 40s  ⚙  Jobs:   🏁  arr_sync.sh   🏁  arrs_failed_stalled_re…


Let me look at `conf_sync.sh` to see what's happening.


Bash

find /boot/config/plugins/varaverk -name "conf_sync.sh" 2>/dev/null

Find conf_sync.sh


1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= Conf Cache Sync ================================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Maintains a RAM-resident conf cache at /tmp/.vv/config/cached/.confs/.
9	# Credentials and partner keys live in RAM only — never on disk across hosts.
10	#
11	# On array start (default / --array-start):
12	#   1. Copy own conf to local cache
13	#   2. Pull each available partner's conf from their disk → local cache
14	#   3. Push own conf to each available partner's /tmp/.vv/ cache
15	#
16	# On conf save (--push-only):
17	#   Fast path — push updated own conf to all partners' /tmp/.vv/ cache only.
18	#   No pulls, no local cache rebuild.
19	#
20	# Cache is /tmp (tmpfs) — cleared every reboot, repopulated by this script
21	# on next array start. Scripts source from cache for partner vars; own vars
22	# always come from disk (load_config.sh skips cached copy of own conf).
23	#
24	# ==============================================================================================
25	# RUNTIME MODES
26	# ==============================================================================================
27	#
28	#   conf_sync.sh                   Full sync: pull from all partners + push to all partners
29	#   conf_sync.sh --push-only       Push own conf to all partners (fast, for conf-save hook)
30	#   conf_sync.sh --pull-only       Pull partner confs into local cache only (for intermediate orch)
31	#   conf_sync.sh --dry-run         Show what would happen, no changes
32	#   conf_sync.sh --log             Verbose output
33	#
34	# ==============================================================================================
35
36	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
37	source "$SCRIPT_DIR/../load_config.sh"
38
39	PUSH_ONLY=false
40	PULL_ONLY=false
41	FILTERED_ARGS=()
42	for arg in "$@"; do
43	    case "$arg" in
44	        --push-only) PUSH_ONLY=true ;;
45	        --pull-only) PULL_ONLY=true ;;
46	        *)           FILTERED_ARGS+=("$arg") ;;
47	    esac
48	done
49	parse_args "${FILTERED_ARGS[@]}"
50	detect_hosts
51
52	CACHE_DIR="/tmp/.vv/config/cached/.confs"
53	MY_CONF="$SCRIPTS_ROOT/Configurations/${MY_ID,,}.conf"
54	SSH_TIMEOUT=10
55
56	[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
57
58	# ── Ensure cache dir exists ───────────────────────────────────────────────────
59	if [[ "$DRY_RUN" == false ]]; then
60	    mkdir -p "$CACHE_DIR"
61	fi
62
63	# ── Copy own conf into local cache ───────────────────────────────────────────
64	if [[ "$PUSH_ONLY" == false ]] && [[ "$PULL_ONLY" == false ]]; then
65	    if [[ -f "$MY_CONF" ]]; then
66	        if [[ "$DRY_RUN" == true ]]; then
67	            warn "DRY RUN — would copy $(basename "$MY_CONF") → $CACHE_DIR/"
68	        else
69	            cp "$MY_CONF" "$CACHE_DIR/${MY_ID,,}.conf" && \
70	                log "Own conf cached ✅" || warn "Failed to cache own conf"
71	        fi
72	    else
73	        warn "Own conf not found: $MY_CONF"
74	    fi
75	fi
76
77	# ── Per-partner sync ──────────────────────────────────────────────────────────
78	PUSHED=0
79	PULLED=0
80	FAILED=0
81
82	for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
83	    partner_host="${!host_var}"
84	    [[ -z "$partner_host" ]] && continue
85	    [[ "${host_var,,}" == "${MY_ID,,}" ]] && continue
86
87	    partner_slot="${host_var,,}"   # e.g. host2
88	    partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
89
90	    if [[ -z "$partner_ip" ]]; then
91	        warn "$partner_host — cannot resolve Tailscale IP, skipping"
92	        (( FAILED++ ))
93	        continue
94	    fi
95
96	    # ── Pull: grab partner's conf from their disk → our local cache ──────────
97	    if [[ "$PUSH_ONLY" == false ]]; then
98	        remote_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
99	        if [[ "$DRY_RUN" == true ]]; then
100	            warn "DRY RUN — would pull $partner_host:$remote_conf → $CACHE_DIR/${partner_slot}.conf"
101	        elif timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
102	            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
103	            "root@${partner_ip}:${remote_conf}" \
104	            "$CACHE_DIR/${partner_slot}.conf" 2>/dev/null; then
105	            log "Pulled ${partner_slot}.conf from $partner_host ✅"
106	            (( PULLED++ ))
107	        else
108	            warn "Could not pull ${partner_slot}.conf from $partner_host"
109	            (( FAILED++ ))
110	        fi
111	    fi
112
113	    # ── Push: send own conf to partner's /tmp/.vv/ cache ────────────────────
114	    if [[ "$PULL_ONLY" == true ]]; then
115	        continue
116	    fi
117
118	    if [[ "$DRY_RUN" == true ]]; then
119	        warn "DRY RUN — would push ${MY_ID,,}.conf → $partner_host:/tmp/.vv/config/cached/.confs/"
120	        continue
121	    fi
122
123	    # Ensure partner's cache dir exists, then SCP own conf into it
124	    timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
125	        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
126	        "root@${partner_ip}" "mkdir -p '$CACHE_DIR'" 2>/dev/null
127
128	    if timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
129	        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
130	        "$MY_CONF" \
131	        "root@${partner_ip}:${CACHE_DIR}/${MY_ID,,}.conf" 2>/dev/null; then
132	        log "Pushed ${MY_ID,,}.conf to $partner_host ✅"
133	        (( PUSHED++ ))
134	    else
135	        warn "Could not push to $partner_host"
136	        (( FAILED++ ))
137	    fi
138	done
139
140	# ── Summary ───────────────────────────────────────────────────────────────────
141	if [[ "$PUSH_ONLY" == true ]]; then
142	    info "Conf push complete — pushed to $PUSHED host(s)${FAILED:+, $FAILED failed}"
143	elif [[ "$PULL_ONLY" == true ]]; then
144	    info "Conf pull complete — pulled $PULLED partner conf(s)${FAILED:+, $FAILED failed}"
145	else
146	    info "Conf sync complete — pulled $PULLED, pushed $PUSHED${FAILED:+, $FAILED failed}"
147	fi
148
149	if [[ "$FAILED" -gt 0 ]]; then
150	    notify "Conf sync on $LOCAL_SERVER_NAME ($MY_ID) — $FAILED partner(s) failed. Partner config cache may be stale." \
151	        "Conf Sync" "warning"
152	    exit 1
153	fi
154

#!/bin/bash
# ==============================================================================================
# ============================= Conf Cache Sync ================================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Maintains a RAM-resident conf cache at /tmp/.vv/config/cached/.confs/.
# Credentials and partner keys live in RAM only — never on disk across hosts.
#
# On array start (default / --array-start):
#   1. Copy own conf to local cache
#   2. Pull each available partner's conf from their disk → local cache
#   3. Push own conf to each available partner's /tmp/.vv/ cache
#
# On conf save (--push-only):
#   Fast path — push updated own conf to all partners' /tmp/.vv/ cache only.
#   No pulls, no local cache rebuild.
#
# Cache is /tmp (tmpfs) — cleared every reboot, repopulated by this script
# on next array start. Scripts source from cache for partner vars; own vars
# always come from disk (load_config.sh skips cached copy of own conf).
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
#   conf_sync.sh                   Full sync: pull from all partners + push to all partners
#   conf_sync.sh --push-only       Push own conf to all partners (fast, for conf-save hook)
#   conf_sync.sh --pull-only       Pull partner confs into local cache only (for intermediate orch)
#   conf_sync.sh --dry-run         Show what would happen, no changes
#   conf_sync.sh --log             Verbose output
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"

PUSH_ONLY=false
PULL_ONLY=false
FILTERED_ARGS=()
for arg in "$@"; do
    case "$arg" in
        --push-only) PUSH_ONLY=true ;;
        --pull-only) PULL_ONLY=true ;;
        *)           FILTERED_ARGS+=("$arg") ;;
    esac
done
parse_args "${FILTERED_ARGS[@]}"
detect_hosts

CACHE_DIR="/tmp/.vv/config/cached/.confs"
MY_CONF="$SCRIPTS_ROOT/Configurations/${MY_ID,,}.conf"
SSH_TIMEOUT=10

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# ── Ensure cache dir exists ───────────────────────────────────────────────────
if [[ "$DRY_RUN" == false ]]; then
    mkdir -p "$CACHE_DIR"
fi

# ── Copy own conf into local cache ───────────────────────────────────────────
if [[ "$PUSH_ONLY" == false ]] && [[ "$PULL_ONLY" == false ]]; then
    if [[ -f "$MY_CONF" ]]; then
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would copy $(basename "$MY_CONF") → $CACHE_DIR/"
        else
            cp "$MY_CONF" "$CACHE_DIR/${MY_ID,,}.conf" && \
                log "Own conf cached ✅" || warn "Failed to cache own conf"
        fi
    else
        warn "Own conf not found: $MY_CONF"
    fi
fi

# ── Per-partner sync ──────────────────────────────────────────────────────────
PUSHED=0
PULLED=0
FAILED=0

for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
    partner_host="${!host_var}"
    [[ -z "$partner_host" ]] && continue
    [[ "${host_var,,}" == "${MY_ID,,}" ]] && continue

    partner_slot="${host_var,,}"   # e.g. host2
    partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)

    if [[ -z "$partner_ip" ]]; then
        warn "$partner_host — cannot resolve Tailscale IP, skipping"
        (( FAILED++ ))
        continue
    fi

    # ── Pull: grab partner's conf from their disk → our local cache ──────────
    if [[ "$PUSH_ONLY" == false ]]; then
        remote_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would pull $partner_host:$remote_conf → $CACHE_DIR/${partner_slot}.conf"
        elif timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
            "root@${partner_ip}:${remote_conf}" \
            "$CACHE_DIR/${partner_slot}.conf" 2>/dev/null; then
            log "Pulled ${partner_slot}.conf from $partner_host ✅"
            (( PULLED++ ))
        else
            warn "Could not pull ${partner_slot}.conf from $partner_host"
            (( FAILED++ ))
        fi
    fi

    # ── Push: send own conf to partner's /tmp/.vv/ cache ────────────────────
    if [[ "$PULL_ONLY" == true ]]; then
        continue
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would push ${MY_ID,,}.conf → $partner_host:/tmp/.vv/config/cached/.confs/"
        continue
    fi

    # Ensure partner's cache dir exists, then SCP own conf into it
    timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
        "root@${partner_ip}" "mkdir -p '$CACHE_DIR'" 2>/dev/null

    if timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
        "$MY_CONF" \
        "root@${partner_ip}:${CACHE_DIR}/${MY_ID,,}.conf" 2>/dev/null; then
        log "Pushed ${MY_ID,,}.conf to $partner_host ✅"
        (( PUSHED++ ))
    else
        warn "Could not push to $partner_host"
        (( FAILED++ ))
    fi
done

# ── Summary ───────────────────────────────────────────────────────────────────
if [[ "$PUSH_ONLY" == true ]]; then
    info "Conf push complete — pushed to $PUSHED host(s)${FAILED:+, $FAILED failed}"
elif [[ "$PULL_ONLY" == true ]]; then
    info "Conf pull complete — pulled $PULLED partner conf(s)${FAILED:+, $FAILED failed}"
else
    info "Conf sync complete — pulled $PULLED, pushed $PUSHED${FAILED:+, $FAILED failed}"
fi

if [[ "$FAILED" -gt 0 ]]; then
    notify "Conf sync on $LOCAL_SERVER_NAME ($MY_ID) — $FAILED partner(s) failed. Partner config cache may be stale." \
        "Conf Sync" "warning"
    exit 1
fi


HOST2 is being rebuilt and unreachable over SSH, so the pull always times out and exits 1. Let me check master.conf for the toggle pattern to add one for conf sync.


grep -n "ENABLED\|conf_sync\|INTERMEDIATE_MAINTENANCE" /boot/config/plugins/varaverk/Configurations/master.conf | head -60

Find enabled toggles and intermediate scripts list in master.conf


135:    ARR_SYNC_ENABLED=true
174:    PARTNERSHIP_ENABLED=false
285:        "System_Essentials/conf_sync.sh"              # pull partner confs + push own conf into /tmp/.vv/ RAM cache
351:    INTERMEDIATE_MAINTENANCE_SCRIPTS=(
355:    # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).
463:#   RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions
471:#   RSYNC_ENABLED=true                ← rsync works, individual scripts run fine
472:#   CRITICAL_RSYNC_ENABLED=true       ← 30min auth stack sync still runs
473:#   INTERMEDIATE_RSYNC_ENABLED=false  ← skip 4h arr/mid-day rsync during rebuild
474:#   DAILY_RSYNC_ENABLED=false         ← skip daily HDD syncs during rebuild
475:#   WEEKLY_RSYNC_ENABLED=true         ← Emby + Critical-Data still sync (NVMe)
476:#   FALLBACK_RSYNC_ENABLED=true       ← handback writeback still works
478:#   → When ready: INTERMEDIATE_RSYNC_ENABLED=true  DAILY_RSYNC_ENABLED=true
479:    RSYNC_ENABLED=true                  # Tier 1 — global gate, overrides everything below
480:    CRITICAL_RSYNC_ENABLED=true        # Tier 2 — disabled during HOST2 rebuild, re-enable when ready
481:    INTERMEDIATE_RSYNC_ENABLED=true     # Tier 2 — intermediate_sync_maintenance.sh rsync section
482:    DAILY_RSYNC_ENABLED=true           # Tier 2 — HOST2 rebuild in progress, re-enable when ready
483:    WEEKLY_RSYNC_ENABLED=true           # Tier 2 — weekly_sync_maintenance.sh rsync section
484:    FALLBACK_RSYNC_ENABLED=true         # Tier 2 — fallback.sh writeback jobs on handback
653:    FALLBACK_ENABLED=true           # HOST2 back online
868:    NETWORK_WATCHDOG_ENABLED=true
1283:    RW_ENABLED=true
1303:    RW_SABNZBD_ENABLED=true
1309:    RW_QBIT_ENABLED=true


275
276	# ━━━ Array Start ━━━
277	# Scripts launched by array_started.sh when the array comes online.
278	# Launched in order — each as a background process.
279	# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
280	# Continuous scripts (failover) run until array stops.
281	# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
282	# watchdog_orchestrator.sh — NOT launched here.
283	    ARRAY_START_SCRIPTS=(
284	        "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register Varaverk API key — registry is ephemeral
285	        "System_Essentials/conf_sync.sh"              # pull partner confs + push own conf into /tmp/.vv/ RAM cache
286	        "Transcodes/ramdisk_setup.sh"                 # creates ramdisk + symlink before Emby starts
287	        "System_Essentials/docker_syslog_filter.sh"   # suppress veth noise before logs fill
288	        "Plugin/unraid/System_Essentials/php_fpm_max_children.sh"   # WebGUI performance tuning
289	        "System_Essentials/inotify_tuning.sh"         # bump inotify limits — containers miss events if exhausted
290	        "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
291	        "Tools/claude_startup.sh"                     # persist Claude data + binary to appdata; re-symlink on boot
292	        "Fallback/fallback.sh"                        # mutual failover — continuous
293	    )
294
295	# ━━━ Array Stop ━━━
296	# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order.
297	# Run sequentially (foreground) — each must complete before the next starts.
298	# Order matters: user scripts first (prevents new ops), then data movement, then containers.
299	    ARRAY_STOP_SCRIPTS=(
300	        "Plugin/unraid/System_Essentials/user_scripts_stop.sh"        # stop background scripts before they start new ops
301	        "Fallback/fallback.sh --stop"                   # gracefully stop fallback (not caught by user_scripts_stop)
302	        "System_Essentials/rsync_stop.sh --rsync-only"  # kill rsync; skip container recovery (handled below)
303	        "Plugin/unraid/System_Essentials/mover_stop.sh"               # stop mover after rsync (they conflict on same files)
304	        "Docker_Essentials/docker_container_stop.sh"    # stop all containers last

# ━━━ Array Start ━━━
# Scripts launched by array_started.sh when the array comes online.
# Launched in order — each as a background process.
# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally.
# Continuous scripts (failover) run until array stops.
# Watchdogs (resource_watchdog, docker_watchdog, system_watchdog) are cronned via
# watchdog_orchestrator.sh — NOT launched here.
    ARRAY_START_SCRIPTS=(
        "Plugin/unraid/System_Essentials/unraid_api_key_renew.sh"   # re-register Varaverk API key — registry is ephemeral
        "System_Essentials/conf_sync.sh"              # pull partner confs + push own conf into /tmp/.vv/ RAM cache
        "Transcodes/ramdisk_setup.sh"                 # creates ramdisk + symlink before Emby starts
        "System_Essentials/docker_syslog_filter.sh"   # suppress veth noise before logs fill
        "Plugin/unraid/System_Essentials/php_fpm_max_children.sh"   # WebGUI performance tuning
        "System_Essentials/inotify_tuning.sh"         # bump inotify limits — containers miss events if exhausted
        "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers
        "Tools/claude_startup.sh"                     # persist Claude data + binary to appdata; re-symlink on boot
        "Fallback/fallback.sh"                        # mutual failover — continuous
    )

# ━━━ Array Stop ━━━
# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order.
# Run sequentially (foreground) — each must complete before the next starts.
# Order matters: user scripts first (prevents new ops), then data movement, then containers.
    ARRAY_STOP_SCRIPTS=(
        "Plugin/unraid/System_Essentials/user_scripts_stop.sh"        # stop background scripts before they start new ops
        "Fallback/fallback.sh --stop"                   # gracefully stop fallback (not caught by user_scripts_stop)
        "System_Essentials/rsync_stop.sh --rsync-only"  # kill rsync; skip container recovery (handled below)
        "Plugin/unraid/System_Essentials/mover_stop.sh"               # stop mover after rsync (they conflict on same files)
        "Docker_Essentials/docker_container_stop.sh"    # stop all containers last


345	# ━━━ Intermediate Sync Maintenance ━━━
346	# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch,
347	# and optional mid-day rsync for any shares that need sub-daily propagation.
348	# Schedule: 0 */4 * * *
349	    # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf.
350
351	    INTERMEDIATE_MAINTENANCE_SCRIPTS=(
352	        "Media/arrs_failed_stalled_recovery.sh"  # blocklist + re-search failed/stalled arr queue items
353	    )
354	    # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
355	    # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).
356
357	# ━━━ Daily Sync Maintenance ━━━
358	# daily_sync_maintenance.sh runs media share sync first, then iterates
359	# DAILY_MAINTENANCE_SCRIPTS for all jobs.
360	# Schedule: 0 1 * * * (1am daily)
361	    DAILY_MAINTENANCE_SCRIPTS=(
362	        "git_pull_execute.sh"                         # pull latest scripts — always runs first
363	        "Media/media_shares_permissions.sh"           # apply permissions — runs before cleaners
364	        "Media/media_cleaner.sh anime"                # remove junk from anime shares
365	        "Media/media_cleaner.sh media"                # remove junk from media shares
366	        #"Media/lidarr_cleanup.sh"                    # remove orphaned music files — enable when ready
367	        #"Media/sonarr_cleanup.sh"                    # remove orphaned TV files — enable when ready
368	        "Media/radarr_cleanup.sh"                     # remove orphaned movie files
369	        "Media/lidarr_missing_art.sh"                 # fetch missing album/artist artwork (HOST1 only — self-guards)
370	        "Media/radarr_tmdb_removed.sh"                # remove movies dropped from TMDb
371	        "Media/sonarr_tvdb_removed.sh"                # remove series dropped from TVDB
372	        "Docker_Essentials/docker_update.sh"          # pull container image updates before restart
373	        "Docker_Essentials/docker_daily_restart.sh"   # daily container restarts — runs last
374	    )

# ━━━ Intermediate Sync Maintenance ━━━
# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch,
# and optional mid-day rsync for any shares that need sub-daily propagation.
# Schedule: 0 */4 * * *
    # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf.

    INTERMEDIATE_MAINTENANCE_SCRIPTS=(
        "Media/arrs_failed_stalled_recovery.sh"  # blocklist + re-search failed/stalled arr queue items
    )
    # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
    # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).

# ━━━ Daily Sync Maintenance ━━━
# daily_sync_maintenance.sh runs media share sync first, then iterates
# DAILY_MAINTENANCE_SCRIPTS for all jobs.
# Schedule: 0 1 * * * (1am daily)
    DAILY_MAINTENANCE_SCRIPTS=(
        "git_pull_execute.sh"                         # pull latest scripts — always runs first
        "Media/media_shares_permissions.sh"           # apply permissions — runs before cleaners
        "Media/media_cleaner.sh anime"                # remove junk from anime shares
        "Media/media_cleaner.sh media"                # remove junk from media shares
        #"Media/lidarr_cleanup.sh"                    # remove orphaned music files — enable when ready
        #"Media/sonarr_cleanup.sh"                    # remove orphaned TV files — enable when ready
        "Media/radarr_cleanup.sh"                     # remove orphaned movie files
        "Media/lidarr_missing_art.sh"                 # fetch missing album/artist artwork (HOST1 only — self-guards)
        "Media/radarr_tmdb_removed.sh"                # remove movies dropped from TMDb
        "Media/sonarr_tvdb_removed.sh"                # remove series dropped from TVDB
        "Docker_Essentials/docker_update.sh"          # pull container image updates before restart
        "Docker_Essentials/docker_daily_restart.sh"   # daily container restarts — runs last
    )


grep -n "conf_sync" /boot/config/plugins/varaverk/Orchestrators/*.sh

Find conf_sync.sh calls in orchestrators


/boot/config/plugins/varaverk/Orchestrators/intermediate_sync_maintenance.sh:9:#   1. conf_sync.sh --pull-only    — refresh partner conf cache in RAM (/tmp/.vv/)
/boot/config/plugins/varaverk/Orchestrators/intermediate_sync_maintenance.sh:164:CONF_SYNC_SCRIPT="$SCRIPTS_ROOT/System_Essentials/conf_sync.sh"
/boot/config/plugins/varaverk/Orchestrators/intermediate_sync_maintenance.sh:166:    warn "conf_sync.sh not found — skipping partner conf refresh"
/boot/config/plugins/varaverk/Orchestrators/intermediate_sync_maintenance.sh:172:        JOB_PASS+=("conf_sync.sh --pull-only")
/boot/config/plugins/varaverk/Orchestrators/intermediate_sync_maintenance.sh:175:        JOB_FAIL+=("conf_sync.sh --pull-only")


155	echo ""
156	echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
157
158	# ==============================================================================================
159	# ━━━ Conf Pull ━━━
160	# ==============================================================================================
161	echo ""
162	echo "━━━ $ICON_GEAR Conf Pull ━━━"
163
164	CONF_SYNC_SCRIPT="$SCRIPTS_ROOT/System_Essentials/conf_sync.sh"
165	if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
166	    warn "conf_sync.sh not found — skipping partner conf refresh"
167	else
168	    _conf_args=("--pull-only")
169	    [[ "$DRY_RUN" == true ]] && _conf_args+=("--dry-run")
170	    if bash "$CONF_SYNC_SCRIPT" "${_conf_args[@]}"; then
171	        log "Partner conf cache refreshed ✅"
172	        JOB_PASS+=("conf_sync.sh --pull-only")
173	    else
174	        warn "Partner conf pull failed — cache may be stale"
175	        JOB_FAIL+=("conf_sync.sh --pull-only")
176	    fi
177	    unset _conf_args
178	fi
179
180	# ==============================================================================================
181	# ━━━ Arr Sync ━━━
182	# ==============================================================================================
183	echo ""
184	echo "━━━ $ICON_SYNC Arr Sync ━━━"

echo ""
echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

# ==============================================================================================
# ━━━ Conf Pull ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_GEAR Conf Pull ━━━"

CONF_SYNC_SCRIPT="$SCRIPTS_ROOT/System_Essentials/conf_sync.sh"
if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
    warn "conf_sync.sh not found — skipping partner conf refresh"
else
    _conf_args=("--pull-only")
    [[ "$DRY_RUN" == true ]] && _conf_args+=("--dry-run")
    if bash "$CONF_SYNC_SCRIPT" "${_conf_args[@]}"; then
        log "Partner conf cache refreshed ✅"
        JOB_PASS+=("conf_sync.sh --pull-only")
    else
        warn "Partner conf pull failed — cache may be stale"
        JOB_FAIL+=("conf_sync.sh --pull-only")
    fi
    unset _conf_args
fi

# ==============================================================================================
# ━━━ Arr Sync ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Arr Sync ━━━"


The pull fails every 4 hours because HOST2 is down — Tailscale can't resolve it, SCP times out, `FAILED++`, `exit 1`. Also triggers a notification. Let me add a `CONF_SYNC_ENABLED` toggle (same pattern as the rsync gates) and gate the check inside `conf_sync.sh` itself so it applies to both the array-start run and the intermediate pull.


340	# Shares synced every 30 minutes — defined per host in host*.conf.
341	# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES
342	# Format: "/path/to/share" or "/path/to/share|profile-name"
343	# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
344
345	# ━━━ Intermediate Sync Maintenance ━━━
346	# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch,
347	# and optional mid-day rsync for any shares that need sub-daily propagation.
348	# Schedule: 0 */4 * * *
349	    # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf.
350
351	    INTERMEDIATE_MAINTENANCE_SCRIPTS=(
352	        "Media/arrs_failed_stalled_recovery.sh"  # blocklist + re-search failed/stalled arr queue items
353	    )
354	    # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
355	    # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).
356
357	# ━━━ Daily Sync Maintenance ━━━
358	# daily_sync_maintenance.sh runs media share sync first, then iterates
359	# DAILY_MAINTENANCE_SCRIPTS for all jobs.

# Shares synced every 30 minutes — defined per host in host*.conf.
# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES
# Format: "/path/to/share" or "/path/to/share|profile-name"
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.

# ━━━ Intermediate Sync Maintenance ━━━
# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch,
# and optional mid-day rsync for any shares that need sub-daily propagation.
# Schedule: 0 */4 * * *
    # INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf.

    INTERMEDIATE_MAINTENANCE_SCRIPTS=(
        "Media/arrs_failed_stalled_recovery.sh"  # blocklist + re-search failed/stalled arr queue items
    )
    # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
    # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).

# ━━━ Daily Sync Maintenance ━━━
# daily_sync_maintenance.sh runs media share sync first, then iterates
# DAILY_MAINTENANCE_SCRIPTS for all jobs.


460	# Two-tier toggle system — Tier 1 overrides Tier 2.
461	#
462	# Tier 1 — Global gate:
463	#   RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions
464	#   Use when: remote completely offline, major maintenance, disaster recovery
465	#
466	# Tier 2 — Per-orchestrator (only applies when Tier 1 is true):
467	#   Fine grained control — disable specific orchestrators while keeping others
468	#   Use when: rebuilding secondary, testing, per-window bandwidth management
469	#
470	# Example — HOST2 data rebuild:
471	#   RSYNC_ENABLED=true                ← rsync works, individual scripts run fine
472	#   CRITICAL_RSYNC_ENABLED=true       ← 30min auth stack sync still runs
473	#   INTERMEDIATE_RSYNC_ENABLED=false  ← skip 4h arr/mid-day rsync during rebuild
474	#   DAILY_RSYNC_ENABLED=false         ← skip daily HDD syncs during rebuild
475	#   WEEKLY_RSYNC_ENABLED=true         ← Emby + Critical-Data still sync (NVMe)
476	#   FALLBACK_RSYNC_ENABLED=true       ← handback writeback still works
477	#   → Run individual: bash Rsync/rsync.sh /mnt/user/Movies
478	#   → When ready: INTERMEDIATE_RSYNC_ENABLED=true  DAILY_RSYNC_ENABLED=true
479	    RSYNC_ENABLED=true                  # Tier 1 — global gate, overrides everything below
480	    CRITICAL_RSYNC_ENABLED=true        # Tier 2 — disabled during HOST2 rebuild, re-enable when ready
481	    INTERMEDIATE_RSYNC_ENABLED=true     # Tier 2 — intermediate_sync_maintenance.sh rsync section
482	    DAILY_RSYNC_ENABLED=true           # Tier 2 — HOST2 rebuild in progress, re-enable when ready
483	    WEEKLY_RSYNC_ENABLED=true           # Tier 2 — weekly_sync_maintenance.sh rsync section
484	    FALLBACK_RSYNC_ENABLED=true         # Tier 2 — fallback.sh writeback jobs on handback
485
486	# ━━━ Rsync Defaults ━━━
487	# Global fallback values used when no profile match is found.
488	# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed.
489	# Appdata shares match profiles by directory basename (lowercased).

# Two-tier toggle system — Tier 1 overrides Tier 2.
#
# Tier 1 — Global gate:
#   RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions
#   Use when: remote completely offline, major maintenance, disaster recovery
#
# Tier 2 — Per-orchestrator (only applies when Tier 1 is true):
#   Fine grained control — disable specific orchestrators while keeping others
#   Use when: rebuilding secondary, testing, per-window bandwidth management
#
# Example — HOST2 data rebuild:
#   RSYNC_ENABLED=true                ← rsync works, individual scripts run fine
#   CRITICAL_RSYNC_ENABLED=true       ← 30min auth stack sync still runs
#   INTERMEDIATE_RSYNC_ENABLED=false  ← skip 4h arr/mid-day rsync during rebuild
#   DAILY_RSYNC_ENABLED=false         ← skip daily HDD syncs during rebuild
#   WEEKLY_RSYNC_ENABLED=true         ← Emby + Critical-Data still sync (NVMe)
#   FALLBACK_RSYNC_ENABLED=true       ← handback writeback still works
#   → Run individual: bash Rsync/rsync.sh /mnt/user/Movies
#   → When ready: INTERMEDIATE_RSYNC_ENABLED=true  DAILY_RSYNC_ENABLED=true
    RSYNC_ENABLED=true                  # Tier 1 — global gate, overrides everything below
    CRITICAL_RSYNC_ENABLED=true        # Tier 2 — disabled during HOST2 rebuild, re-enable when ready
    INTERMEDIATE_RSYNC_ENABLED=true     # Tier 2 — intermediate_sync_maintenance.sh rsync section
    DAILY_RSYNC_ENABLED=true           # Tier 2 — HOST2 rebuild in progress, re-enable when ready
    WEEKLY_RSYNC_ENABLED=true           # Tier 2 — weekly_sync_maintenance.sh rsync section
    FALLBACK_RSYNC_ENABLED=true         # Tier 2 — fallback.sh writeback jobs on handback

# ━━━ Rsync Defaults ━━━
# Global fallback values used when no profile match is found.
# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed.
# Appdata shares match profiles by directory basename (lowercased).


Two changes needed: add `CONF_SYNC_ENABLED=false` to master.conf (near the intermediate section), and add the gate check at the top of `conf_sync.sh`.


Edit

    INTERMEDIATE_MAINTENANCE_SCRIPTS=(
        "Media/arrs_failed_stalled_recovery.sh"  # blocklist + re-search failed/stalled arr queue items
    )
    # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
    # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).

    CONF_SYNC_ENABLED=false             # HOST2 rebuild in progress — re-enable when HOST2 is back online

    INTERMEDIATE_MAINTENANCE_SCRIPTS=(
        "Media/arrs_failed_stalled_recovery.sh"  # blocklist + re-search failed/stalled arr queue items
    )
    # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
    # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).


The file /boot/config/plugins/varaverk/Configurations/master.conf has been updated successfully. (file state is current in your context — no need to Read it back)

    CONF_SYNC_ENABLED=false             # HOST2 rebuild in progress — re-enable when HOST2 is back online

    INTERMEDIATE_MAINTENANCE_SCRIPTS=(
        "Media/arrs_failed_stalled_recovery.sh"  # blocklist + re-search failed/stalled arr queue items
    )
    # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
    # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).

# INTERMEDIATE_SYNC_SHARES is host-specific — configure HOST*_INTERMEDIATE_SYNC_SHARES in host*.conf.

+    CONF_SYNC_ENABLED=false             # HOST2 rebuild in progress — re-enable when HOST2 is back online


Now add the gate to `conf_sync.sh` right after `detect_hosts`:


parse_args "${FILTERED_ARGS[@]}"
detect_hosts

CACHE_DIR="/tmp/.vv/config/cached/.confs"

parse_args "${FILTERED_ARGS[@]}"
detect_hosts

if [[ "${CONF_SYNC_ENABLED:-true}" == false ]]; then
    log "CONF_SYNC_ENABLED=false — skipping"
    exit 0
fi

CACHE_DIR="/tmp/.vv/config/cached/.confs"


The file /boot/config/plugins/varaverk/System_Essentials/conf_sync.sh has been updated successfully. (file state is current in your context — no need to Read it back)

parse_args "${FILTERED_ARGS[@]}"
detect_hosts

if [[ "${CONF_SYNC_ENABLED:-true}" == false ]]; then
    log "CONF_SYNC_ENABLED=false — skipping"
    exit 0
fi

CACHE_DIR="/tmp/.vv/config/cached/.confs"

#!/bin/bash
# ==============================================================================================
# ============================= Conf Cache Sync ================================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Maintains a RAM-resident conf cache at /tmp/.vv/config/cached/.confs/.
# Credentials and partner keys live in RAM only — never on disk across hosts.
#
# On array start (default / --array-start):
#   1. Copy own conf to local cache
#   2. Pull each available partner's conf from their disk → local cache
#   3. Push own conf to each available partner's /tmp/.vv/ cache
#
# On conf save (--push-only):
#   Fast path — push updated own conf to all partners' /tmp/.vv/ cache only.
#   No pulls, no local cache rebuild.
#
# Cache is /tmp (tmpfs) — cleared every reboot, repopulated by this script
# on next array start. Scripts source from cache for partner vars; own vars
# always come from disk (load_config.sh skips cached copy of own conf).
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
#   conf_sync.sh                   Full sync: pull from all partners + push to all partners
#   conf_sync.sh --push-only       Push own conf to all partners (fast, for conf-save hook)
#   conf_sync.sh --pull-only       Pull partner confs into local cache only (for intermediate orch)
#   conf_sync.sh --dry-run         Show what would happen, no changes
#   conf_sync.sh --log             Verbose output
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"

PUSH_ONLY=false
PULL_ONLY=false
FILTERED_ARGS=()
for arg in "$@"; do
    case "$arg" in
        --push-only) PUSH_ONLY=true ;;
        --pull-only) PULL_ONLY=true ;;
        *)           FILTERED_ARGS+=("$arg") ;;
    esac
done
parse_args "${FILTERED_ARGS[@]}"
detect_hosts

CACHE_DIR="/tmp/.vv/config/cached/.confs"
MY_CONF="$SCRIPTS_ROOT/Configurations/${MY_ID,,}.conf"
SSH_TIMEOUT=10

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# ── Ensure cache dir exists ───────────────────────────────────────────────────
if [[ "$DRY_RUN" == false ]]; then
    mkdir -p "$CACHE_DIR"
fi

# ── Copy own conf into local cache ───────────────────────────────────────────
if [[ "$PUSH_ONLY" == false ]] && [[ "$PULL_ONLY" == false ]]; then
    if [[ -f "$MY_CONF" ]]; then
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would copy $(basename "$MY_CONF") → $CACHE_DIR/"
        else
            cp "$MY_CONF" "$CACHE_DIR/${MY_ID,,}.conf" && \
                log "Own conf cached ✅" || warn "Failed to cache own conf"
        fi
    else
        warn "Own conf not found: $MY_CONF"
    fi
fi

# ── Per-partner sync ──────────────────────────────────────────────────────────
PUSHED=0
PULLED=0
FAILED=0

for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
    partner_host="${!host_var}"
    [[ -z "$partner_host" ]] && continue
    [[ "${host_var,,}" == "${MY_ID,,}" ]] && continue

    partner_slot="${host_var,,}"   # e.g. host2
    partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)

    if [[ -z "$partner_ip" ]]; then
        warn "$partner_host — cannot resolve Tailscale IP, skipping"
        (( FAILED++ ))
        continue
    fi

    # ── Pull: grab partner's conf from their disk → our local cache ──────────
    if [[ "$PUSH_ONLY" == false ]]; then
        remote_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would pull $partner_host:$remote_conf → $CACHE_DIR/${partner_slot}.conf"
        elif timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
            "root@${partner_ip}:${remote_conf}" \
            "$CACHE_DIR/${partner_slot}.conf" 2>/dev/null; then
            log "Pulled ${partner_slot}.conf from $partner_host ✅"
            (( PULLED++ ))
        else
            warn "Could not pull ${partner_slot}.conf from $partner_host"
            (( FAILED++ ))
        fi
    fi

    # ── Push: send own conf to partner's /tmp/.vv/ cache ────────────────────
    if [[ "$PULL_ONLY" == true ]]; then
        continue
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would push ${MY_ID,,}.conf → $partner_host:/tmp/.vv/config/cached/.confs/"
        continue
    fi

    # Ensure partner's cache dir exists, then SCP own conf into it
    timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
        "root@${partner_ip}" "mkdir -p '$CACHE_DIR'" 2>/dev/null

    if timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
        "$MY_CONF" \
        "root@${partner_ip}:${CACHE_DIR}/${MY_ID,,}.conf" 2>/dev/null; then
        log "Pushed ${MY_ID,,}.conf to $partner_host ✅"
        (( PUSHED++ ))
    else
        warn "Could not push to $partner_host"
        (( FAILED++ ))
    fi
done

# ── Summary ───────────────────────────────────────────────────────────────────
if [[ "$PUSH_ONLY" == true ]]; then
    info "Conf push complete — pushed to $PUSHED host(s)${FAILED:+, $FAILED failed}"
elif [[ "$PULL_ONLY" == true ]]; then
    info "Conf pull complete — pulled $PULLED partner conf(s)${FAILED:+, $FAILED failed}"
else
    info "Conf sync complete — pulled $PULLED, pushed $PUSHED${FAILED:+, $FAILED failed}"
fi

if [[ "$FAILED" -gt 0 ]]; then
    notify "Conf sync on $LOCAL_SERVER_NAME ($MY_ID) — $FAILED partner(s) failed. Partner config cache may be stale." \
        "Conf Sync" "warning"
    exit 1
fi

+if [[ "${CONF_SYNC_ENABLED:-true}" == false ]]; then

+    log "CONF_SYNC_ENABLED=false — skipping"


Now commit and push:


Bash

git add Configurations/master.conf System_Essentials/conf_sync.sh && git diff --cached --stat

Stage changes and show diff stat


The following paths are ignored by one of your .gitignore files:
Configurations/master.conf
hint: Use -f if you really want to add them.
hint: Disable this message with "git config set advice.addIgnoredFile false"


cat /boot/config/plugins/varaverk/.gitignore 2>/dev/null | head -20; git -C /boot/config/plugins/varaverk status --short

Check gitignore and git status


# ── Personal configuration — never push to public repo ────────────────────────
# These contain API keys, passwords, SSH paths, personal hostnames.
# Templates (*.template) are safe and remain tracked.
Configurations/host*.conf
Configurations/master.conf
Configurations/*.bak
.claude
.vscode

# ── Runtime state, data, logs ─────────────────────────────────────────────────
data/
State_Files/
*.log
*.lock

# ── Plugin runtime state (per-host, never synced) ────────────────────────────
schedule.json
docker_folders.json
*.db

 M Fallback/fallback.sh
 M Orchestrators/array_started.sh
 M Plugin/unraid/System_Essentials/unraid_api_key_renew.sh
 M Plugin/unraid/api/stop.php
 M Plugin/unraid/include/config.php
 M Plugin/unraid/js/varaverk.js
 M Plugin/unraid/pages/arrs.php
 M Plugin/unraid/pages/docker.php
 M Plugin/unraid/pages/fallback.php
 M Plugin/unraid/pages/partnership.php
 M Plugin/unraid/pages/setup.php
M  System_Essentials/conf_sync.sh
 M common.sh


`master.conf` is gitignored (it has credentials — only the template is tracked). The toggle change there is saved to `/boot` which persists across reboots without git. Only `conf_sync.sh` gets committed.


git add System_Essentials/conf_sync.sh && git commit -m "Add CONF_SYNC_ENABLED gate — skip partner sync when HOST2 is offline"

Commit conf_sync.sh gate


[main bd73e87] Add CONF_SYNC_ENABLED gate — skip partner sync when HOST2 is offline
 1 file changed, 11 insertions(+)


Push to remote


remote: . Processing 1 references
remote: Processed 1 references in total
To ssh://192.168.50.2:221/FailedProxy/Varaverk.git
   273a297..bd73e87  main -> main


Done. `CONF_SYNC_ENABLED=false` in master.conf silences the failure for the rest of the HOST2 rebuild — both the array-start full sync and the intermediate `--pull-only` will just log "skipping" and exit 0. Flip it back to `true` when HOST2 is back online.


Fixed a recurring intermediate sync failure. When HOST2 is back online, set CONF_SYNC_ENABLED=true in master.conf.


unRAID-Gmer4Lfe
HOST1
USOWNER
✓ Local setup complete · waiting for partner
⚡ No API key — Monitor stats unavailable monitor page is fine... the button almost needs to map to the croned script


Let me look at the setup page and the API key script to understand what the button currently does vs. what it should do.


grep -n "api.key\|api_key\|No API\|USOWNER\|monitor\|button\|btn" /boot/config/plugins/varaverk/Plugin/unraid/pages/setup.php | head -60

Find API key and button references in setup.php


1	#!/bin/bash
2	# ==============================================================================================
3	# ================================= Unraid API Key Renewal ====================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Creates/overwrites the Varaverk API key in the unraid-api service registry at
9	# array start. The registry is ephemeral — OS updates and service restarts clear
10	# it. This script re-registers the key every boot so Varaverk's enhanced
11	# monitoring self-heals without manual intervention.
12	#
13	# Also updates HOST*_UNRAID_API_KEY in the local host conf so the partnership
14	# page always reflects the live key value.
15	#
16	# ==============================================================================================
17	# RUNTIME MODES
18	# ==============================================================================================
19	#
20	# unraid_api_key_renew.sh
21	#     Renew the key. Silent on success.
22	#
23	# unraid_api_key_renew.sh --dry-run
24	#     Show what would happen — no changes made.
25	#
26	# unraid_api_key_renew.sh --log
27	#     Verbose output.
28	#
29	# ==============================================================================================
30
31	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
32
33	source "$SCRIPT_DIR/../../../load_config.sh"
34
35	parse_args "$@"
36	acquire_lock
37	detect_hosts
38
39	# ──────────────────────────────────────────────────────────────────────────────
40	CONF_FILE="$SCRIPT_DIR/../../../Configurations/${MY_ID,,}.conf"
41	VAR_NAME="${MY_ID}_UNRAID_API_KEY"
42
43	# Key name: "Varaverk <hostname>" stripping any unraid- prefix
44	# Space separator — unRAID API only allows letters, numbers, and spaces
45	HOSTNAME_SUFFIX=$(hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//' || hostname -s)
46	KEY_NAME="Varaverk ${HOSTNAME_SUFFIX}"
47
48	log "$ICON_GEAR Conf file: $CONF_FILE"
49	log "$ICON_GEAR Key var:   $VAR_NAME"
50	log "$ICON_GEAR Key name:  $KEY_NAME"
51
52	if [[ ! -f "$CONF_FILE" ]]; then
53	    error "Conf file not found: $CONF_FILE"
54	    exit 1
55	fi
56
57	if [[ "$DRY_RUN" == true ]]; then
58	    warn "DRY RUN — would check registry for $KEY_NAME, renew only if missing"
59	    exit 0
60	fi
61
62	# ──────────────────────────────────────────────────────────────────────────────
63	# Check if key already exists in the unraid-api registry before creating.
64	# --overwrite generates a new key value every time, invalidating the old one.
65	# Only renew if the registry has lost it.
66	log "Checking unraid-api registry for $KEY_NAME..."
67	EXISTING=$(timeout 5 /usr/local/sbin/unraid-api apikey --name "$KEY_NAME" --json </dev/null 2>/dev/null)
68	KEY=$(echo "$EXISTING" | jq -r '.key // empty' 2>/dev/null)
69
70	if [[ -n "$KEY" ]]; then
71	    PREVIEW="${KEY:0:8}...${KEY: -4}"
72	    # Always sync registry key → conf, even if the key was already there.
73	    # Conf gets wiped on git pull / conf regeneration without touching the registry.
74	    CONF_HAS_KEY=$(grep -oP "(?<=^\s*${VAR_NAME}=\")[^\"]*" "$CONF_FILE" 2>/dev/null || true)
75	    if [[ "$CONF_HAS_KEY" == "$KEY" ]]; then
76	        echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
77	        log "Key in registry and conf — no action needed"
78	        exit 0
79	    fi
80	    log "Key in registry but conf is stale — syncing..."
81	    if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
82	        sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
83	    else
84	        echo "    ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
85	    fi
86	    echo "API key synced to conf ✅ — $VAR_NAME = $PREVIEW"
87	    exit 0
88	fi
89
90	log "Key not found in registry — creating new key..."
91
92	RAW=$(timeout 10 /usr/local/sbin/unraid-api apikey \
93	    --name "$KEY_NAME" --create --overwrite \
94	    --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1)
95
96	if [[ -z "$RAW" ]]; then
97	    error "unraid-api returned no output"
98	    exit 1
99	fi
100
101	KEY=$(echo "$RAW" | jq -r '.key // empty' 2>/dev/null)
102	if [[ -z "$KEY" ]]; then
103	    error "No key in unraid-api response: ${RAW:0:200}"
104	    exit 1
105	fi
106
107	# ──────────────────────────────────────────────────────────────────────────────
108	if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
109	    sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
110	else
111	    echo "    ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
112	fi
113
114	PREVIEW="${KEY:0:8}...${KEY: -4}"
115	log "Writing new key to: $CONF_FILE"
116	warn "API key renewed ✅ — $VAR_NAME = $PREVIEW  (registry had lost it)"
117
118	# ── Push renewed key into each partner's OWN conf ─────────────────────────────
119	# Each host's conf is its complete keychest — no cross-host conf files needed.
120	# SSH_KEY is set by detect_hosts() — this server's outbound private key.
121	if [[ -z "$SSH_KEY" ]]; then
122	    log "No SSH key configured — skipping partner push"
123	    exit 0
124	fi
125
126	for host_var in $(compgen -v | grep -E '^HOST[0-9]+$'); do
127	    partner_host="${!host_var}"
128	    [[ -z "$partner_host" ]] && continue
129	    [[ "${host_var,,}" == "${MY_ID,,}" ]] && continue
130
131	    partner_slot="${host_var,,}"   # e.g. host2
132	    partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
133	    [[ -z "$partner_ip" ]] && { log "Cannot resolve IP for $partner_host — skipping"; continue; }
134
135	    # Target is the partner's OWN conf on their machine
136	    partner_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
137	    tmp=$(mktemp /tmp/vv_kp_XXXXXX.sh)
138	    remote="/tmp/vv_kp_${RANDOM}.sh"
139	    chmod 700 "$tmp"
140
141	    # Key stays in the temp file — never appears in SSH command args
142	    cat > "$tmp" <<PUSHSCRIPT
143	#!/bin/sh
144	target='${partner_conf}'
145	if grep -q "\b${VAR_NAME}\b" "\$target" 2>/dev/null; then
146	  sed -i 's|^\(\\s*${VAR_NAME}\\s*=\\s*\)"[^"]*"|\1"${KEY}"|' "\$target"
147	else
148	  printf '    ${VAR_NAME}="%s"\n' '${KEY}' >> "\$target"
149	fi
150	echo ok
151	PUSHSCRIPT
152
153	    if timeout 10 scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
154	        -o StrictHostKeyChecking=no "$tmp" "root@${partner_ip}:${remote}" 2>/dev/null; then
155	        if timeout 10 ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
156	            -o StrictHostKeyChecking=no "root@${partner_ip}" \
157	            "bash '${remote}'; rc=\$?; rm -f '${remote}'; exit \$rc" 2>/dev/null | grep -q ok; then
158	            log "Key pushed to $partner_host ✅"
159	        else
160	            warn "Key push to $partner_host failed — they can create their own copy"
161	        fi
162	    else
163	        warn "SCP to $partner_host failed — skipping"
164	    fi
165	    rm -f "$tmp"
166	done
167

#!/bin/bash
# ==============================================================================================
# ================================= Unraid API Key Renewal ====================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Creates/overwrites the Varaverk API key in the unraid-api service registry at
# array start. The registry is ephemeral — OS updates and service restarts clear
# it. This script re-registers the key every boot so Varaverk's enhanced
# monitoring self-heals without manual intervention.
#
# Also updates HOST*_UNRAID_API_KEY in the local host conf so the partnership
# page always reflects the live key value.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# unraid_api_key_renew.sh
#     Renew the key. Silent on success.
#
# unraid_api_key_renew.sh --dry-run
#     Show what would happen — no changes made.
#
# unraid_api_key_renew.sh --log
#     Verbose output.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../../../load_config.sh"

parse_args "$@"
acquire_lock
detect_hosts

# ──────────────────────────────────────────────────────────────────────────────
CONF_FILE="$SCRIPT_DIR/../../../Configurations/${MY_ID,,}.conf"
VAR_NAME="${MY_ID}_UNRAID_API_KEY"

# Key name: "Varaverk <hostname>" stripping any unraid- prefix
# Space separator — unRAID API only allows letters, numbers, and spaces
HOSTNAME_SUFFIX=$(hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//' || hostname -s)
KEY_NAME="Varaverk ${HOSTNAME_SUFFIX}"

log "$ICON_GEAR Conf file: $CONF_FILE"
log "$ICON_GEAR Key var:   $VAR_NAME"
log "$ICON_GEAR Key name:  $KEY_NAME"

if [[ ! -f "$CONF_FILE" ]]; then
    error "Conf file not found: $CONF_FILE"
    exit 1
fi

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would check registry for $KEY_NAME, renew only if missing"
    exit 0
fi

# ──────────────────────────────────────────────────────────────────────────────
# Check if key already exists in the unraid-api registry before creating.
# --overwrite generates a new key value every time, invalidating the old one.
# Only renew if the registry has lost it.
log "Checking unraid-api registry for $KEY_NAME..."
EXISTING=$(timeout 5 /usr/local/sbin/unraid-api apikey --name "$KEY_NAME" --json </dev/null 2>/dev/null)
KEY=$(echo "$EXISTING" | jq -r '.key // empty' 2>/dev/null)

if [[ -n "$KEY" ]]; then
    PREVIEW="${KEY:0:8}...${KEY: -4}"
    # Always sync registry key → conf, even if the key was already there.
    # Conf gets wiped on git pull / conf regeneration without touching the registry.
    CONF_HAS_KEY=$(grep -oP "(?<=^\s*${VAR_NAME}=\")[^\"]*" "$CONF_FILE" 2>/dev/null || true)
    if [[ "$CONF_HAS_KEY" == "$KEY" ]]; then
        echo "API key valid ✅ — $VAR_NAME = $PREVIEW"
        log "Key in registry and conf — no action needed"
        exit 0
    fi
    log "Key in registry but conf is stale — syncing..."
    if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
        sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
    else
        echo "    ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
    fi
    echo "API key synced to conf ✅ — $VAR_NAME = $PREVIEW"
    exit 0
fi

log "Key not found in registry — creating new key..."

RAW=$(timeout 10 /usr/local/sbin/unraid-api apikey \
    --name "$KEY_NAME" --create --overwrite \
    --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1)

if [[ -z "$RAW" ]]; then
    error "unraid-api returned no output"
    exit 1
fi

KEY=$(echo "$RAW" | jq -r '.key // empty' 2>/dev/null)
if [[ -z "$KEY" ]]; then
    error "No key in unraid-api response: ${RAW:0:200}"
    exit 1
fi

# ──────────────────────────────────────────────────────────────────────────────
if grep -q "^\s*${VAR_NAME}\s*=" "$CONF_FILE"; then
    sed -i "s|^\(\s*${VAR_NAME}\s*=\s*\)\"[^\"]*\"|\1\"${KEY}\"|" "$CONF_FILE"
else
    echo "    ${VAR_NAME}=\"${KEY}\"" >> "$CONF_FILE"
fi

PREVIEW="${KEY:0:8}...${KEY: -4}"
log "Writing new key to: $CONF_FILE"
warn "API key renewed ✅ — $VAR_NAME = $PREVIEW  (registry had lost it)"

# ── Push renewed key into each partner's OWN conf ─────────────────────────────
# Each host's conf is its complete keychest — no cross-host conf files needed.
# SSH_KEY is set by detect_hosts() — this server's outbound private key.
if [[ -z "$SSH_KEY" ]]; then
    log "No SSH key configured — skipping partner push"
    exit 0
fi

for host_var in $(compgen -v | grep -E '^HOST[0-9]+$'); do
    partner_host="${!host_var}"
    [[ -z "$partner_host" ]] && continue
    [[ "${host_var,,}" == "${MY_ID,,}" ]] && continue

    partner_slot="${host_var,,}"   # e.g. host2
    partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)
    [[ -z "$partner_ip" ]] && { log "Cannot resolve IP for $partner_host — skipping"; continue; }

    # Target is the partner's OWN conf on their machine
    partner_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
    tmp=$(mktemp /tmp/vv_kp_XXXXXX.sh)
    remote="/tmp/vv_kp_${RANDOM}.sh"
    chmod 700 "$tmp"

    # Key stays in the temp file — never appears in SSH command args
    cat > "$tmp" <<PUSHSCRIPT
#!/bin/sh
target='${partner_conf}'
if grep -q "\b${VAR_NAME}\b" "\$target" 2>/dev/null; then
  sed -i 's|^\(\\s*${VAR_NAME}\\s*=\\s*\)"[^"]*"|\1"${KEY}"|' "\$target"
else
  printf '    ${VAR_NAME}="%s"\n' '${KEY}' >> "\$target"
fi
echo ok
PUSHSCRIPT

    if timeout 10 scp -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
        -o StrictHostKeyChecking=no "$tmp" "root@${partner_ip}:${remote}" 2>/dev/null; then
        if timeout 10 ssh -i "$SSH_KEY" -o ConnectTimeout=10 -o BatchMode=yes \
            -o StrictHostKeyChecking=no "root@${partner_ip}" \
            "bash '${remote}'; rc=\$?; rm -f '${remote}'; exit \$rc" 2>/dev/null | grep -q ok; then
            log "Key pushed to $partner_host ✅"
        else
            warn "Key push to $partner_host failed — they can create their own copy"
        fi
    else
        warn "SCP to $partner_host failed — skipping"
    fi
    rm -f "$tmp"
done


unRAID-Gmer4Lfe HOST1 USOWNER ✓ Local setup complete · waiting for partner ⚡ No API key — Monitor stats unavailable monitor page is fine... the button almost needs to map to the croned script


30:.vv-role-btn  { flex: 1; padding: 9px 0; background: #1a1a1a; border: 1px solid #333;
33:.vv-role-btn.active { border-color: #555; color: #ccc; background: #1e1e1e; }
37:.vv-btn       { width: 100%; padding: 10px; background: #1e1e1e; border: 1px solid #444;
40:.vv-btn:hover { border-color: #666; color: #eee; }
41:.vv-btn:disabled { opacity: .4; cursor: default; }
73:.vv-cl-act button { padding: 4px 10px; background: #1a1a1a; border: 1px solid #333; color: #888;
75:.vv-cl-act button:hover { border-color: #555; color: #bbb; }
98:

101:

131:    <button class="vv-btn" id="vv-main-btn" onclick="vvDoSave()">Save and continue →</button>
145:
Create your Unraid API key below — needed for live monitor stats

158:      <button id="vv-key-btn" onclick="vvCreateKey(this)" class="vv-btn" style="flex:1;background:#1a3a1a;border-color:#2e6b2e;color:#6fcf97;">
160:      </button>
218:  const b = document.getElementById('vv-main-btn');
232:    const btn    = document.getElementById('vv-key-btn');
234:    if (btn)    { btn.textContent = 'Created ✓'; btn.disabled = true; btn.style.opacity = '.6'; }
293:            act = `<div class="vv-cl-act"><button onclick="vvCreateKey(this)">${lbl}</button></div>`;
295:            act = `<div class="vv-cl-act"><button onclick="vvRunPopulateBtn(this)">${lbl}</button></div>`;
297:            act = `<div class="vv-cl-act"><button onclick="vvPullMaster(this)">${lbl}</button><div id="vv-pull-err" class="vv-cl-err"></div></div>`;
312:function vvRunPopulateBtn(btn) {
313:  btn.disabled = true; btn.textContent = '…';
317:  }).then(() => { btn.textContent = 'Done'; vvLoadChecklist(); })
318:    .catch(() => { btn.disabled = false; btn.textContent = 'Retry'; });
321:function vvPullMaster(btn) {
322:  btn.disabled = true; btn.textContent = '⟳ Pulling…';
330:      btn.textContent = '✓ Done';
334:      btn.disabled = false; btn.textContent = 'Retry';
336:  }).catch(() => { btn.disabled = false; btn.textContent = 'Retry'; });
340:function vvCreateKey(btn) {
342:  btn.disabled = true; btn.textContent = '⟳ Creating…';
343:  fetch('/plugins/varaverk/api/create_api_key.php?_=' + Date.now())
348:        btn.textContent = 'Created ✓'; btn.style.opacity = '.6';
353:        btn.disabled = false; btn.textContent = 'Retry';
357:      btn.disabled = false; btn.textContent = 'Retry';
382:    if (d.ok) { vvShowStep2(d.redirect || '?tab=scheduler', d.api_key); }


Now let me see what `create_api_key.php` does and what the setup page's checklist shows for the "No API key" state:


find /boot/config/plugins/varaverk/Plugin/unraid -name "create_api_key.php" 2>/dev/null

Find create_api_key.php


140
⟳ Running auto-populate…

141
142

143	      Quick start
144

145
Create your Unraid API key below — needed for live monitor stats

146
Open Scheduler → Edit host.conf — only three things need manual entry:

147
148	              EMBY_API_KEY — Emby Dashboard → API Keys → + New Key

149	              DISCORD_WEBHOOK — for notifications (optional)

150	              DAILY_SYNC_SHARES — media paths to rsync nightly

151	              Everything else was auto-populated or has working defaults
152

153
If partnering: the checklist below will guide you through pulling HOST1's config and running onboard

154

155

156
157

158	      <button id="vv-key-btn" onclick="vvCreateKey(this)" class="vv-btn" style="flex:1;background:#1a3a1a;border-color:#2e6b2e;color:#6fcf97;">
159	        Create API Key
160	      </button>
161	      Skip →
162

163

164
165

166
Setup checklist

167

Loading…

168
169

170	      Go to Scheduler →
171

172

173
174

175
176

⟳ Running auto-populate…

      Quick start

Create your Unraid API key below — needed for live monitor stats

Open Scheduler → Edit host.conf — only three things need manual entry:

              EMBY_API_KEY — Emby Dashboard → API Keys → + New Key

              DISCORD_WEBHOOK — for notifications (optional)

              DAILY_SYNC_SHARES — media paths to rsync nightly

              Everything else was auto-populated or has working defaults

If partnering: the checklist below will guide you through pulling HOST1's config and running onboard

      <button id="vv-key-btn" onclick="vvCreateKey(this)" class="vv-btn" style="flex:1;background:#1a3a1a;border-color:#2e6b2e;color:#6fcf97;">
        Create API Key
      </button>
      Skip →

Setup checklist

Loading…

      Go to Scheduler →


1	<?php
2	header('Content-Type: application/json');
3	header('Cache-Control: no-store, no-cache');
4	require_once dirname(__DIR__) . '/include/config.php';
5
6	$host = vv_detect_host();
7	if (!preg_match('/^host\d+$/', $host)) {
8	    echo json_encode(['ok' => false, 'error' => 'Cannot detect local host']);
9	    exit;
10	}
11
12	echo json_encode(vv_auto_create_api_key($host, $host . '.conf'));
13

<?php
header('Content-Type: application/json');
header('Cache-Control: no-store, no-cache');
require_once dirname(__DIR__) . '/include/config.php';

$host = vv_detect_host();
if (!preg_match('/^host\d+$/', $host)) {
    echo json_encode(['ok' => false, 'error' => 'Cannot detect local host']);
    exit;
}

echo json_encode(vv_auto_create_api_key($host, $host . '.conf'));


270	  pull_master:  'Pull from HOST1',
271	  onboard:      'Partnership tab →',
272	};
273	const vvActionHref = {
274	  ssh_setup: '?tab=partnership',
275	  onboard:   '?tab=partnership',
276	};
277
278	function vvLoadChecklist() {
279	  fetch('/plugins/varaverk/api/checklist.php?_=' + Date.now())
280	    .then(r => r.json()).then(d => {
281	      const el = document.getElementById('vv-checklist');
282	      if (!d.ok || !d.items) { el.innerHTML = 'Unable to load checklist'; return; }
283	      el.innerHTML = d.items.map(item => {
284	        const icon = item.ok === null ? '○' : (item.ok ? '✓' : '✗');
285	        const iclr = item.ok === null ? '#444' : (item.ok ? '#4a8' : '#a66');
286	        let act = '';
287	        if (item.action) {
288	          const lbl  = vvActionLabels[item.action] || item.action;
289	          const href = vvActionHref[item.action];
290	          if (href) {
291	            act = `<div class="vv-cl-act"><a href="${href}" style="font-size:11px;color:#556;">${lbl}</a></div>`;
292	          } else if (item.action === 'create_key') {
293	            act = `<div class="vv-cl-act"><button onclick="vvCreateKey(this)">${lbl}</button></div>`;
294	          } else if (item.action === 'run_populate') {
295	            act = `<div class="vv-cl-act"><button onclick="vvRunPopulateBtn(this)">${lbl}</button></div>`;
296	          } else if (item.action === 'pull_master') {
297	            act = `<div class="vv-cl-act"><button onclick="vvPullMaster(this)">${lbl}</button><div id="vv-pull-err" class="vv-cl-err"></div></div>`;
298	          }
299	        }
300	        return `

301
${icon}

302

303
${item.label}

304
${item.detail || ''}

305	            ${act}
306

307
`;
308	      }).join('');
309	    }).catch(() => {});
310	}
311
312	function vvRunPopulateBtn(btn) {
313	  btn.disabled = true; btn.textContent = '…';
314	  fetch('/plugins/varaverk/api/setup.php', {
315	    method: 'POST', headers: {'Content-Type': 'application/x-www-form-urlencoded'},
316	    body: new URLSearchParams({action: 'populate'})
317	  }).then(() => { btn.textContent = 'Done'; vvLoadChecklist(); })
318	    .catch(() => { btn.disabled = false; btn.textContent = 'Retry'; });
319	}
320
321	function vvPullMaster(btn) {
322	  btn.disabled = true; btn.textContent = '⟳ Pulling…';
323	  const errEl = document.getElementById('vv-pull-err');
324	  if (errEl) errEl.textContent = '';
325	  fetch('/plugins/varaverk/api/setup.php', {
326	    method: 'POST', headers: {'Content-Type': 'application/x-www-form-urlencoded'},
327	    body: new URLSearchParams({action: 'pull'})
328	  }).then(r => r.json()).then(d => {
329	    if (d.ok) {
330	      btn.textContent = '✓ Done';
331	      setTimeout(vvLoadChecklist, 600);
332	    } else {
333	      if (errEl) errEl.textContent = d.error || 'Pull failed';
334	      btn.disabled = false; btn.textContent = 'Retry';
335	    }
336	  }).catch(() => { btn.disabled = false; btn.textContent = 'Retry'; });
337	}
338
339	// ── API key ───────────────────────────────────────────────────────────────────
340	function vvCreateKey(btn) {
341	  const status = document.getElementById('vv-key-status');
342	  btn.disabled = true; btn.textContent = '⟳ Creating…';
343	  fetch('/plugins/varaverk/api/create_api_key.php?_=' + Date.now())
344	    .then(r => r.json()).then(d => {
345	      if (d.ok) {
346	        status.textContent = '✓ Key created — ' + d.key_preview;
347	        status.style.color = '#4a8';
348	        btn.textContent = 'Created ✓'; btn.style.opacity = '.6';
349	        vvLoadChecklist();
350	      } else {
351	        status.textContent = '✗ ' + (d.error || 'Failed');
352	        status.style.color = '#a44';
353	        btn.disabled = false; btn.textContent = 'Retry';
354	      }
355	    }).catch(e => {
356	      status.textContent = '✗ ' + e; status.style.color = '#a44';
357	      btn.disabled = false; btn.textContent = 'Retry';
358	    });
359	}
360
361	// ── Save ──────────────────────────────────────────────────────────────────────
362	function vvDoSave() {
363	  const hostname = document.getElementById('vv-hostname')?.value.trim();
364	  if (!hostname) { vvSetStatus('✗ Hostname is required', 'err'); return; }
365	  let host1 = '', host2 = '', mySlot = 'host1';
366	  if (vvRole === 'primary') {
367	    host1  = hostname;
368	    host2  = document.getElementById('vv-partner-hostname')?.value.trim() || '';
369	    mySlot = 'host1';
370	  } else {
371	    const primary = document.getElementById('vv-primary-hostname')?.value.trim();
372	    if (!primary) { vvSetStatus('✗ Primary hostname required', 'err'); return; }
373	    mySlot = document.getElementById('vv-partner-slot')?.value || 'host2';
374	    host1  = primary;
375	    if (mySlot === 'host2') host2 = hostname;
376	  }
377	  vvSetBtn('Saving…', true);
378	  fetch('/plugins/varaverk/api/setup.php', {
379	    method: 'POST', headers: {'Content-Type': 'application/x-www-form-urlencoded'},
380	    body: new URLSearchParams({action:'save', host1, host2, my_slot:mySlot, my_hostname:hostname})
381	  }).then(r => r.json()).then(d => {
382	    if (d.ok) { vvShowStep2(d.redirect || '?tab=scheduler', d.api_key); }
383	    else { vvSetBtn('Save and continue →', false); vvSetStatus('✗ ' + (d.error||'Error'), 'err'); }
384	  }).catch(() => { vvSetBtn('Save and continue →', false); vvSetStatus('✗ Request failed', 'err'); });
385	}
386	</script>
387

pull_master:  'Pull from HOST1',
  onboard:      'Partnership tab →',
};
const vvActionHref = {
  ssh_setup: '?tab=partnership',
  onboard:   '?tab=partnership',
};

function vvLoadChecklist() {
  fetch('/plugins/varaverk/api/checklist.php?_=' + Date.now())
    .then(r => r.json()).then(d => {
      const el = document.getElementById('vv-checklist');
      if (!d.ok || !d.items) { el.innerHTML = 'Unable to load checklist'; return; }
      el.innerHTML = d.items.map(item => {
        const icon = item.ok === null ? '○' : (item.ok ? '✓' : '✗');
        const iclr = item.ok === null ? '#444' : (item.ok ? '#4a8' : '#a66');
        let act = '';
        if (item.action) {
          const lbl  = vvActionLabels[item.action] || item.action;
          const href = vvActionHref[item.action];
          if (href) {
            act = `<div class="vv-cl-act"><a href="${href}" style="font-size:11px;color:#556;">${lbl}</a></div>`;
          } else if (item.action === 'create_key') {
            act = `<div class="vv-cl-act"><button onclick="vvCreateKey(this)">${lbl}</button></div>`;
          } else if (item.action === 'run_populate') {
            act = `<div class="vv-cl-act"><button onclick="vvRunPopulateBtn(this)">${lbl}</button></div>`;
          } else if (item.action === 'pull_master') {
            act = `<div class="vv-cl-act"><button onclick="vvPullMaster(this)">${lbl}</button><div id="vv-pull-err" class="vv-cl-err"></div></div>`;
          }
        }
        return `

${icon}

${item.label}

${item.detail || ''}

            ${act}

`;
      }).join('');
    }).catch(() => {});
}

function vvRunPopulateBtn(btn) {
  btn.disabled = true; btn.textContent = '…';
  fetch('/plugins/varaverk/api/setup.php', {
    method: 'POST', headers: {'Content-Type': 'application/x-www-form-urlencoded'},
    body: new URLSearchParams({action: 'populate'})
  }).then(() => { btn.textContent = 'Done'; vvLoadChecklist(); })
    .catch(() => { btn.disabled = false; btn.textContent = 'Retry'; });
}

function vvPullMaster(btn) {
  btn.disabled = true; btn.textContent = '⟳ Pulling…';
  const errEl = document.getElementById('vv-pull-err');
  if (errEl) errEl.textContent = '';
  fetch('/plugins/varaverk/api/setup.php', {
    method: 'POST', headers: {'Content-Type': 'application/x-www-form-urlencoded'},
    body: new URLSearchParams({action: 'pull'})
  }).then(r => r.json()).then(d => {
    if (d.ok) {
      btn.textContent = '✓ Done';
      setTimeout(vvLoadChecklist, 600);
    } else {
      if (errEl) errEl.textContent = d.error || 'Pull failed';
      btn.disabled = false; btn.textContent = 'Retry';
    }
  }).catch(() => { btn.disabled = false; btn.textContent = 'Retry'; });
}

// ── API key ───────────────────────────────────────────────────────────────────
function vvCreateKey(btn) {
  const status = document.getElementById('vv-key-status');
  btn.disabled = true; btn.textContent = '⟳ Creating…';
  fetch('/plugins/varaverk/api/create_api_key.php?_=' + Date.now())
    .then(r => r.json()).then(d => {
      if (d.ok) {
        status.textContent = '✓ Key created — ' + d.key_preview;
        status.style.color = '#4a8';
        btn.textContent = 'Created ✓'; btn.style.opacity = '.6';
        vvLoadChecklist();
      } else {
        status.textContent = '✗ ' + (d.error || 'Failed');
        status.style.color = '#a44';
        btn.disabled = false; btn.textContent = 'Retry';
      }
    }).catch(e => {
      status.textContent = '✗ ' + e; status.style.color = '#a44';
      btn.disabled = false; btn.textContent = 'Retry';
    });
}

// ── Save ──────────────────────────────────────────────────────────────────────
function vvDoSave() {
  const hostname = document.getElementById('vv-hostname')?.value.trim();
  if (!hostname) { vvSetStatus('✗ Hostname is required', 'err'); return; }
  let host1 = '', host2 = '', mySlot = 'host1';
  if (vvRole === 'primary') {
    host1  = hostname;
    host2  = document.getElementById('vv-partner-hostname')?.value.trim() || '';
    mySlot = 'host1';
  } else {
    const primary = document.getElementById('vv-primary-hostname')?.value.trim();
    if (!primary) { vvSetStatus('✗ Primary hostname required', 'err'); return; }
    mySlot = document.getElementById('vv-partner-slot')?.value || 'host2';
    host1  = primary;
    if (mySlot === 'host2') host2 = hostname;
  }
  vvSetBtn('Saving…', true);
  fetch('/plugins/varaverk/api/setup.php', {
    method: 'POST', headers: {'Content-Type': 'application/x-www-form-urlencoded'},
    body: new URLSearchParams({action:'save', host1, host2, my_slot:mySlot, my_hostname:hostname})
  }).then(r => r.json()).then(d => {
    if (d.ok) { vvShowStep2(d.redirect || '?tab=scheduler', d.api_key); }
    else { vvSetBtn('Save and continue →', false); vvSetStatus('✗ ' + (d.error||'Error'), 'err'); }
  }).catch(() => { vvSetBtn('Save and continue →', false); vvSetStatus('✗ Request failed', 'err'); });
}
</script>


grep -n "vv_auto_create_api_key\|unraid.api\|apikey\|UNRAID_API_KEY" /boot/config/plugins/varaverk/Plugin/unraid/include/config.php | head -40

Find API key functions in config.php


218:function vv_unraid_api_query(string $hostId, string $gql, int $timeoutSec = 5, string $apiKey = ''): ?array {
220:    $key  = $apiKey ?: ($vars[strtoupper($hostId) . '_UNRAID_API_KEY'] ?? '');
362:function vv_auto_create_api_key(string $hostId, string $confFile): array {
363:    $varName = strtoupper($hostId) . '_UNRAID_API_KEY';
366:    $output   = shell_exec('timeout 10 /usr/local/sbin/unraid-api apikey --name ' . escapeshellarg($keyName) . ' --create --overwrite --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1');
368:        return ['ok' => false, 'error' => 'unraid-api returned no output'];


362	function vv_auto_create_api_key(string $hostId, string $confFile): array {
363	    $varName = strtoupper($hostId) . '_UNRAID_API_KEY';
364	    $hostname = trim((string)shell_exec("hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//'")) ?: 'Varaverk';
365	    $keyName  = 'Varaverk ' . $hostname;
366	    $output   = shell_exec('timeout 10 /usr/local/sbin/unraid-api apikey --name ' . escapeshellarg($keyName) . ' --create --overwrite --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1');
367	    if (!$output) {
368	        return ['ok' => false, 'error' => 'unraid-api returned no output'];
369	    }
370	    $data = json_decode(trim($output), true);
371	    $key  = $data['key'] ?? null;
372	    if (!$key) {
373	        return ['ok' => false, 'error' => 'No key in response'];
374	    }
375	    $raw = vv_read_conf_raw($confFile);
376	    if ($raw === '') {
377	        return ['ok' => false, 'error' => 'Cannot read ' . $confFile];
378	    }
379	    if (!str_contains($raw, $varName)) {
380	        foreach ([strtoupper($hostId) . '_OWNER_EMAIL', strtoupper($hostId) . '_SSH_KEY'] as $anchor) {
381	            if (str_contains($raw, $anchor)) {
382	                $raw = preg_replace('/^(\s*' . preg_quote($anchor, '/') . '\s*=.*$)/m',
383	                    '$1' . "\n    " . $varName . '=""', $raw, 1);
384	                break;
385	            }
386	        }
387	    }
388	    $raw = preg_replace('/^(\s*' . preg_quote($varName, '/') . '\s*=\s*)"[^"]*"/m',
389	        '${1}"' . $key . '"', $raw);
390	    vv_write_conf_raw($confFile, $raw);
391	    return ['ok' => true, 'key_preview' => substr($key, 0, 8) . '...' . substr($key, -4)];
392	}
393
394	// Local LAN IP via routing table — static-cached per request.
395	// Previously duplicated in include/docker_folders.php and inline in include/docker.php.
396	function vv_local_ip(): string {
397	    static $ip = null;
398	    if ($ip !== null) return $ip;
399	    $ip = trim(shell_exec("ip route get 8.8.8.8 2>/dev/null | awk '/src/{for(i=1;i<=NF;i++)if(\$i==\"src\")print \$(i+1)}'") ?? '');
400	    return $ip;
401	}
402

function vv_auto_create_api_key(string $hostId, string $confFile): array {
    $varName = strtoupper($hostId) . '_UNRAID_API_KEY';
    $hostname = trim((string)shell_exec("hostname -s 2>/dev/null | sed 's/^[Uu][Nn][Rr][Aa][Ii][Dd]-//'")) ?: 'Varaverk';
    $keyName  = 'Varaverk ' . $hostname;
    $output   = shell_exec('timeout 10 /usr/local/sbin/unraid-api apikey --name ' . escapeshellarg($keyName) . ' --create --overwrite --description "Varaverk plugin" --roles ADMIN --json </dev/null 2>&1');
    if (!$output) {
        return ['ok' => false, 'error' => 'unraid-api returned no output'];
    }
    $data = json_decode(trim($output), true);
    $key  = $data['key'] ?? null;
    if (!$key) {
        return ['ok' => false, 'error' => 'No key in response'];
    }
    $raw = vv_read_conf_raw($confFile);
    if ($raw === '') {
        return ['ok' => false, 'error' => 'Cannot read ' . $confFile];
    }
    if (!str_contains($raw, $varName)) {
        foreach ([strtoupper($hostId) . '_OWNER_EMAIL', strtoupper($hostId) . '_SSH_KEY'] as $anchor) {
            if (str_contains($raw, $anchor)) {
                $raw = preg_replace('/^(\s*' . preg_quote($anchor, '/') . '\s*=.*$)/m',
                    '$1' . "\n    " . $varName . '=""', $raw, 1);
                break;
            }
        }
    }
    $raw = preg_replace('/^(\s*' . preg_quote($varName, '/') . '\s*=\s*)"[^"]*"/m',
        '${1}"' . $key . '"', $raw);
    vv_write_conf_raw($confFile, $raw);
    return ['ok' => true, 'key_preview' => substr($key, 0, 8) . '...' . substr($key, -4)];
}

// Local LAN IP via routing table — static-cached per request.
// Previously duplicated in include/docker_folders.php and inline in include/docker.php.
function vv_local_ip(): string {
    static $ip = null;
    if ($ip !== null) return $ip;
    $ip = trim(shell_exec("ip route get 8.8.8.8 2>/dev/null | awk '/src/{for(i=1;i<=NF;i++)if(\$i==\"src\")print \$(i+1)}'") ?? '');
    return $ip;
}


grep -n "SCRIPTS_ROOT\|define.*ROOT\|define.*PATH" /boot/config/plugins/varaverk/Plugin/unraid/include/config.php | head -20

Find SCRIPTS_ROOT constant in config.php


(Bash completed with no output)


grep -n "define\|SCRIPTS_ROOT\|PLUGIN_ROOT\|__DIR__" /boot/config/plugins/varaverk/Plugin/unraid/include/config.php | head -20

Find root path constants in config.php


5:define('PLUGIN_CFG',  '/boot/config/plugins/varaverk/varaverk.cfg');
8:define('SCRIPTS_DIR', $_vv_cfg['SCRIPTS_DIR'] ?? '/boot/config/plugins/varaverk');
9:define('CONF_DIR',    SCRIPTS_DIR . '/Configurations');
10:define('DATA_DIR',    SCRIPTS_DIR . '/data');
11:define('STATE_DIR',   SCRIPTS_DIR . '/State_Files');
12:define('LOG_DIR',     '/var/log/varaverk');
15:define('VV_SETUP_STATE_FILE', STATE_DIR . '/varaverk_setup.db');
16:define('VV_CACHE_DIR',        '/tmp/vv_cache');


grep -rn "vv_auto_create_api_key" /boot/config/plugins/varaverk/Plugin/unraid/

Check all callers of vv_auto_create_api_key


/boot/config/plugins/varaverk/Plugin/unraid/include/config.php:362:function vv_auto_create_api_key(string $hostId, string $confFile): array {
/boot/config/plugins/varaverk/Plugin/unraid/api/create_api_key.php:12:echo json_encode(vv_auto_create_api_key($host, $host . '.conf'));
/boot/config/plugins/varaverk/Plugin/unraid/api/setup.php:159:    $apiKeyResult = vv_auto_create_api_key($hostId, $confFile);
/boot/config/plugins/varaverk/Plugin/unraid/api/setup.php:252:$apiKeyResult = vv_auto_create_api_key($hostId, $confFile);


145	            $storageInternal2 = ($transport2 !== 'usb') ? 'true' : 'false';
146	            $conf = str_replace('HOSTN', $hostId,    $template);
147	            $conf = str_replace('hostn', $hostIdLow, $conf);
148	            $conf = preg_replace('/^(\s*' . $hostId . '_SSH_KEY\s*=\s*)""/m',
149	                '${1}"' . $sshKey . '"', $conf);
150	            $conf = preg_replace('/^(\s*' . $hostId . '_STORAGE_MODE_INTERNAL\s*=\s*)\S+/m',
151	                '${1}' . $storageInternal2, $conf);
152	            vv_write_conf_raw($confFile, $conf);
153	        }
154	    }
155
156	    if (file_exists($sshScript)) {
157	        exec('bash ' . escapeshellarg($sshScript) . ' --local-only 2>/dev/null');
158	    }
159	    $apiKeyResult = vv_auto_create_api_key($hostId, $confFile);
160
161	    $state = vv_setup_state_read();
162	    $state['master_conf_pulled'] = 'true';
163	    vv_setup_state_write($state);
164
165	    echo json_encode(['ok' => true, 'host_id' => $hostId, 'conf_file' => $confFile,
166	                      'api_key'  => $apiKeyResult,
167	                      'redirect' => '?tab=scheduler&vv_setup=' . $confFile]);
168	    exit;
169	}
170
171	// ── Default action: save (HOST1 first-run wizard) ────────────────────────────────────────────
172	$host1      = trim($_POST['host1']      ?? '');
173	$host2      = trim($_POST['host2']      ?? '');
174	$mySlot     = trim($_POST['my_slot']    ?? 'host1');
175	$myHostname = trim($_POST['my_hostname'] ?? '');
176
177	if (empty($host1)) {
178	    echo json_encode(['ok' => false, 'error' => 'HOST1 hostname is required']);
179	    exit;
180	}
181	if (!preg_match('/^host\d+$/', $mySlot)) {
182	    echo json_encode(['ok' => false, 'error' => 'Invalid slot']);
183	    exit;
184	}

            $storageInternal2 = ($transport2 !== 'usb') ? 'true' : 'false';
            $conf = str_replace('HOSTN', $hostId,    $template);
            $conf = str_replace('hostn', $hostIdLow, $conf);
            $conf = preg_replace('/^(\s*' . $hostId . '_SSH_KEY\s*=\s*)""/m',
                '${1}"' . $sshKey . '"', $conf);
            $conf = preg_replace('/^(\s*' . $hostId . '_STORAGE_MODE_INTERNAL\s*=\s*)\S+/m',
                '${1}' . $storageInternal2, $conf);
            vv_write_conf_raw($confFile, $conf);
        }
    }

    if (file_exists($sshScript)) {
        exec('bash ' . escapeshellarg($sshScript) . ' --local-only 2>/dev/null');
    }
    $apiKeyResult = vv_auto_create_api_key($hostId, $confFile);

    $state = vv_setup_state_read();
    $state['master_conf_pulled'] = 'true';
    vv_setup_state_write($state);

    echo json_encode(['ok' => true, 'host_id' => $hostId, 'conf_file' => $confFile,
                      'api_key'  => $apiKeyResult,
                      'redirect' => '?tab=scheduler&vv_setup=' . $confFile]);
    exit;
}

// ── Default action: save (HOST1 first-run wizard) ────────────────────────────────────────────
$host1      = trim($_POST['host1']      ?? '');
$host2      = trim($_POST['host2']      ?? '');
$mySlot     = trim($_POST['my_slot']    ?? 'host1');
$myHostname = trim($_POST['my_hostname'] ?? '');

if (empty($host1)) {
    echo json_encode(['ok' => false, 'error' => 'HOST1 hostname is required']);
    exit;
}
if (!preg_match('/^host\d+$/', $mySlot)) {
    echo json_encode(['ok' => false, 'error' => 'Invalid slot']);
    exit;
}


240	    }
241	}
242
243	// Write setup state file — lets partner servers know HOST1 is configured
244	vv_setup_state_write(['host1_hostname' => $host1]);
245
246	// Auto-generate SSH keypair (local only — remote copy happens during onboarding)
247	if (file_exists($sshScript)) {
248	    exec('bash ' . escapeshellarg($sshScript) . ' --local-only 2>/dev/null');
249	}
250
251	// Auto-create Unraid API key and write into the fresh conf
252	$apiKeyResult = vv_auto_create_api_key($hostId, $confFile);
253
254	echo json_encode([
255	    'ok'       => true,
256	    'host_id'  => $hostId,
257	    'api_key'  => $apiKeyResult,
258	    'redirect' => '?tab=scheduler&vv_setup=master.conf',
259	]);
260

    }
}

// Write setup state file — lets partner servers know HOST1 is configured
vv_setup_state_write(['host1_hostname' => $host1]);

// Auto-generate SSH keypair (local only — remote copy happens during onboarding)
if (file_exists($sshScript)) {
    exec('bash ' . escapeshellarg($sshScript) . ' --local-only 2>/dev/null');
}

// Auto-create Unraid API key and write into the fresh conf
$apiKeyResult = vv_auto_create_api_key($hostId, $confFile);

echo json_encode([
    'ok'       => true,
    'host_id'  => $hostId,
    'api_key'  => $apiKeyResult,
    'redirect' => '?tab=scheduler&vv_setup=master.conf',
]);


The PHP function always `--create --overwrite`, which generates a brand-new key every time (invalidating the previous one). The shell script checks the registry first, only creates if it's actually gone. Cleanest fix: rewrite `vv_auto_create_api_key` to delegate to the shell script — setup.php and create_api_key.php both pick it up automatically.