#!/bin/bash
# ==============================================================================================
# ================================= Unraid Platform Adapter ====================================
# ==============================================================================================
# Sourced by load_config.sh when PLATFORM=unraid.
# Provides the platform_*() API — bash scripts call these instead of OS-specific commands.
#
# ── API CONTRACT ──────────────────────────────────────────────────────────────────────────────
# Every function returns 0 on success / 1 on failure unless noted.
# Functions that produce output write to stdout; callers capture with $().
# No function calls exit — callers decide what failure means for their flow.
#
# ── ADDING A PLATFORM ─────────────────────────────────────────────────────────────────────────
# Create Plugin/truenas/adapter.sh (or ubuntu/adapter.sh) implementing the same function names.
# load_config.sh sources Plugin/$PLATFORM/adapter.sh — no other changes needed.
#
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
#   platform_require_cmd          — verify a platform command exists and is executable
#   platform_storage_healthy      — array mounted and shfs active on /mnt/user
#   platform_disk_states_path     — path to the platform disk state file
#   platform_get_disk_states      — raw disk state content from platform
#   platform_get_temp_thresholds  — disk warn/crit °C from platform config
#   platform_is_maintenance_running — parity check/sync in progress
#   platform_is_service_enabled   — docker or libvirt enabled in boot config
#   platform_restart_service      — restart a named service via rc.d
#   platform_stop_service         — stop a named service via rc.d
#   platform_is_service_running   — check if a named service process is alive
#   platform_is_mover_running     — unRAID mover process check
#   platform_stop_user_scripts    — kill all user.scripts background processes
#   platform_send_os_notification — native unRAID notify (dynamix)
#   platform_storage_path         — root path for user shares/storage (e.g. /mnt/user)
#   platform_webui_install_path   — where the platform serves the WebGUI plugin files from
#   platform_scripts_dir_probe_cmd — shell command to run on a remote to discover its SCRIPTS_DIR
#   platform_get_templates_dir    — path to Unraid CA docker templates-user directory
#   platform_rebuild_container    — rebuild a container from its stored XML template
#   platform_push_conf            — push master.conf to all listed hosts via WebGUI PHP
#   platform_push_setup_state     — push wizard setup state to WebGUI PHP
# ==============================================================================================

# ──────────────────────────────────────────────────────────────────────────────────────────────
# Internal: map a logical service name → its rc.d script path
# ──────────────────────────────────────────────────────────────────────────────────────────────
_platform_rc_script() {
    case "$1" in
        docker)  echo "/etc/rc.d/rc.docker" ;;
        sshd)    echo "/etc/rc.d/rc.sshd" ;;
        libvirt) echo "/etc/rc.d/rc.libvirt" ;;
        rsyslog) echo "/etc/rc.d/rc.rsyslogd" ;;
        nginx)   echo "/etc/rc.d/rc.nginx" ;;
        php-fpm) echo "/etc/rc.d/rc.php-fpm" ;;
        emhttp)  echo "/etc/rc.d/rc.emhttp" ;;
        *)       echo "/etc/rc.d/rc.$1" ;;
    esac
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_require_cmd <path> [test_arg] [expected_output] [label]
#   Replaces platform_require_cmd. Returns 0 if the command exists and (optionally) its output
#   matches expected_output. Returns 1 and prints a warning if not.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_require_cmd() {
    local cmd="$1"
    local test_arg="${2:-}"
    local expected="${3:-}"
    local label="${4:-$cmd}"

    if [[ ! -x "$cmd" ]]; then
        return 1
    fi
    if [[ -n "$test_arg" && -n "$expected" ]]; then
        "$cmd" $test_arg 2>&1 | grep -q "$expected" || return 1
    fi
    return 0
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_storage_healthy
#   Returns 0 if /mnt/user is mounted as shfs (array is up and healthy).
#   Returns 1 if the mount is absent or is not shfs (array stopped / degraded).
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_storage_healthy() {
    df --output=fstype /mnt/user 2>/dev/null | grep -q shfs
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_get_disk_states
#   Writes raw /var/local/emhttp/disks.ini to stdout.
#   Returns 1 if the file is absent (array not started or emhttp not running).
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_disk_states_path() {
    echo "/var/local/emhttp/disks.ini"
}

platform_get_disk_states() {
    local disks_ini
    disks_ini=$(platform_disk_states_path)
    [[ -f "$disks_ini" ]] || return 1
    cat "$disks_ini"
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_get_temp_thresholds
#   Writes four space-separated values to stdout: HDD_HOT HDD_MAX SSD_HOT SSD_MAX (°C integers).
#   Falls back to 45/55/60/70 if dynamix.cfg is absent or keys are missing.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_get_temp_thresholds() {
    local cfg="/boot/config/plugins/dynamix/dynamix.cfg"
    local hdd_hot hdd_max ssd_hot ssd_max
    hdd_hot=$(grep -m1 '^hot='    "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"')
    hdd_max=$(grep -m1 '^max='    "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"')
    ssd_hot=$(grep -m1 '^hotssd=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"')
    ssd_max=$(grep -m1 '^maxssd=' "$cfg" 2>/dev/null | cut -d= -f2 | tr -d '"')
    echo "${hdd_hot:-45} ${hdd_max:-55} ${ssd_hot:-60} ${ssd_max:-70}"
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_is_maintenance_running
#   Returns 0 if a parity check or sync is currently in progress.
#   Checks var.ini (Unraid 7.3+) then falls back to parity-date.txt (older).
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_is_maintenance_running() {
    local resync
    resync=$(awk -F'"' '/^mdResync=/{print $2}' /var/local/emhttp/var.ini 2>/dev/null)
    if [[ -n "$resync" && "$resync" != "0" ]]; then
        return 0
    fi
    grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_is_service_enabled <service>
#   service: docker | libvirt
#   Returns 0 if the service is enabled in the Unraid boot config.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_is_service_enabled() {
    case "$1" in
        docker)  grep -q '^DOCKER_ENABLED=yes' /boot/config/docker.cfg  2>/dev/null ;;
        libvirt) grep -q '^DOMAIN_ENABLE=yes'  /boot/config/domain.cfg  2>/dev/null ;;
        *)       return 1 ;;
    esac
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_restart_service <service>
#   Restarts the named service via its rc.d script.
#   Returns 1 if the rc.d script does not exist or is not executable.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_restart_service() {
    local rc
    rc=$(_platform_rc_script "$1")
    [[ -x "$rc" ]] || return 1
    "$rc" restart >/dev/null 2>&1
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_stop_service <service>
#   Stops the named service via its rc.d script.
#   Returns 1 if the rc.d script does not exist or is not executable.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_stop_service() {
    local rc
    rc=$(_platform_rc_script "$1")
    [[ -x "$rc" ]] || return 1
    "$rc" stop >/dev/null 2>&1
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_is_service_running <service>
#   Returns 0 if the service's main process is alive.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_is_service_running() {
    case "$1" in
        docker)  pgrep -x dockerd   >/dev/null 2>&1 ;;
        emhttp)  pgrep -x emhttpd   >/dev/null 2>&1 ;;
        nginx)   pgrep -x nginx     >/dev/null 2>&1 ;;
        php-fpm) pgrep -x php-fpm   >/dev/null 2>&1 ;;
        sshd)    pgrep -x sshd      >/dev/null 2>&1 ;;
        rsyslog) pgrep -x rsyslogd  >/dev/null 2>&1 ;;
        libvirt) pgrep -x libvirtd  >/dev/null 2>&1 ;;
        *)       pgrep -x "$1"      >/dev/null 2>&1 ;;
    esac
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_is_mover_running
#   Returns 0 if the Unraid mover is currently active.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_is_mover_running() {
    pgrep -f "emhttp.*Mover" >/dev/null 2>&1
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_get_mover_pid
#   Writes the mover's PID to stdout. Returns 1 if the mover is not running.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_get_mover_pid() {
    local pid
    pid=$(pgrep -f "emhttp.*Mover" | head -1)
    [[ -n "$pid" ]] || return 1
    echo "$pid"
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_stop_user_scripts
#   Kills all Unraid user.scripts background processes.
#   Returns 0 whether or not any processes were found (pkill exits 1 on no match).
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_stop_user_scripts() {
    pkill -f "/tmp/user.scripts" 2>/dev/null || true
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_send_os_notification <message> [subject] [severity]
#   Sends a native Unraid notification via the dynamix notify script.
#   severity: normal | warning | alert  (default: normal)
#   Returns 1 if the notify script is absent.
# ──────────────────────────────────────────────────────────────────────────────────────────────
# ── Platform path defaults ─────────────────────────────────────────────────────────────────────
# Set here so master.conf and core scripts never contain OS-specific path literals.
# host*.conf may override any of these after the adapter is sourced.
DOCKER_APPDATA_BASE="${DOCKER_APPDATA_BASE:-/mnt/user/appdata}"

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_storage_path
#   Writes the root path where user shares and storage are accessible to stdout.
#   Used by array health checks and disk verification.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_storage_path() {
    echo "/mnt/user"
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_webui_install_path
#   Writes the path where the platform serves WebGUI plugin files from.
#   On Unraid the PHP WebGUI is served from /boot/, separate from SCRIPTS_DIR.
#   Returns 1 (empty) if the platform serves directly from SCRIPTS_DIR (no sync needed).
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_webui_install_path() {
    echo "/boot/config/plugins/varaverk"
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_scripts_dir_probe_cmd
#   Writes a shell command suitable for running on a remote via SSH to discover
#   that remote's SCRIPTS_DIR. Each platform exposes its install path differently.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_scripts_dir_probe_cmd() {
    echo "grep -m1 'SCRIPTS_DIR' /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d '\"'"
}

platform_send_os_notification() {
    local message="$1"
    local subject="${2:-Varaverk}"
    local severity="${3:-normal}"
    local notify_script="/usr/local/emhttp/plugins/dynamix/scripts/notify"

    [[ -x "$notify_script" ]] || return 1
    "$notify_script" -e "Varaverk" -s "$subject" -d "$message" -i "$severity" 2>/dev/null
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_get_templates_dir
#   Writes the path to the Unraid CA docker templates-user directory to stdout.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_get_templates_dir() {
    echo "/boot/config/plugins/dockerMan/templates-user"
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_rebuild_container <container_name>
#   Rebuilds a container from its stored Unraid CA XML template (stops old, recreates on new
#   image digest, prunes old image). Returns 1 if the rebuild script is absent or fails.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_rebuild_container() {
    local container="$1"
    local rebuild_script="/usr/local/emhttp/plugins/dynamix.docker.manager/scripts/rebuild_container"
    [[ -x "$rebuild_script" ]] || return 1
    "$rebuild_script" "$container" >/dev/null 2>&1
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_push_conf
#   Pushes master.conf to all listed hosts via the WebGUI PHP API.
#   Writes per-host push results to stdout. Returns 1 if any host failed.
#   Returns 0 silently if php is unavailable (caller should warn manually).
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_push_conf() {
    command -v php &>/dev/null || return 0
    local rc=0
    php -r "
require_once '/usr/local/emhttp/plugins/varaverk/include/config.php';
\$results = vv_push_master_conf();
if (empty(\$results)) { echo 'no remote hosts'; exit(0); }
\$failed = 0;
foreach (\$results as \$r) {
    echo \$r['host'] . ': ' . (\$r['ok'] ? 'pushed' : 'FAILED — ' . \$r['error']) . PHP_EOL;
    if (!\$r['ok']) \$failed++;
}
exit(\$failed > 0 ? 1 : 0);
" 2>/dev/null
    rc=$?
    return $rc
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_push_setup_state
#   Pushes the Varaverk wizard setup state to all partners via the WebGUI PHP API.
#   No-op if php is unavailable. Always returns 0.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_push_setup_state() {
    command -v php &>/dev/null || return 0
    php -r "require_once '/usr/local/emhttp/plugins/varaverk/include/config.php'; vv_push_setup_state();" 2>/dev/null
    return 0
}


#!/bin/bash
# ==============================================================================================
# ============================= Docker Update — Remaining ======================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Weekly sweep that pulls the latest image for every running container not
# already covered by the daily or weekly managed update cycles. Restarts
# containers that received a new image, then prunes dangling images.
#
# Called by weekly_sync_maintenance.sh as the final step in the weekly window.
# Derives its target list automatically from docker ps minus the two managed
# lists — there is nothing to configure for this script.
#
# Together with docker_update.sh (normal + remainder modes), every deployed
# container receives at least one image pull per week without any per-container
# configuration required here.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Lock Acquisition
#   Prevents concurrent execution via acquire_lock(). Safe to call from
#   weekly maintenance scripts without risk of overlap.
#
# Host Detection
#   detect_hosts() identifies which server is running the script and aliases
#   HOST*_DAILY_RESTART_CONTAINERS and HOST*_WEEKLY_RESTART_CONTAINERS to
#   the correct host's values for exclusion.
#
# Root Enforcement
#   Docker operations require root privileges.
#
# WEEKLY_REMAINING_UPDATES Toggle
#   Exits cleanly when disabled via master.conf.
#
# Running-Only Filter
#   Stopped containers excluded — intentionally down, pulling adds no value.
#
# Image ID Comparison
#   Containers not restarted unless their image actually changed.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   WEEKLY_REMAINING_UPDATES
#       Enable or disable this script. (default: true)
#       To disable without the toggle: remove from WEEKLY_MAINTENANCE_SCRIPTS.
#
# host*.conf
#
#   HOST*_DAILY_RESTART_CONTAINERS
#       Excluded from this script — already updated daily. Aliased by
#       detect_hosts() → DAILY_RESTART_CONTAINERS
#
#   HOST*_WEEKLY_RESTART_CONTAINERS
#       Excluded from this script — already updated by weekly sync window.
#       Aliased by detect_hosts() → WEEKLY_RESTART_CONTAINERS
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# docker_update_remaining.sh
#     Pull all remaining running containers, restart those updated, prune images
#
# docker_update_remaining.sh --dry-run
#     Preview which containers would be pulled and restarted
#
# docker_update_remaining.sh --status
#     Show exclusion lists and current remaining container count
#
# docker_update_remaining.sh --log
#     Verbose per-container pull and restart output
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts

if [[ "${WEEKLY_REMAINING_UPDATES:-true}" != "true" ]]; then
    echo "WEEKLY_REMAINING_UPDATES=false — skipping remaining container updates"
    exit 0
fi

# ── Build exclusion set from daily + weekly managed lists ─────────────────────────────────────
declare -A EXCLUDED
for c in "${DAILY_RESTART_CONTAINERS[@]}"  "${WEEKLY_RESTART_CONTAINERS[@]}"; do
    [[ -n "$c" ]] && EXCLUDED["$c"]=1
done

# ── Get all running containers ────────────────────────────────────────────────────────────────
mapfile -t ALL_RUNNING < <(docker ps --format '{{.Names}}' 2>/dev/null | sort)

# ── Derive remainder: running minus excluded ──────────────────────────────────────────────────
REMAINING=()
for c in "${ALL_RUNNING[@]}"; do
    [[ -z "$c" ]] && continue
    [[ -n "${EXCLUDED[$c]:-}" ]] && continue
    REMAINING+=("$c")
done

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR       Enabled:     ${WEEKLY_REMAINING_UPDATES:-true}"
    echo "$ICON_CONTAINERS All running: ${#ALL_RUNNING[@]}"
    echo "$ICON_CONTAINERS Excluded:    ${!EXCLUDED[*]}"
    echo "$ICON_CONTAINERS Remaining:   ${REMAINING[*]:-none}"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

if [[ ${#REMAINING[@]} -eq 0 ]]; then
    echo "No remaining containers to update — all running containers are covered by daily/weekly lists"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled or containers restarted"

# ==============================================================================================
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
# ==============================================================================================

# docker_cmd, retry_docker, verify_running — defined in common.sh

# ==============================================================================================
# ━━━ Pull Updates ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_CONTAINERS Docker Update (Remaining) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
log "$ICON_CONTAINERS Containers: ${REMAINING[*]}"
log "$ICON_CONTAINERS Excluded (managed elsewhere): ${!EXCLUDED[*]}"
echo ""

START=$(date +%s)
UPDATED=()
UP_TO_DATE=()
FAILED=()
OLD_IMAGE_IDS=()

for container in "${REMAINING[@]}"; do
    [[ -z "$container" ]] && continue
    log "━━━ $ICON_CONTAINERS $container ━━━"

    IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
    if [[ -z "$IMAGE" ]]; then
        warn "$container — could not determine image, skipping"
        FAILED+=("$container")
        continue
    fi

    log "$container — image: $IMAGE"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would pull: $IMAGE"
        UPDATED+=("$container")
        continue
    fi

    CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
    OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")

    log "$ICON_SYNC  Pulling $IMAGE..."
    if [[ "$ENABLE_LOGGING" == "true" ]]; then
        docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/  /'
        _pull_rc=${PIPESTATUS[0]}
    else
        docker pull "$IMAGE" >/dev/null 2>&1
        _pull_rc=$?
    fi
    NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")

    if [[ $_pull_rc -eq 0 ]]; then
        _pull_new=$(  [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
        _container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false)

        if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then
            [[ "$_pull_new" == true ]]       && log "$ICON_DONE  $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})"
            [[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE  $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})"
            UPDATED+=("$container")
            OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID")
        else
            log "$container — up to date (${NEW_ID:7:12})"
            UP_TO_DATE+=("$container")
        fi
    else
        warn "$container — pull failed ($IMAGE)"
        FAILED+=("$container")
    fi

done

# ==============================================================================================
# ━━━ Restart Updated Containers ━━━
# ==============================================================================================
RESTARTED=()
RESTART_FAILED=()
SKIPPED_STOPPED=()

if [[ ${#UPDATED[@]} -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_CONTAINERS Restarting Updated Containers — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
    log "$ICON_CONTAINERS Containers with new image: ${UPDATED[*]}"

    for container in "${UPDATED[@]}"; do
        [[ -z "$container" ]] && continue
        log "━━━ $ICON_CONTAINERS $container ━━━"

        STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)

        if [[ "$STATUS" != "true" ]]; then
            log "$ICON_NOT_RUNNING $container is stopped — skipping restart (respecting stopped state)"
            SKIPPED_STOPPED+=("$container")
            continue
        fi

        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would restart $container"
            RESTARTED+=("$container")
            continue
        fi

        log "$ICON_RUNNING $container — recreating from template on new image..."
        if platform_rebuild_container "$container"; then
            if verify_running "$container"; then
                log "$ICON_DONE  $container recreated and running ✅"
                RESTARTED+=("$container")
            else
                error "$container recreated but not running — may be intentionally stopped"
                RESTARTED+=("$container")
            fi
        else
            error "Failed to rebuild $container from template"
            notify "$container failed to rebuild after update on $(hostname)" "Docker Update Remaining" "warning"
            RESTART_FAILED+=("$container")
        fi
    done
else
    log "No containers received a new image — nothing to restart"
fi

# ==============================================================================================
# ━━━ Prune Old Images ━━━
# ==============================================================================================
echo ""
echo "━━━ $ICON_SYNC Pruning Dangling Images — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling"
    PRUNED_SUMMARY="(dry run)"
else
    for _old_id in "${OLD_IMAGE_IDS[@]}"; do
        docker rmi "$_old_id" >/dev/null 2>&1 || true
    done
    PRUNED_OUTPUT=$(docker image prune -f 2>&1)
    [[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/  /'
    PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
fi

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINING) SUMMARY ━━━━━"
echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME       Duration:    $(format_duration $(( END - START )))"
echo "$ICON_CONTAINERS Scope:       ${#ALL_RUNNING[@]} running — ${#EXCLUDED[@]} managed = ${#REMAINING[@]} checked"
if [[ ${#UPDATED[@]} -gt 0 ]]; then
    echo "$ICON_DONE       New image:   ${#UPDATED[@]}"
    log "  ${UPDATED[*]}"
fi
[[ ${#UP_TO_DATE[@]} -gt 0 ]]    && log "$ICON_RUNNING    Up to date:  ${UP_TO_DATE[*]}"
[[ ${#FAILED[@]} -gt 0 ]]        && echo "$ICON_ERROR      Pull failed: ${FAILED[*]}"
if [[ ${#RESTARTED[@]} -gt 0 ]]; then
    echo "$ICON_DONE       Restarted:   ${#RESTARTED[@]}"
    log "  ${RESTARTED[*]}"
fi
[[ ${#SKIPPED_STOPPED[@]} -gt 0 ]] && log "$ICON_WARN     Not running: ${SKIPPED_STOPPED[*]} (skipped restart)"
[[ ${#RESTART_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR     Restart fail:${RESTART_FAILED[*]}"
echo "$ICON_SYNC       Pruned:      ${PRUNED_SUMMARY:-none}"

ALL_FAILED=$(( ${#FAILED[@]} + ${#RESTART_FAILED[@]} ))
if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no changes made"
elif [[ "$ALL_FAILED" -eq 0 ]]; then
    echo "$ICON_DONE Status: done ✅ — ${#RESTARTED[@]} restarted, ${#UP_TO_DATE[@]} current"
else
    warn "Status: $ALL_FAILED error(s) — ${#FAILED[@]} pull failure(s), ${#RESTART_FAILED[@]} restart failure(s)"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

[[ "$ALL_FAILED" -gt 0 ]] && exit 1
exit 0


#!/bin/bash
# ==============================================================================================
# ================================= Docker Update ==============================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Pulls the latest images for configured containers. Two modes: normal (daily)
# and remainder (weekly).
#
# Normal mode is called by daily_sync_maintenance.sh before docker_daily_restart.sh.
# Containers stay running during the pull — no extra downtime beyond what the
# nightly restart already causes.
#
# Remainder mode is called by weekly_sync_maintenance.sh as the final update step.
# It catches everything that normal mode and the weekly sync window did not already
# update — derived automatically from docker ps, nothing to configure.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Normal mode (daily):
#   Targets DAILY_RESTART_CONTAINERS — same list used by docker_daily_restart.sh.
#   Pull → compare old vs new image ID → mark updated or already current.
#   docker_daily_restart.sh runs after — containers restart onto the fresh image.
#
# Remainder mode (weekly):
#   Targets all currently running containers NOT in:
#     DAILY_RESTART_CONTAINERS       — already updated daily
#     emby + critical-data profiles  — updated inline by the weekly sync window
#     FALLBACK_*_TIER*               — owned by the remote server's update cycle
#   Pull → compare → prune dangling images.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Single List
#   Normal mode reuses DAILY_RESTART_CONTAINERS rather than maintaining a
#   separate update list. Adding or removing a container from the restart list
#   automatically updates the image pull list — one change, both places.
#
# Version Ownership
#   Fallback containers are excluded from remainder mode. This server only runs
#   them during a fallback. The remote server owns their version — if remainder
#   updates them independently and a handback occurs, the remote's older image
#   may not handle data written by the newer version.
#
# State Respect
#   Stopped containers are never targeted. Pulling while stopped adds no value
#   and a stopped container was likely halted intentionally.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Lock Acquisition
#   Prevents concurrent execution via acquire_lock(). Safe to call from
#   maintenance scripts without risk of overlap.
#
# Host Detection
#   detect_hosts() identifies which server is running the script and aliases
#   HOST*_DAILY_RESTART_CONTAINERS to the correct host's values.
#
# Root Enforcement
#   Docker operations require root privileges.
#
# DAILY_CONTAINER_UPDATES Toggle
#   Normal mode exits cleanly when disabled. docker_daily_restart.sh still runs
#   regardless — update and restart are independent operations.
#
# Fallback Exclusion
#   Remainder mode excludes containers owned by the remote server's update cycle
#   to prevent version divergence across the fallback boundary.
#
# Running-Only Filter
#   Stopped containers excluded from remainder mode — intentionally down.
#
# Image ID Comparison
#   Containers not restarted unless their image actually changed. Pulls that
#   result in "already up to date" produce no restart.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   DAILY_CONTAINER_UPDATES
#       Enable or disable normal mode. docker_daily_restart.sh runs regardless.
#       (default: true)
#
#   PROFILE_CRITICAL_CONTAINER_NAMES[emby|critical-data]
#       Container names for emby and critical-data profiles — excluded from
#       remainder mode (already updated by the weekly sync window)
#
# host*.conf
#
#   HOST*_DAILY_RESTART_CONTAINERS
#       Containers updated in normal mode. Aliased by detect_hosts() →
#       DAILY_RESTART_CONTAINERS
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# docker_update.sh
#     Normal mode — pull latest images for DAILY_RESTART_CONTAINERS
#
# docker_update.sh --remainder
#     Remainder mode — pull all running containers not in managed lists,
#     restart those that received updates, prune dangling images
#
# docker_update.sh --dry-run
#     Preview which containers would be pulled without making changes
#
# docker_update.sh --status
#     Show configuration and container list for current mode
#
# docker_update.sh --log
#     Verbose per-container pull and comparison output
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

# Pre-parse --remainder before parse_args (unknown args pass through to PARSED_ARGS)
REMAINDER_MODE=false
_filtered_args=()
for _arg in "$@"; do
    if [[ "$_arg" == "--remainder" ]]; then
        REMAINDER_MODE=true
    else
        _filtered_args+=("$_arg")
    fi
done
unset _arg

parse_args "${_filtered_args[@]}"
unset _filtered_args

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

if ! command -v docker &>/dev/null; then
    error "Docker command not found"
    exit 1
fi

detect_hosts

# ==============================================================================================
# ━━━ Container Discovery ━━━
# ==============================================================================================
if [[ "$REMAINDER_MODE" == true ]]; then
    declare -A _exclude=()

    # Daily containers — updated by docker_update.sh normal mode
    for _c in "${DAILY_RESTART_CONTAINERS[@]}"; do
        [[ -n "$_c" ]] && _exclude["$_c"]=1
    done

    # Weekly sync-window containers (emby + critical-data) — updated inline by weekly_sync_maintenance.sh
    _weekly_str="${PROFILE_CRITICAL_CONTAINER_NAMES[emby]:-} ${PROFILE_CRITICAL_CONTAINER_NAMES[critical-data]:-}"
    read -r -a _weekly_arr <<< "$_weekly_str"
    for _c in "${_weekly_arr[@]}"; do
        [[ -n "$_c" ]] && _exclude["$_c"]=1
    done
    unset _weekly_str _weekly_arr

    # Fallback coverage containers — owned by the remote server's update cycle.
    # This server runs them during fallback but should never update them independently.
    # Updating them here risks version divergence: if remote's writeback after handback
    # encounters data written by a newer version, it may not handle it correctly.
    for _tier in 1 2 3 4; do
        _tier_var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER${_tier}"
        eval "_tier_arr=(\"\${${_tier_var}[@]:-}\")" 2>/dev/null
        for _c in "${_tier_arr[@]}"; do
            [[ -n "$_c" ]] && _exclude["$_c"]=1
        done
    done
    unset _tier _tier_var _tier_arr _c

    mapfile -t _all_running < <(docker ps --format '{{.Names}}' | sort)
    TARGET_CONTAINERS=()
    for _c in "${_all_running[@]}"; do
        [[ -z "${_exclude[$_c]+x}" ]] && TARGET_CONTAINERS+=("$_c")
    done
    unset _all_running _exclude _c
else
    if [[ "${DAILY_CONTAINER_UPDATES:-true}" != "true" ]]; then
        echo "DAILY_CONTAINER_UPDATES=false — skipping container updates"
        exit 0
    fi

    if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
        warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to update"
        warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in host*.conf"
        exit 0
    fi

    TARGET_CONTAINERS=("${DAILY_RESTART_CONTAINERS[@]}")
fi

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST       Identity:   $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR       Mode:       $([[ "$REMAINDER_MODE" == true ]] && echo "remainder" || echo "normal (daily)")"
    if [[ "$REMAINDER_MODE" == true ]]; then
        echo "$ICON_CONTAINERS Containers: ${#TARGET_CONTAINERS[@]} running (excluding daily, weekly sync, and fallback)"
        for _c in "${TARGET_CONTAINERS[@]}"; do echo "  $_c"; done
    else
        echo "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
        echo "$ICON_GEAR       Enabled:    ${DAILY_CONTAINER_UPDATES:-true}"
    fi
    echo "$ICON_GEAR       Dry Run:    $DRY_RUN"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no images will be pulled"

if [[ ${#TARGET_CONTAINERS[@]} -eq 0 ]]; then
    echo "No containers to update"
    exit 0
fi

# ==============================================================================================
# ━━━ Pull Updates ━━━
# ==============================================================================================
echo ""
if [[ "$REMAINDER_MODE" == true ]]; then
    echo "━━━ $ICON_CONTAINERS Docker Update (remainder) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
    echo "$ICON_CONTAINERS Updating ${#TARGET_CONTAINERS[@]} container(s) (not in daily or weekly sync)"
    log "$ICON_CONTAINERS Remainder targets: ${TARGET_CONTAINERS[*]}"
else
    echo "━━━ $ICON_CONTAINERS Docker Update — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
    log "$ICON_CONTAINERS Containers: ${TARGET_CONTAINERS[*]}"
fi
echo ""

START=$(date +%s)
UPDATED=()
UP_TO_DATE=()
FAILED=()
OLD_IMAGE_IDS=()   # old image IDs to explicitly remove after rebuilds
SKIPPED=()

for container in "${TARGET_CONTAINERS[@]}"; do
    [[ -z "$container" ]] && continue
    log "━━━ $ICON_CONTAINERS $container ━━━"

    if ! docker inspect "$container" &>/dev/null; then
        warn "$container — not found, skipping"
        SKIPPED+=("$container")
        continue
    fi

    IMAGE=$(docker inspect --format='{{.Config.Image}}' "$container" 2>/dev/null)
    if [[ -z "$IMAGE" ]]; then
        warn "$container — could not determine image, skipping"
        SKIPPED+=("$container")
        continue
    fi

    log "$container — image: $IMAGE"

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would pull: $IMAGE"
        UPDATED+=("$container")
        continue
    fi

    # Capture the image ID the container is currently running on, and the
    # image ID :latest points to before the pull. After pulling, we rebuild if
    # either a new digest landed OR the container is behind what :latest is now.
    CONTAINER_IMAGE_ID=$(docker inspect "$container" --format='{{.Image}}' 2>/dev/null || echo "")
    OLD_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")

    log "$ICON_SYNC  Pulling $IMAGE..."
    if [[ "$ENABLE_LOGGING" == "true" ]]; then
        docker pull "$IMAGE" 2>&1 | grep -E "^(Status:|Digest:|Error|error)" | sed 's/^/  /'
        _pull_rc=${PIPESTATUS[0]}
    else
        docker pull "$IMAGE" >/dev/null 2>&1
        _pull_rc=$?
    fi
    NEW_ID=$(docker image inspect "$IMAGE" --format='{{.Id}}' 2>/dev/null || echo "")

    if [[ $_pull_rc -eq 0 ]]; then
        _pull_new=$(  [[ -n "$OLD_ID" && "$OLD_ID" != "$NEW_ID" ]] && echo true || echo false)
        _container_behind=$([[ -n "$CONTAINER_IMAGE_ID" && -n "$NEW_ID" && "$CONTAINER_IMAGE_ID" != "$NEW_ID" ]] && echo true || echo false)

        if [[ "$_pull_new" == true || "$_container_behind" == true ]]; then
            [[ "$_pull_new" == true ]]       && log "$ICON_DONE  $container — new image (${OLD_ID:7:12} → ${NEW_ID:7:12})"
            [[ "$_container_behind" == true && "$_pull_new" == false ]] && log "$ICON_DONE  $container — image already pulled, container behind (${CONTAINER_IMAGE_ID:7:12} → ${NEW_ID:7:12})"
            UPDATED+=("$container")
            OLD_IMAGE_IDS+=("$CONTAINER_IMAGE_ID")
        else
            log "$container — up to date (${NEW_ID:7:12})"
            UP_TO_DATE+=("$container")
        fi
    else
        warn "$container — pull failed ($IMAGE)"
        FAILED+=("$container")
    fi

done

# ── Recreate containers that received a new image ────────────────────────────
# docker restart uses the image ID baked in at creation time — it never picks
# up the new digest. rebuild_container reads the stored XML template, stops the
# old container, recreates it (new image, same config), then prunes the old image.
REBUILT=()
REBUILD_FAILED=()
if [[ ${#UPDATED[@]} -gt 0 ]]; then
    for container in "${UPDATED[@]}"; do
        [[ -z "$container" ]] && continue
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would rebuild $container from template"
            REBUILT+=("$container")
            continue
        fi
        log "$ICON_SYNC Rebuilding $container from template on new image..."
        if platform_rebuild_container "$container"; then
            log "$ICON_DONE $container rebuilt ✅"
            REBUILT+=("$container")
        else
            error "Failed to rebuild $container — will be picked up by docker_daily_restart.sh"
            notify "$container failed to rebuild after image update on $(hostname)" "Docker Update" "warning"
            REBUILD_FAILED+=("$container")
        fi
    done
fi

# ── Remove old images ────────────────────────────────────────────────────────
# Explicitly rmi by the IDs captured before each pull. Tagged images are never
# caught by dangling-only prune, so this is the only reliable cleanup path.
# Fall through to dangling prune to catch any leftovers from other update paths.
if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — would remove ${#OLD_IMAGE_IDS[@]} old image(s) and prune dangling"
    PRUNED_SUMMARY="(dry run)"
else
    for _old_id in "${OLD_IMAGE_IDS[@]}"; do
        docker rmi "$_old_id" >/dev/null 2>&1 || true
    done
    PRUNED_OUTPUT=$(docker image prune -f 2>&1)
    [[ "$ENABLE_LOGGING" == "true" ]] && echo "$PRUNED_OUTPUT" | sed 's/^/  /'
    PRUNED_SUMMARY=$(echo "$PRUNED_OUTPUT" | grep -E "^Total reclaimed" || echo "nothing reclaimed")
fi

END=$(date +%s)

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
if [[ "$REMAINDER_MODE" == true ]]; then
    echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE (REMAINDER) SUMMARY ━━━━━"
else
    echo "━━━━━ $ICON_SUMMARY DOCKER UPDATE SUMMARY ━━━━━"
fi
echo "$ICON_HOST       Identity:    $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME       Duration:    $(format_duration $(( END - START )))"
if [[ ${#UPDATED[@]} -gt 0 ]]; then
    echo "$ICON_DONE       Updated:     ${#UPDATED[@]}"
    log "  ${UPDATED[*]}"
fi
[[ ${#REBUILT[@]} -gt 0 ]]       && echo "$ICON_SYNC       Rebuilt:     ${#REBUILT[@]}"
[[ ${#REBUILD_FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR      Rebuild fail:${REBUILD_FAILED[*]}"
[[ ${#UP_TO_DATE[@]} -gt 0 ]]    && log "$ICON_RUNNING    Up to date:  ${UP_TO_DATE[*]}"
[[ ${#SKIPPED[@]} -gt 0 ]]       && log "$ICON_WARN       Skipped:     ${SKIPPED[*]}"
[[ ${#FAILED[@]} -gt 0 ]]        && echo "$ICON_ERROR      Failed:      ${FAILED[*]}"
echo "$ICON_SYNC       Pruned:      ${PRUNED_SUMMARY:-none}"

if [[ "$DRY_RUN" == true ]]; then
    warn "DRY RUN — no images pulled"
elif [[ ${#FAILED[@]} -eq 0 ]]; then
    echo "$ICON_DONE Status: done ✅ — ${#UPDATED[@]} updated, ${#UP_TO_DATE[@]} current"
else
    warn "Status: ${#FAILED[@]} pull(s) failed — restart will proceed with existing images"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

# Pull failures are non-fatal — restart proceeds regardless
exit 0


lets do it.... ive gotton close to the cor scipts being universal, then app layer and adpaters make everything work.. long term goal is running on other distros, and being able be cross comunicate tru…


Monitor media server sync and backup health


Now continuing with `partnership_manager.sh` — `check_both_healthy()`:


860	        return 0
861	    fi
862
863	    local del_code
864	    del_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \
865	        -X DELETE \
866	        -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
867	        "${emby_url}/Users/${user_id}" 2>/dev/null)
868
869	    if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then
870	        log "Emby admin '$username' removed ✅"
871	    else
872	        warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually"
873	    fi
874	}
875
876	check_both_healthy() {
877	    mountpoint -q /mnt/user 2>/dev/null || { error "Local array not healthy"; return 1; }
878
879	    local mirror_ip
880	    mirror_ip=$(resolve_tailscale_ip "$MIRROR")
881	    [[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }
882
883	    timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
884	        -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
885	        "mountpoint -q /mnt/user && timeout 10 docker ps" >/dev/null 2>&1 || {
886	        error "Mirror $MIRROR not healthy"
887	        return 1
888	    }
889	    return 0
890	}
891
892	do_final_sync() {
893	    log "Running final critical sync..."
894	    if [[ "$DRY_RUN" == false ]]; then
895	        if [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then
896	            for _share in "${CRITICAL_SYNC_SHARES[@]}"; do
897	                [[ -z "$_share" ]] && continue
898	                local _path="${_share%%|*}"
899	                local _profile="${_share##*|}"
900	                if [[ "$_path" == "$_profile" ]]; then
901	                    bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" --log
902	                else
903	                    bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" \
904	                        --profile="$_profile" --log
905	                fi
906	            done
907	        else
908	            bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
909	                "/mnt/user/appdata-Fallback/Critical-Data" \
910	                --profile=critical-fallback --log
911	            bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
912	                "/mnt/user/Media_Server/Emby" \
913	                --profile=emby-fallback --log
914	        fi
915	    else
916	        warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]:-hardcoded} shares)"
917	    fi
918	    warn "Final sync complete — mirror has current state ✅"
919	}

        return 0
    fi

    local del_code
    del_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \
        -X DELETE \
        -H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
        "${emby_url}/Users/${user_id}" 2>/dev/null)

    if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then
        log "Emby admin '$username' removed ✅"
    else
        warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually"
    fi
}

check_both_healthy() {
    mountpoint -q /mnt/user 2>/dev/null || { error "Local array not healthy"; return 1; }

    local mirror_ip
    mirror_ip=$(resolve_tailscale_ip "$MIRROR")
    [[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }

    timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
        "mountpoint -q /mnt/user && timeout 10 docker ps" >/dev/null 2>&1 || {
        error "Mirror $MIRROR not healthy"
        return 1
    }
    return 0
}

do_final_sync() {
    log "Running final critical sync..."
    if [[ "$DRY_RUN" == false ]]; then
        if [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then
            for _share in "${CRITICAL_SYNC_SHARES[@]}"; do
                [[ -z "$_share" ]] && continue
                local _path="${_share%%|*}"
                local _profile="${_share##*|}"
                if [[ "$_path" == "$_profile" ]]; then
                    bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" --log
                else
                    bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" \
                        --profile="$_profile" --log
                fi
            done
        else
            bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
                "/mnt/user/appdata-Fallback/Critical-Data" \
                --profile=critical-fallback --log
            bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
                "/mnt/user/Media_Server/Emby" \
                --profile=emby-fallback --log
        fi
    else
        warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]:-hardcoded} shares)"
    fi
    warn "Final sync complete — mirror has current state ✅"
}


Edit

check_both_healthy() {
    mountpoint -q /mnt/user 2>/dev/null || { error "Local array not healthy"; return 1; }

    local mirror_ip
    mirror_ip=$(resolve_tailscale_ip "$MIRROR")
    [[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }

    timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
        "mountpoint -q /mnt/user && timeout 10 docker ps" >/dev/null 2>&1 || {
        error "Mirror $MIRROR not healthy"
        return 1
    }
    return 0
}

check_both_healthy() {
    platform_storage_healthy || { error "Local array not healthy"; return 1; }

    local mirror_ip _spath
    mirror_ip=$(resolve_tailscale_ip "$MIRROR")
    [[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }
    _spath=$(platform_storage_path)

    timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
        "mountpoint -q '$_spath' && timeout 10 docker ps" >/dev/null 2>&1 || {
        error "Mirror $MIRROR not healthy"
        return 1
    }
    return 0
}


The file /boot/config/plugins/varaverk/Partnership/partnership_manager.sh has been updated successfully. (file state is current in your context — no need to Read it back)

check_both_healthy() {
    mountpoint -q /mnt/user 2>/dev/null || { error "Local array not healthy"; return 1; }

    local mirror_ip
    mirror_ip=$(resolve_tailscale_ip "$MIRROR")
    [[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }

    timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
        "mountpoint -q /mnt/user && timeout 10 docker ps" >/dev/null 2>&1 || {
        error "Mirror $MIRROR not healthy"
        return 1
    }
    return 0
}

check_both_healthy() {
    platform_storage_healthy || { error "Local array not healthy"; return 1; }

    local mirror_ip _spath
    mirror_ip=$(resolve_tailscale_ip "$MIRROR")
    [[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }
    _spath=$(platform_storage_path)

    timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
        "mountpoint -q '$_spath' && timeout 10 docker ps" >/dev/null 2>&1 || {
        error "Mirror $MIRROR not healthy"
        return 1
    }
    return 0
}

-    mountpoint -q /mnt/user 2>/dev/null || { error "Local array not healthy"; return 1; }

+    platform_storage_healthy || { error "Local array not healthy"; return 1; }

-        "mountpoint -q /mnt/user && timeout 10 docker ps" >/dev/null 2>&1 || {

+        "mountpoint -q '$_spath' && timeout 10 docker ps" >/dev/null 2>&1 || {


Now fix `do_final_sync()` — warn if `CRITICAL_SYNC_SHARES` not configured instead of using hardcoded paths:


        else
            bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
                "/mnt/user/appdata-Fallback/Critical-Data" \
                --profile=critical-fallback --log
            bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
                "/mnt/user/Media_Server/Emby" \
                --profile=emby-fallback --log
        fi

        else
            warn "CRITICAL_SYNC_SHARES is empty — skipping final sync (configure in host*.conf)"
        fi


        else
            bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
                "/mnt/user/appdata-Fallback/Critical-Data" \
                --profile=critical-fallback --log
            bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
                "/mnt/user/Media_Server/Emby" \
                --profile=emby-fallback --log
        fi

        else
            warn "CRITICAL_SYNC_SHARES is empty — skipping final sync (configure in host*.conf)"
        fi

-            bash "$SCRIPT_DIR/../Rsync/rsync.sh" \

-                "/mnt/user/appdata-Fallback/Critical-Data" \

-                --profile=critical-fallback --log

-                "/mnt/user/Media_Server/Emby" \

-                --profile=emby-fallback --log

+            warn "CRITICAL_SYNC_SHARES is empty — skipping final sync (configure in host*.conf)"

warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]:-hardcoded} shares)"


Now fix `partnership_onboard.sh:315-316` — remote SCRIPTS_DIR discovery:


308	    OWNER_IP=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true)
309	    PHASE2_TRIGGERED=false
310
311	    if [[ -n "$OWNER_IP" ]]; then
312	        # Read OWNER's SCRIPTS_DIR from their varaverk.cfg — don't assume same path as mirror
313	        OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
314	            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
315	            'grep SCRIPTS_DIR /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d "\"'"'"'" 2>/dev/null' 2>/dev/null | tr -d '[:space:]')
316	        OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-/boot/config/plugins/varaverk}"
317
318	        if [[ "$DRY_RUN" == true ]]; then
319	            warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"
320	            PHASE2_TRIGGERED=true
321	        elif timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
322	            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
323	            "nohup bash '${OWNER_SCRIPTS_DIR}/Partnership/partnership_onboard.sh' --phase2-only > /tmp/vv_phase2_onboard.log 2>&1 & echo triggered" \
324	            2>/dev/null | grep -q triggered; then
325	            log "Phase 2 triggered on $OWNER ✅"
326	            log "Watch progress on $OWNER: tail -f /tmp/vv_phase2_onboard.log"
327	            PHASE2_TRIGGERED=true

    OWNER_IP=$(resolve_tailscale_ip "$OWNER" 2>/dev/null || true)
    PHASE2_TRIGGERED=false

    if [[ -n "$OWNER_IP" ]]; then
        # Read OWNER's SCRIPTS_DIR from their varaverk.cfg — don't assume same path as mirror
        OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
            'grep SCRIPTS_DIR /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d "\"'"'"'" 2>/dev/null' 2>/dev/null | tr -d '[:space:]')
        OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-/boot/config/plugins/varaverk}"

        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"
            PHASE2_TRIGGERED=true
        elif timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
            "nohup bash '${OWNER_SCRIPTS_DIR}/Partnership/partnership_onboard.sh' --phase2-only > /tmp/vv_phase2_onboard.log 2>&1 & echo triggered" \
            2>/dev/null | grep -q triggered; then
            log "Phase 2 triggered on $OWNER ✅"
            log "Watch progress on $OWNER: tail -f /tmp/vv_phase2_onboard.log"
            PHASE2_TRIGGERED=true


        # Read OWNER's SCRIPTS_DIR from their varaverk.cfg — don't assume same path as mirror
        OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
            'grep SCRIPTS_DIR /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d "\"'"'"'" 2>/dev/null' 2>/dev/null | tr -d '[:space:]')
        OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-/boot/config/plugins/varaverk}"

        # Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror
        _probe_cmd=$(platform_scripts_dir_probe_cmd)
        OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
            "$_probe_cmd" 2>/dev/null | tr -d '[:space:]')
        OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-$SCRIPTS_DIR}"


The file /boot/config/plugins/varaverk/Partnership/partnership_onboard.sh has been updated successfully. (file state is current in your context — no need to Read it back)

        # Read OWNER's SCRIPTS_DIR from their varaverk.cfg — don't assume same path as mirror
        OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
            'grep SCRIPTS_DIR /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d "\"'"'"'" 2>/dev/null' 2>/dev/null | tr -d '[:space:]')
        OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-/boot/config/plugins/varaverk}"

        # Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror
        _probe_cmd=$(platform_scripts_dir_probe_cmd)
        OWNER_SCRIPTS_DIR=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$OWNER_IP" \
            "$_probe_cmd" 2>/dev/null | tr -d '[:space:]')
        OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-$SCRIPTS_DIR}"

-        # Read OWNER's SCRIPTS_DIR from their varaverk.cfg — don't assume same path as mirror

+        # Read OWNER's SCRIPTS_DIR via platform probe command — don't assume same path as mirror

+        _probe_cmd=$(platform_scripts_dir_probe_cmd)

-            'grep SCRIPTS_DIR /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d "\"'"'"'" 2>/dev/null' 2>/dev/null | tr -d '[:space:]')

-        OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-/boot/config/plugins/varaverk}"

+            "$_probe_cmd" 2>/dev/null | tr -d '[:space:]')

+        OWNER_SCRIPTS_DIR="${OWNER_SCRIPTS_DIR:-$SCRIPTS_DIR}"

warn "DRY RUN — would SSH to $OWNER ($OWNER_IP) and trigger Phase 2"


Now fix `partnership_transfer.sh:291-293`:


285	    update_master_conf "PARTNERSHIP_OWNER_HOST" "\"$NEW_OWNER_ID\""
286
287	    # Push updated master.conf to new owner so both servers agree immediately.
288	    # master.conf is shared — host-specific credentials live in host*.conf.
289	    _REMOTE_SD=$(ssh -i "$SSH_KEY" -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
290	        "root@${MIRROR_IP}" \
291	        "grep -m1 '^SCRIPTS_DIR' /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d '\"'" \
292	        2>/dev/null | tr -d '[:space:]')
293	    _REMOTE_SD="${_REMOTE_SD:-/boot/config/plugins/varaverk}"
294	    scp -i "$SSH_KEY" \
295	        -o ConnectTimeout="$SSH_TIMEOUT" \
296	        -o StrictHostKeyChecking=no \
297	        "$SCRIPTS_ROOT/Configurations/master.conf" \
298	        "root@${MIRROR_IP}:${_REMOTE_SD}/Configurations/master.conf" 2>/dev/null && \
299	        log "master.conf pushed to $NEW_OWNER ✅" || \
300	        error "Failed to push master.conf to $NEW_OWNER — set PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\" manually"
301	else
302	    warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers"
303	fi
304

    update_master_conf "PARTNERSHIP_OWNER_HOST" "\"$NEW_OWNER_ID\""

    # Push updated master.conf to new owner so both servers agree immediately.
    # master.conf is shared — host-specific credentials live in host*.conf.
    _REMOTE_SD=$(ssh -i "$SSH_KEY" -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
        "root@${MIRROR_IP}" \
        "grep -m1 '^SCRIPTS_DIR' /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d '\"'" \
        2>/dev/null | tr -d '[:space:]')
    _REMOTE_SD="${_REMOTE_SD:-/boot/config/plugins/varaverk}"
    scp -i "$SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" \
        -o StrictHostKeyChecking=no \
        "$SCRIPTS_ROOT/Configurations/master.conf" \
        "root@${MIRROR_IP}:${_REMOTE_SD}/Configurations/master.conf" 2>/dev/null && \
        log "master.conf pushed to $NEW_OWNER ✅" || \
        error "Failed to push master.conf to $NEW_OWNER — set PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\" manually"
else
    warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers"
fi


    _REMOTE_SD=$(ssh -i "$SSH_KEY" -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
        "root@${MIRROR_IP}" \
        "grep -m1 '^SCRIPTS_DIR' /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d '\"'" \
        2>/dev/null | tr -d '[:space:]')
    _REMOTE_SD="${_REMOTE_SD:-/boot/config/plugins/varaverk}"

    _probe_cmd=$(platform_scripts_dir_probe_cmd)
    _REMOTE_SD=$(ssh -i "$SSH_KEY" -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
        "root@${MIRROR_IP}" "$_probe_cmd" \
        2>/dev/null | tr -d '[:space:]')
    _REMOTE_SD="${_REMOTE_SD:-$SCRIPTS_DIR}"


The file /boot/config/plugins/varaverk/Partnership/partnership_transfer.sh has been updated successfully. (file state is current in your context — no need to Read it back)

    _REMOTE_SD=$(ssh -i "$SSH_KEY" -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
        "root@${MIRROR_IP}" \
        "grep -m1 '^SCRIPTS_DIR' /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d '\"'" \
        2>/dev/null | tr -d '[:space:]')
    _REMOTE_SD="${_REMOTE_SD:-/boot/config/plugins/varaverk}"

    _probe_cmd=$(platform_scripts_dir_probe_cmd)
    _REMOTE_SD=$(ssh -i "$SSH_KEY" -o ConnectTimeout=5 -o StrictHostKeyChecking=no \
        "root@${MIRROR_IP}" "$_probe_cmd" \
        2>/dev/null | tr -d '[:space:]')
    _REMOTE_SD="${_REMOTE_SD:-$SCRIPTS_DIR}"

# Push updated master.conf to new owner so both servers agree immediately.

# master.conf is shared — host-specific credentials live in host*.conf.

+    _probe_cmd=$(platform_scripts_dir_probe_cmd)

-        "grep -m1 '^SCRIPTS_DIR' /boot/config/plugins/varaverk/varaverk.cfg 2>/dev/null | cut -d= -f2 | tr -d '\"'" \

+        "root@${MIRROR_IP}" "$_probe_cmd" \

-    _REMOTE_SD="${_REMOTE_SD:-/boot/config/plugins/varaverk}"

+    _REMOTE_SD="${_REMOTE_SD:-$SCRIPTS_DIR}"


Now fix `onboard_cancel.sh:91`:


85	            warn "DRY RUN — would remove HOST1 key from $MIRROR:/root/.ssh/authorized_keys"
86	            H1_DONE=true
87	        else
88	            timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
89	                -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
90	                "sed -i \"|${KEY_BLOB}|d\" /root/.ssh/authorized_keys 2>/dev/null
91	                 sed -i \"/^${MIRROR_ID}_PHASE\|^${MIRROR_ID}_KEY_READY/d\" /boot/config/varaverk_setup.db 2>/dev/null
92	                 echo ok" 2>/dev/null | grep -q ok && {
93	                log "HOST1 key removed from $MIRROR authorized_keys ✅"
94	                H1_DONE=true
95	            } || warn "Could not SSH to $MIRROR — remove HOST1 key there manually"
96	        fi
97	    fi
98
99	    # Delete local key pair

            warn "DRY RUN — would remove HOST1 key from $MIRROR:/root/.ssh/authorized_keys"
            H1_DONE=true
        else
            timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
                -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
                "sed -i \"|${KEY_BLOB}|d\" /root/.ssh/authorized_keys 2>/dev/null
                 sed -i \"/^${MIRROR_ID}_PHASE\|^${MIRROR_ID}_KEY_READY/d\" /boot/config/varaverk_setup.db 2>/dev/null
                 echo ok" 2>/dev/null | grep -q ok && {
                log "HOST1 key removed from $MIRROR authorized_keys ✅"
                H1_DONE=true
            } || warn "Could not SSH to $MIRROR — remove HOST1 key there manually"
        fi
    fi

    # Delete local key pair


The `/boot/config/varaverk_setup.db` in the SSH command needs a platform adapter function. Let me add `platform_setup_db_path()` to the adapter first:


platform_get_templates_dir() {
    echo "/boot/config/plugins/dockerMan/templates-user"
}

platform_get_templates_dir() {
    echo "/boot/config/plugins/dockerMan/templates-user"
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_setup_db_path
#   Writes the path to the persistent Varaverk setup/wizard state database.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_setup_db_path() {
    echo "/boot/config/varaverk_setup.db"
}


The file /boot/config/plugins/varaverk/Plugin/unraid/adapter.sh has been updated successfully. (file state is current in your context — no need to Read it back)

platform_get_templates_dir() {
    echo "/boot/config/plugins/dockerMan/templates-user"
}

# ──────────────────────────────────────────────────────────────────────────────────────────────
# platform_setup_db_path
#   Writes the path to the persistent Varaverk setup/wizard state database.
# ──────────────────────────────────────────────────────────────────────────────────────────────
platform_setup_db_path() {
    echo "/boot/config/varaverk_setup.db"
}

+#   Writes the path to the persistent Varaverk setup/wizard state database.

+    echo "/boot/config/varaverk_setup.db"

#   Rebuilds a container from its stored Unraid CA XML template (stops old, recreates on new

#   image digest, prunes old image). Returns 1 if the rebuild script is absent or fails.


Now update the adapter header to document the new function:


#   platform_get_templates_dir    — path to Unraid CA docker templates-user directory
#   platform_rebuild_container    — rebuild a container from its stored XML template

#   platform_get_templates_dir    — path to Unraid CA docker templates-user directory
#   platform_setup_db_path        — path to the persistent Varaverk setup/wizard state database
#   platform_rebuild_container    — rebuild a container from its stored XML template


#   platform_get_templates_dir    — path to Unraid CA docker templates-user directory
#   platform_rebuild_container    — rebuild a container from its stored XML template

#   platform_get_templates_dir    — path to Unraid CA docker templates-user directory
#   platform_setup_db_path        — path to the persistent Varaverk setup/wizard state database
#   platform_rebuild_container    — rebuild a container from its stored XML template

#   platform_webui_install_path   — where the platform serves the WebGUI plugin files from

#   platform_scripts_dir_probe_cmd — shell command to run on a remote to discover its SCRIPTS_DIR

+#   platform_setup_db_path        — path to the persistent Varaverk setup/wizard state database

#   platform_push_conf            — push master.conf to all listed hosts via WebGUI PHP

#   platform_push_setup_state     — push wizard setup state to WebGUI PHP


1	#!/bin/bash
2	# ==============================================================================================
3	# ============================= Delete Keys ====================================================
4	# ==============================================================================================
5	#
6	# PURPOSE
7	# ─────────────────────────────────────────────────────────────────────────────
8	# Removes SSH keys between HOST1 and HOST2 in the specified direction.
9	# Safe to run at any phase. Clears related setup.db flags.
10	#
11	# ==============================================================================================
12	# RUNTIME MODES
13	# ==============================================================================================
14	#
15	# Partnership/onboard_cancel.sh --direction=h1   (default)
16	#     HOST1 → HOST2: remove HOST1's public key from HOST2's authorized_keys,
17	#     delete local HOST1 key pair, clear HOST2 phase flags from setup.db.
18	#
19	# Partnership/onboard_cancel.sh --direction=h2
20	#     HOST2 → HOST1: remove HOST2's public key from HOST1's authorized_keys.
21	#     Identifies the key by HOST2's hostname in the key comment.
22	#
23	# Partnership/onboard_cancel.sh --direction=both
24	#     Both directions.
25	#
26	# Partnership/onboard_cancel.sh --dry-run
27	#     Preview without making changes.
28	#
29	# ==============================================================================================
30

#!/bin/bash
# ==============================================================================================
# ============================= Delete Keys ====================================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Removes SSH keys between HOST1 and HOST2 in the specified direction.
# Safe to run at any phase. Clears related setup.db flags.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# Partnership/onboard_cancel.sh --direction=h1   (default)
#     HOST1 → HOST2: remove HOST1's public key from HOST2's authorized_keys,
#     delete local HOST1 key pair, clear HOST2 phase flags from setup.db.
#
# Partnership/onboard_cancel.sh --direction=h2
#     HOST2 → HOST1: remove HOST2's public key from HOST1's authorized_keys.
#     Identifies the key by HOST2's hostname in the key comment.
#
# Partnership/onboard_cancel.sh --direction=both
#     Both directions.
#
# Partnership/onboard_cancel.sh --dry-run
#     Preview without making changes.
#
# ==============================================================================================


30
31	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
32	SCRIPTS_ROOT="$SCRIPT_DIR/.."
33	SSH_TIMEOUT=15
34	DIRECTION="h1"
35	FILTERED_ARGS=()
36
37	for arg in "$@"; do
38	    case "$arg" in
39	        --direction=*) DIRECTION="${arg#--direction=}" ;;
40	        *)             FILTERED_ARGS+=("$arg") ;;
41	    esac
42	done
43
44	source "$SCRIPTS_ROOT/load_config.sh"
45	parse_args "${FILTERED_ARGS[@]}"
46
47	[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }
48
49	acquire_lock

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SCRIPTS_ROOT="$SCRIPT_DIR/.."
SSH_TIMEOUT=15
DIRECTION="h1"
FILTERED_ARGS=()

for arg in "$@"; do
    case "$arg" in
        --direction=*) DIRECTION="${arg#--direction=}" ;;
        *)             FILTERED_ARGS+=("$arg") ;;
    esac
done

source "$SCRIPTS_ROOT/load_config.sh"
parse_args "${FILTERED_ARGS[@]}"

[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }

acquire_lock


                "sed -i \"|${KEY_BLOB}|d\" /root/.ssh/authorized_keys 2>/dev/null
                 sed -i \"/^${MIRROR_ID}_PHASE\|^${MIRROR_ID}_KEY_READY/d\" /boot/config/varaverk_setup.db 2>/dev/null
                 echo ok" 2>/dev/null | grep -q ok && {

                "sed -i \"|${KEY_BLOB}|d\" /root/.ssh/authorized_keys 2>/dev/null
                 sed -i \"/^${MIRROR_ID}_PHASE\|^${MIRROR_ID}_KEY_READY/d\" $(platform_setup_db_path) 2>/dev/null
                 echo ok" 2>/dev/null | grep -q ok && {


The file /boot/config/plugins/varaverk/Partnership/onboard_cancel.sh has been updated successfully. (file state is current in your context — no need to Read it back)

                "sed -i \"|${KEY_BLOB}|d\" /root/.ssh/authorized_keys 2>/dev/null
                 sed -i \"/^${MIRROR_ID}_PHASE\|^${MIRROR_ID}_KEY_READY/d\" $(platform_setup_db_path) 2>/dev/null
                 echo ok" 2>/dev/null | grep -q ok && {

#!/bin/bash
# ==============================================================================================
# ============================= Delete Keys ====================================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Removes SSH keys between HOST1 and HOST2 in the specified direction.
# Safe to run at any phase. Clears related setup.db flags.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# Partnership/onboard_cancel.sh --direction=h1   (default)
#     HOST1 → HOST2: remove HOST1's public key from HOST2's authorized_keys,
#     delete local HOST1 key pair, clear HOST2 phase flags from setup.db.
#
# Partnership/onboard_cancel.sh --direction=h2
#     HOST2 → HOST1: remove HOST2's public key from HOST1's authorized_keys.
#     Identifies the key by HOST2's hostname in the key comment.
#
# Partnership/onboard_cancel.sh --direction=both
#     Both directions.
#
# Partnership/onboard_cancel.sh --dry-run
#     Preview without making changes.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SCRIPTS_ROOT="$SCRIPT_DIR/.."
SSH_TIMEOUT=15
DIRECTION="h1"
FILTERED_ARGS=()

for arg in "$@"; do
    case "$arg" in
        --direction=*) DIRECTION="${arg#--direction=}" ;;
        *)             FILTERED_ARGS+=("$arg") ;;
    esac
done

source "$SCRIPTS_ROOT/load_config.sh"
parse_args "${FILTERED_ARGS[@]}"

[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }

acquire_lock

detect_hosts

OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
MIRROR="${!MIRROR_ID}"
SSH_KEY_PUB="${SSH_KEY}.pub"
STATE_FILE="${VARAVERK_SETUP_FILE:-${STATE_DIR:-/boot/config}/varaverk_setup.db}"
AUTH_KEYS="/root/.ssh/authorized_keys"
MIRROR_SHORT="${MIRROR%%.*}"

START=$(date +%s)

echo ""
echo "━━━ Delete Keys — direction:${DIRECTION} — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
echo ""

H1_DONE=false
H2_DONE=false

# ── HOST1 → HOST2: remove HOST1's key from HOST2 + delete local pair ──────────
if [[ "$DIRECTION" == "h1" || "$DIRECTION" == "both" ]]; then
    echo "━━━ HOST1 → HOST2: Remove HOST1 key from $MIRROR ━━━"

    if [[ ! -f "$SSH_KEY_PUB" ]]; then
        log "No local public key — nothing to remove from $MIRROR"
        H1_DONE=true
    else
        KEY_BLOB=$(awk '{print $2}' "$SSH_KEY_PUB")
        MIRROR_IP=$(resolve_tailscale_ip "$MIRROR" 2>/dev/null || true)

        if [[ -z "$MIRROR_IP" ]]; then
            warn "Cannot resolve $MIRROR Tailscale IP — remove HOST1 key from $MIRROR manually"
        elif [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would remove HOST1 key from $MIRROR:/root/.ssh/authorized_keys"
            H1_DONE=true
        else
            timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
                -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$MIRROR_IP" \
                "sed -i \"|${KEY_BLOB}|d\" /root/.ssh/authorized_keys 2>/dev/null
                 sed -i \"/^${MIRROR_ID}_PHASE\|^${MIRROR_ID}_KEY_READY/d\" /boot/config/varaverk_setup.db 2>/dev/null
                 echo ok" 2>/dev/null | grep -q ok && {
                log "HOST1 key removed from $MIRROR authorized_keys ✅"
                H1_DONE=true
            } || warn "Could not SSH to $MIRROR — remove HOST1 key there manually"
        fi
    fi

    # Delete local key pair
    if [[ ! -f "$SSH_KEY" && ! -f "$SSH_KEY_PUB" ]]; then
        log "Local key already gone"
    elif [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would delete: $SSH_KEY and ${SSH_KEY}.pub"
    else
        rm -f "$SSH_KEY" "$SSH_KEY_PUB" && log "Local key pair deleted ✅" || \
            warn "Failed to delete local key — check permissions"
    fi

    # Clear HOST2 phase flags from local setup.db
    if [[ -f "$STATE_FILE" ]]; then
        if [[ "$DRY_RUN" == false ]]; then
            sed -i "/^${MIRROR_ID}_PHASE/d; /^${MIRROR_ID}_KEY_READY/d" "$STATE_FILE"
            log "Phase flags cleared from local setup.db ✅"
        else
            warn "DRY RUN — would clear ${MIRROR_ID}_PHASE* from setup.db"
        fi
    fi
    echo ""
fi

# ── HOST2 → HOST1: remove HOST2's key from HOST1's authorized_keys ────────────
if [[ "$DIRECTION" == "h2" || "$DIRECTION" == "both" ]]; then
    echo "━━━ HOST2 → HOST1: Remove $MIRROR key from HOST1 ━━━"

    if [[ ! -f "$AUTH_KEYS" ]]; then
        log "No authorized_keys on HOST1 — nothing to remove"
        H2_DONE=true
    elif ! grep -qi "$MIRROR_SHORT" "$AUTH_KEYS" 2>/dev/null; then
        log "$MIRROR key not found in HOST1 authorized_keys (already removed or never added)"
        H2_DONE=true
    elif [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would remove $MIRROR_SHORT key from $AUTH_KEYS"
        H2_DONE=true
    else
        sed -i "/${MIRROR_SHORT}/Id" "$AUTH_KEYS" && {
            log "$MIRROR key removed from HOST1 authorized_keys ✅"
            H2_DONE=true
        } || warn "Failed to remove $MIRROR key from HOST1 authorized_keys"
    fi
    echo ""
fi

# ── Summary ───────────────────────────────────────────────────────────────────
END=$(date +%s)
echo "━━━━━ DONE ━━━━━"
[[ "$DIRECTION" == "h1" || "$DIRECTION" == "both" ]] && \
    echo "  HOST1 → HOST2: $( [[ "$H1_DONE" == true ]] && echo "✅" || echo "⚠ manual step may be needed" )"
[[ "$DIRECTION" == "h2" || "$DIRECTION" == "both" ]] && \
    echo "  HOST2 → HOST1: $( [[ "$H2_DONE" == true ]] && echo "✅" || echo "⚠" )"
echo "  Duration: $(format_duration $(( END - START )))"
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0

-                 sed -i \"/^${MIRROR_ID}_PHASE\|^${MIRROR_ID}_KEY_READY/d\" /boot/config/varaverk_setup.db 2>/dev/null

+                 sed -i \"/^${MIRROR_ID}_PHASE\|^${MIRROR_ID}_KEY_READY/d\" $(platform_setup_db_path) 2>/dev/null


Now fix `conf_sync.sh:103`:


Error: File does not exist. Note: your current working directory is /root.


Bash

find /boot/config/plugins/varaverk -name "conf_sync.sh" 2>/dev/null

Find conf_sync.sh


95	    if [[ -z "$partner_ip" ]]; then
96	        warn "$partner_host — cannot resolve Tailscale IP, skipping"
97	        (( FAILED++ ))
98	        continue
99	    fi
100
101	    # ── Pull: grab partner's conf from their disk → our local cache ──────────
102	    if [[ "$PUSH_ONLY" == false ]]; then
103	        remote_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
104	        if [[ "$DRY_RUN" == true ]]; then
105	            warn "DRY RUN — would pull $partner_host:$remote_conf → $CACHE_DIR/${partner_slot}.conf"
106	        elif timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
107	            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
108	            "root@${partner_ip}:${remote_conf}" \
109	            "$CACHE_DIR/${partner_slot}.conf" 2>/dev/null; then
110	            log "Pulled ${partner_slot}.conf from $partner_host ✅"
111	            (( PULLED++ ))
112	        else
113	            warn "Could not pull ${partner_slot}.conf from $partner_host"
114	            (( FAILED++ ))

    if [[ -z "$partner_ip" ]]; then
        warn "$partner_host — cannot resolve Tailscale IP, skipping"
        (( FAILED++ ))
        continue
    fi

    # ── Pull: grab partner's conf from their disk → our local cache ──────────
    if [[ "$PUSH_ONLY" == false ]]; then
        remote_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would pull $partner_host:$remote_conf → $CACHE_DIR/${partner_slot}.conf"
        elif timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
            "root@${partner_ip}:${remote_conf}" \
            "$CACHE_DIR/${partner_slot}.conf" 2>/dev/null; then
            log "Pulled ${partner_slot}.conf from $partner_host ✅"
            (( PULLED++ ))
        else
            warn "Could not pull ${partner_slot}.conf from $partner_host"
            (( FAILED++ ))


Edit

    # ── Pull: grab partner's conf from their disk → our local cache ──────────
    if [[ "$PUSH_ONLY" == false ]]; then
        remote_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"

    # ── Pull: grab partner's conf from their disk → our local cache ──────────
    if [[ "$PUSH_ONLY" == false ]]; then
        remote_conf="${SCRIPTS_DIR}/Configurations/${partner_slot}.conf"


The file /boot/config/plugins/varaverk/System_Essentials/conf_sync.sh has been updated successfully. (file state is current in your context — no need to Read it back)

    # ── Pull: grab partner's conf from their disk → our local cache ──────────
    if [[ "$PUSH_ONLY" == false ]]; then
        remote_conf="${SCRIPTS_DIR}/Configurations/${partner_slot}.conf"

#!/bin/bash
# ==============================================================================================
# ============================= Conf Cache Sync ================================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Maintains a RAM-resident conf cache at /tmp/.vv/config/cached/.confs/.
# Credentials and partner keys live in RAM only — never on disk across hosts.
#
# On array start (default / --array-start):
#   1. Copy own conf to local cache
#   2. Pull each available partner's conf from their disk → local cache
#   3. Push own conf to each available partner's /tmp/.vv/ cache
#
# On conf save (--push-only):
#   Fast path — push updated own conf to all partners' /tmp/.vv/ cache only.
#   No pulls, no local cache rebuild.
#
# Cache is /tmp (tmpfs) — cleared every reboot, repopulated by this script
# on next array start. Scripts source from cache for partner vars; own vars
# always come from disk (load_config.sh skips cached copy of own conf).
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
#   conf_sync.sh                   Full sync: pull from all partners + push to all partners
#   conf_sync.sh --push-only       Push own conf to all partners (fast, for conf-save hook)
#   conf_sync.sh --pull-only       Pull partner confs into local cache only (for intermediate orch)
#   conf_sync.sh --dry-run         Show what would happen, no changes
#   conf_sync.sh --log             Verbose output
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"

PUSH_ONLY=false
PULL_ONLY=false
FILTERED_ARGS=()
for arg in "$@"; do
    case "$arg" in
        --push-only) PUSH_ONLY=true ;;
        --pull-only) PULL_ONLY=true ;;
        *)           FILTERED_ARGS+=("$arg") ;;
    esac
done
parse_args "${FILTERED_ARGS[@]}"
detect_hosts

if [[ "${CONF_SYNC_ENABLED:-true}" == false ]]; then
    log "CONF_SYNC_ENABLED=false — skipping"
    exit 0
fi

CACHE_DIR="/tmp/.vv/config/cached/.confs"
MY_CONF="$SCRIPTS_ROOT/Configurations/${MY_ID,,}.conf"
SSH_TIMEOUT=10

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# ── Ensure cache dir exists ───────────────────────────────────────────────────
if [[ "$DRY_RUN" == false ]]; then
    mkdir -p "$CACHE_DIR"
fi

# ── Copy own conf into local cache ───────────────────────────────────────────
if [[ "$PUSH_ONLY" == false ]] && [[ "$PULL_ONLY" == false ]]; then
    if [[ -f "$MY_CONF" ]]; then
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would copy $(basename "$MY_CONF") → $CACHE_DIR/"
        else
            cp "$MY_CONF" "$CACHE_DIR/${MY_ID,,}.conf" && \
                log "Own conf cached ✅" || warn "Failed to cache own conf"
        fi
    else
        warn "Own conf not found: $MY_CONF"
    fi
fi

# ── Per-partner sync ──────────────────────────────────────────────────────────
PUSHED=0
PULLED=0
FAILED=0

for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
    partner_host="${!host_var}"
    [[ -z "$partner_host" ]] && continue
    [[ "${host_var,,}" == "${MY_ID,,}" ]] && continue

    partner_slot="${host_var,,}"   # e.g. host2
    partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)

    if [[ -z "$partner_ip" ]]; then
        warn "$partner_host — cannot resolve Tailscale IP, skipping"
        (( FAILED++ ))
        continue
    fi

    # ── Pull: grab partner's conf from their disk → our local cache ──────────
    if [[ "$PUSH_ONLY" == false ]]; then
        remote_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would pull $partner_host:$remote_conf → $CACHE_DIR/${partner_slot}.conf"
        elif timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
            "root@${partner_ip}:${remote_conf}" \
            "$CACHE_DIR/${partner_slot}.conf" 2>/dev/null; then
            log "Pulled ${partner_slot}.conf from $partner_host ✅"
            (( PULLED++ ))
        else
            warn "Could not pull ${partner_slot}.conf from $partner_host"
            (( FAILED++ ))
        fi
    fi

    # ── Push: send own conf to partner's /tmp/.vv/ cache ────────────────────
    if [[ "$PULL_ONLY" == true ]]; then
        continue
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would push ${MY_ID,,}.conf → $partner_host:/tmp/.vv/config/cached/.confs/"
        continue
    fi

    # Ensure partner's cache dir exists, then SCP own conf into it
    timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
        "root@${partner_ip}" "mkdir -p '$CACHE_DIR'" 2>/dev/null

    if timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
        "$MY_CONF" \
        "root@${partner_ip}:${CACHE_DIR}/${MY_ID,,}.conf" 2>/dev/null; then
        log "Pushed ${MY_ID,,}.conf to $partner_host ✅"
        (( PUSHED++ ))
    else
        warn "Could not push to $partner_host"
        (( FAILED++ ))
    fi
done

# ── Summary ───────────────────────────────────────────────────────────────────
if [[ "$PUSH_ONLY" == true ]]; then
    info "Conf push complete — pushed to $PUSHED host(s)${FAILED:+, $FAILED failed}"
elif [[ "$PULL_ONLY" == true ]]; then
    info "Conf pull complete — pulled $PULLED partner conf(s)${FAILED:+, $FAILED failed}"
else
    info "Conf sync complete — pulled $PULLED, pushed $PUSHED${FAILED:+, $FAILED failed}"
fi

if [[ "$FAILED" -gt 0 ]]; then
    notify "Conf sync on $LOCAL_SERVER_NAME ($MY_ID) — $FAILED partner(s) failed. Partner config cache may be stale." \
        "Conf Sync" "warning"
    exit 1
fi

-        remote_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"

+        remote_conf="${SCRIPTS_DIR}/Configurations/${partner_slot}.conf"


Now fix `git_pull_execute.sh:258`:


Bash

find /boot/config/plugins/varaverk -name "git_pull_execute.sh" 2>/dev/null

Find git_pull_execute.sh


250	    echo "━━━ $ICON_GEAR Permissions ━━━"
251	    log "Setting executable permissions on all .sh files..."
252	    find "$TARGET_DIR" -type f -name "*.sh" -exec chmod +x {} \;
253	    echo "  Permissions set on .sh files"
254
255	    # ── Flash mode: sync Plugin/ to /boot/ so the webUI picks up updates ─────
256	    # In flash mode SCRIPTS_DIR is in appdata — Plugin/ lives in the repo there
257	    # but Unraid serves PHP from /boot/. Sync after every pull to keep them in step.
258	    _BOOT_DIR="/boot/config/plugins/varaverk"
259	    if [[ "$TARGET_DIR" != "$_BOOT_DIR" ]]; then
260	        echo ""
261	        echo "━━━ $ICON_SYNC Flash mode: sync Plugin/ → /boot/ ━━━"
262	        if rsync -a --delete "$TARGET_DIR/Plugin/" "$_BOOT_DIR/Plugin/" 2>/dev/null; then
263	            echo "  Plugin/ synced to /boot/ ✅"
264	        else
265	            warn "Plugin/ sync to /boot/ failed — webUI may be stale until next pull"
266	        fi
267	    fi
268	fi
269
270	END=$(date +%s)
271
272	# ==============================================================================================
273	# ━━━ Conf Upgrade ━━━
274	# ==============================================================================================

    echo "━━━ $ICON_GEAR Permissions ━━━"
    log "Setting executable permissions on all .sh files..."
    find "$TARGET_DIR" -type f -name "*.sh" -exec chmod +x {} \;
    echo "  Permissions set on .sh files"

    # ── Flash mode: sync Plugin/ to /boot/ so the webUI picks up updates ─────
    # In flash mode SCRIPTS_DIR is in appdata — Plugin/ lives in the repo there
    # but Unraid serves PHP from /boot/. Sync after every pull to keep them in step.
    _BOOT_DIR="/boot/config/plugins/varaverk"
    if [[ "$TARGET_DIR" != "$_BOOT_DIR" ]]; then
        echo ""
        echo "━━━ $ICON_SYNC Flash mode: sync Plugin/ → /boot/ ━━━"
        if rsync -a --delete "$TARGET_DIR/Plugin/" "$_BOOT_DIR/Plugin/" 2>/dev/null; then
            echo "  Plugin/ synced to /boot/ ✅"
        else
            warn "Plugin/ sync to /boot/ failed — webUI may be stale until next pull"
        fi
    fi
fi

END=$(date +%s)

# ==============================================================================================
# ━━━ Conf Upgrade ━━━
# ==============================================================================================


Edit

    _BOOT_DIR="/boot/config/plugins/varaverk"
    if [[ "$TARGET_DIR" != "$_BOOT_DIR" ]]; then

    _BOOT_DIR=$(platform_webui_install_path)
    if [[ "$TARGET_DIR" != "$_BOOT_DIR" ]]; then


The file /boot/config/plugins/varaverk/git_pull_execute.sh has been updated successfully. (file state is current in your context — no need to Read it back)

    _BOOT_DIR="/boot/config/plugins/varaverk"
    if [[ "$TARGET_DIR" != "$_BOOT_DIR" ]]; then

    _BOOT_DIR=$(platform_webui_install_path)
    if [[ "$TARGET_DIR" != "$_BOOT_DIR" ]]; then

# ── Flash mode: sync Plugin/ to /boot/ so the webUI picks up updates ─────

# In flash mode SCRIPTS_DIR is in appdata — Plugin/ lives in the repo there

# but Unraid serves PHP from /boot/. Sync after every pull to keep them in step.

-    _BOOT_DIR="/boot/config/plugins/varaverk"

+    _BOOT_DIR=$(platform_webui_install_path)

echo "━━━ $ICON_SYNC Flash mode: sync Plugin/ → /boot/ ━━━"


Now the last item — `watchdog_orchestrator.sh:128`:


120
121	# ==============================================================================================
122	# ━━━ Array Check ━━━
123	# ==============================================================================================
124	if ! platform_storage_healthy; then
125	    echo "Array not started — skipping watchdog cycle"
126	    exit 0
127	fi
128	log "$ICON_DISK Array: /mnt/user mounted (shfs) ✅"
129
130	# ==============================================================================================
131	# ━━━ Startup Grace ━━━
132	# ==============================================================================================
133	UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
134	if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
135	    echo "Startup grace — $(format_duration $UPTIME_SECONDS) / $(format_duration $WATCHDOG_STARTUP_GRACE) — skipping cycle"
136	    exit 0
137	fi
138	log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)"
139

# ==============================================================================================
# ━━━ Array Check ━━━
# ==============================================================================================
if ! platform_storage_healthy; then
    echo "Array not started — skipping watchdog cycle"
    exit 0
fi
log "$ICON_DISK Array: /mnt/user mounted (shfs) ✅"

# ==============================================================================================
# ━━━ Startup Grace ━━━
# ==============================================================================================
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
    echo "Startup grace — $(format_duration $UPTIME_SECONDS) / $(format_duration $WATCHDOG_STARTUP_GRACE) — skipping cycle"
    exit 0
fi
log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)"


log "$ICON_DISK Array: /mnt/user mounted (shfs) ✅"

log "$ICON_DISK Array: $(platform_storage_path) mounted ✅"


The file /boot/config/plugins/varaverk/Orchestrators/watchdog_orchestrator.sh has been updated successfully. (file state is current in your context — no need to Read it back)

echo "Array not started — skipping watchdog cycle"

-log "$ICON_DISK Array: /mnt/user mounted (shfs) ✅"

+log "$ICON_DISK Array: $(platform_storage_path) mounted ✅"


All pending tasks from the list are now done. Let me do a final sweep to make sure no stray `/mnt/user` or `/boot/config/plugins/varaverk` hardcodes remain in core bash scripts (excluding adapter.sh and gitea_ssh_setup.sh):


Bash

grep -rn '"/mnt/user' /boot/config/plugins/varaverk \
    --include="*.sh" \
    --exclude="gitea_ssh_setup.sh" \
    --exclude="adapter.sh" \
    | grep -v "Plugin/" \
    | grep -v "^Binary"

Find remaining /mnt/user hardcodes in core bash scripts


/boot/config/plugins/varaverk/common.sh:821:# Usage: check_remote_share "/mnt/user/Movies"
/boot/config/plugins/varaverk/common.sh:968:# Usage: check_remote_disks "/mnt/user/Movies"
/boot/config/plugins/varaverk/common.sh:1539:#   declare -A ARR_PATH_MAP=(["/ext-music"]="/mnt/user/Music-New")
/boot/config/plugins/varaverk/Tools/claude_startup.sh:69:    PERSIST_DIR="/mnt/user/appdata/claude-code"


grep -rn "'/mnt/user" /boot/config/plugins/varaverk \
    --include="*.sh" \
    --exclude="gitea_ssh_setup.sh" \
    --exclude="adapter.sh" \
    | grep -v "Plugin/" \
    | grep -v "^Binary"

Find remaining /mnt/user hardcodes in single-quoted strings


(Bash completed with no output)


grep -rn '/boot/config/plugins/varaverk' /boot/config/plugins/varaverk \
    --include="*.sh" \
    --exclude="gitea_ssh_setup.sh" \
    --exclude="adapter.sh" \
    --exclude="claude_startup.sh" \
    | grep -v "Plugin/" \
    | grep -v "^Binary" \
    | grep -v "# "

Find remaining hardcoded varaverk paths in core bash scripts


/boot/config/plugins/varaverk/Deployment/deploy.sh:17:PROD_ROOT="/boot/config/plugins/varaverk"


10	#   Reads the tool JSON from stdin, deploys only the component that was actually changed.
11	#   Plugin/  → rsync --delete into prod Plugin/unraid/
12	#   Configurations/ → conf_upgrade.sh (adds new keys, preserves prod values)
13	#
14	# ==============================================================================================
15
16	DEV_ROOT="/mnt/cloud-storage/Important Shit/Git/Development/Varaverk"
17	PROD_ROOT="/boot/config/plugins/varaverk"
18	UPGRADE="$PROD_ROOT/Deployment/conf_upgrade.sh"
19	LOG="/tmp/vv_deploy.log"
20
21	# ── Deploy functions ──────────────────────────────────────────────────────────
22
23	deploy_plugin() {
24	    rsync -a --delete --exclude='.git' \
25	        "$DEV_ROOT/Plugin/unraid/" \
26	        "$PROD_ROOT/Plugin/unraid/"
27	}
28
29	deploy_conf() {
30	    for conf in master.conf host1.conf host2.conf; do
31	        tmpl="$DEV_ROOT/Configurations/$conf"
32	        target="$PROD_ROOT/Configurations/$conf"
33	        [[ -f "$tmpl" && -f "$target" ]] && \
34	            "$UPGRADE" --template "$tmpl" --target "$target"

#   Reads the tool JSON from stdin, deploys only the component that was actually changed.
#   Plugin/  → rsync --delete into prod Plugin/unraid/
#   Configurations/ → conf_upgrade.sh (adds new keys, preserves prod values)
#
# ==============================================================================================

DEV_ROOT="/mnt/cloud-storage/Important Shit/Git/Development/Varaverk"
PROD_ROOT="/boot/config/plugins/varaverk"
UPGRADE="$PROD_ROOT/Deployment/conf_upgrade.sh"
LOG="/tmp/vv_deploy.log"

# ── Deploy functions ──────────────────────────────────────────────────────────

deploy_plugin() {
    rsync -a --delete --exclude='.git' \
        "$DEV_ROOT/Plugin/unraid/" \
        "$PROD_ROOT/Plugin/unraid/"
}

deploy_conf() {
    for conf in master.conf host1.conf host2.conf; do
        tmpl="$DEV_ROOT/Configurations/$conf"
        target="$PROD_ROOT/Configurations/$conf"
        [[ -f "$tmpl" && -f "$target" ]] && \
            "$UPGRADE" --template "$tmpl" --target "$target"