Varaverk/System_Essentials/conf_sync.sh

#!/bin/bash
# ==============================================================================================
# ============================= Conf Cache Sync ================================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Maintains a RAM-resident conf cache at /tmp/.vv/config/cached/.confs/.
# Credentials and partner keys live in RAM only — never on disk across hosts.
#
# On array start (default / --array-start):
#   1. Copy own conf to local cache
#   2. Pull each available partner's conf from their disk → local cache
#   3. Push own conf to each available partner's /tmp/.vv/ cache
#
# On conf save (--push-only):
#   Fast path — push updated own conf to all partners' /tmp/.vv/ cache only.
#   No pulls, no local cache rebuild.
#
# Cache is /tmp (tmpfs) — cleared every reboot, repopulated by this script
# on next array start. Scripts source from cache for partner vars; own vars
# always come from disk (load_config.sh skips cached copy of own conf).
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
#   conf_sync.sh                   Full sync: pull from all partners + push to all partners
#   conf_sync.sh --push-only       Push own conf to all partners (fast, for conf-save hook)
#   conf_sync.sh --pull-only       Pull partner confs into local cache only (for intermediate orch)
#   conf_sync.sh --dry-run         Show what would happen, no changes
#   conf_sync.sh --log             Verbose output
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh"

PUSH_ONLY=false
PULL_ONLY=false
FILTERED_ARGS=()
for arg in "$@"; do
    case "$arg" in
        --push-only) PUSH_ONLY=true ;;
        --pull-only) PULL_ONLY=true ;;
        *)           FILTERED_ARGS+=("$arg") ;;
    esac
done
parse_args "${FILTERED_ARGS[@]}"
detect_hosts

if [[ "${CONF_SYNC_ENABLED:-true}" == false ]]; then
    log "CONF_SYNC_ENABLED=false — skipping"
    exit 0
fi

CACHE_DIR="/tmp/.vv/config/cached/.confs"
MY_CONF="$SCRIPTS_ROOT/Configurations/${MY_ID,,}.conf"
SSH_TIMEOUT=10

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"

# ── Ensure cache dir exists ───────────────────────────────────────────────────
if [[ "$DRY_RUN" == false ]]; then
    mkdir -p "$CACHE_DIR"
fi

# ── Copy own conf into local cache ───────────────────────────────────────────
if [[ "$PUSH_ONLY" == false ]] && [[ "$PULL_ONLY" == false ]]; then
    if [[ -f "$MY_CONF" ]]; then
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would copy $(basename "$MY_CONF") → $CACHE_DIR/"
        else
            cp "$MY_CONF" "$CACHE_DIR/${MY_ID,,}.conf" && \
                log "Own conf cached ✅" || warn "Failed to cache own conf"
        fi
    else
        warn "Own conf not found: $MY_CONF"
    fi
fi

# ── Per-partner sync ──────────────────────────────────────────────────────────
PUSHED=0
PULLED=0
FAILED=0

for host_var in $(compgen -v | grep -E '^HOST[0-9]+$' | sort); do
    partner_host="${!host_var}"
    [[ -z "$partner_host" ]] && continue
    [[ "${host_var,,}" == "${MY_ID,,}" ]] && continue

    partner_slot="${host_var,,}"   # e.g. host2
    partner_ip=$(resolve_tailscale_ip "$partner_host" 2>/dev/null || true)

    if [[ -z "$partner_ip" ]]; then
        warn "$partner_host — cannot resolve Tailscale IP, skipping"
        (( FAILED++ ))
        continue
    fi

    # ── Pull: grab partner's conf from their disk → our local cache ──────────
    if [[ "$PUSH_ONLY" == false ]]; then
        remote_conf="/boot/config/plugins/varaverk/Configurations/${partner_slot}.conf"
        if [[ "$DRY_RUN" == true ]]; then
            warn "DRY RUN — would pull $partner_host:$remote_conf → $CACHE_DIR/${partner_slot}.conf"
        elif timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
            -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
            "root@${partner_ip}:${remote_conf}" \
            "$CACHE_DIR/${partner_slot}.conf" 2>/dev/null; then
            log "Pulled ${partner_slot}.conf from $partner_host ✅"
            (( PULLED++ ))
        else
            warn "Could not pull ${partner_slot}.conf from $partner_host"
            (( FAILED++ ))
        fi
    fi

    # ── Push: send own conf to partner's /tmp/.vv/ cache ────────────────────
    if [[ "$PULL_ONLY" == true ]]; then
        continue
    fi

    if [[ "$DRY_RUN" == true ]]; then
        warn "DRY RUN — would push ${MY_ID,,}.conf → $partner_host:/tmp/.vv/config/cached/.confs/"
        continue
    fi

    # Ensure partner's cache dir exists, then SCP own conf into it
    timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
        "root@${partner_ip}" "mkdir -p '$CACHE_DIR'" 2>/dev/null

    if timeout "$SSH_TIMEOUT" scp -i "$SSH_KEY" \
        -o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes -o StrictHostKeyChecking=no \
        "$MY_CONF" \
        "root@${partner_ip}:${CACHE_DIR}/${MY_ID,,}.conf" 2>/dev/null; then
        log "Pushed ${MY_ID,,}.conf to $partner_host ✅"
        (( PUSHED++ ))
    else
        warn "Could not push to $partner_host"
        (( FAILED++ ))
    fi
done

# ── Summary ───────────────────────────────────────────────────────────────────
if [[ "$PUSH_ONLY" == true ]]; then
    info "Conf push complete — pushed to $PUSHED host(s)${FAILED:+, $FAILED failed}"
elif [[ "$PULL_ONLY" == true ]]; then
    info "Conf pull complete — pulled $PULLED partner conf(s)${FAILED:+, $FAILED failed}"
else
    info "Conf sync complete — pulled $PULLED, pushed $PUSHED${FAILED:+, $FAILED failed}"
fi

if [[ "$FAILED" -gt 0 ]]; then
    notify "Conf sync on $LOCAL_SERVER_NAME ($MY_ID) — $FAILED partner(s) failed. Partner config cache may be stale." \
        "Conf Sync" "warning"
    exit 1
fi