Gmer4Lfe
0621f66889
Curated state copied to every node is state that can disagree, so the index, the model and the shared memory stay on the owner and each node reaches them over the SSH trust onboarding already builds. Chats stay on the node that had them; memory and bug reports stay the owner's to write.
461 lines
21 KiB
Bash
Executable File
461 lines
21 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= Git Pull & Execute =========================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Pulls the latest scripts from the Gitea repository via SSH. Lives at the repo
|
|
# root — sources load_config.sh from the same directory.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# 1. detect_hosts() — identifies which server is running the pull (MY_ID)
|
|
# 2. Configure sparse checkout — exclude other servers' credential files
|
|
# 3. Pull or clone latest scripts from Gitea
|
|
# 4. Set executable permissions on all .sh files
|
|
#
|
|
# SPARSE CHECKOUT
|
|
# Each server only receives its own host conf — never peer credentials:
|
|
# HOST1 pulls: master.conf + host1.conf + all scripts
|
|
# HOST1 skips: host2.conf, host3.conf etc.
|
|
# Adding a new server: create host3.conf in the repo — all existing servers
|
|
# automatically exclude it on next pull; new server gets only its own conf.
|
|
#
|
|
# GITEA LOCATION DETECTION
|
|
# Detected at runtime — works through fallback:
|
|
# Gitea local → connects via local IP
|
|
# Gitea remote → connects via Tailscale IP
|
|
# Both fail → falls back to GITEA_DOMAIN if configured
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Credentials Stay Separated
|
|
# Sparse checkout is configured per-server on every pull, not just at clone
|
|
# time. Ensures newly added host confs are automatically excluded on all
|
|
# existing servers without any manual intervention.
|
|
#
|
|
# Runtime Location Detection
|
|
# Gitea's IP is never hardcoded — the script probes whether Gitea is local
|
|
# or remote on every run. Handles Gitea container restarts, migrations, and
|
|
# Tailscale address changes automatically.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root check — git operations on /boot require root
|
|
# acquire_lock — prevents concurrent pulls
|
|
# detect_hosts() — MY_ID required to build correct sparse checkout rules
|
|
# Docker check — Gitea container status probed before any SSH attempt
|
|
# Self re-exec — a pull that replaces this file restarts the run once, so the steps after
|
|
# the sync are the pulled ones rather than the ones already in memory
|
|
#
|
|
# The re-exec is the non-obvious one. This script lives in the repo it pulls, and git installs
|
|
# an updated file by rename — the running bash holds the old inode and finishes the run on the
|
|
# old logic. Nothing looks wrong; the change simply takes effect one run late. Bounded by
|
|
# VV_PULL_REEXECED so it can restart at most once, skipped entirely when the checksum is
|
|
# unchanged, and the lock is released by hand first because exec does not fire EXIT traps.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# GITEA_CONTAINER — Docker container name for Gitea
|
|
# GITEA_REPO_PATH — repo path on Gitea (e.g. Varaverk/varaverk.git)
|
|
# GITEA_DOMAIN — public domain fallback (optional)
|
|
# TARGET_DIR — local path to clone/pull into
|
|
# GITEA_SSH_KEY — SSH key for Gitea authentication
|
|
# SSH_PORT — Gitea SSH port (often 221 or 222)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# git_pull_execute.sh
|
|
# Normal pull.
|
|
#
|
|
# git_pull_execute.sh --dry-run
|
|
# Preview without making changes.
|
|
#
|
|
# git_pull_execute.sh --log
|
|
# Verbose output.
|
|
#
|
|
# git_pull_execute.sh --status
|
|
# Show config and exit.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
# This script lives inside the repo it pulls, so a pull can replace it mid-run. Recorded before
|
|
# anything happens; compared again once the pull lands. See the re-exec block after the sync.
|
|
_VV_SELF="$SCRIPT_DIR/$(basename "${BASH_SOURCE[0]}")"
|
|
_VV_SELF_ARGS=("$@")
|
|
_VV_SELF_SUM="$(md5sum "$_VV_SELF" 2>/dev/null | cut -d' ' -f1)"
|
|
|
|
# Root level script — load_config.sh is in the same directory
|
|
source "$SCRIPT_DIR/load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
# detect_hosts() sets MY_ID — needed for sparse checkout configuration
|
|
detect_hosts
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Locate Gitea ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_CONTAINERS Locate Gitea ━━━"
|
|
|
|
if docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^${GITEA_CONTAINER}$"; then
|
|
# Gitea is running on this server — use local IP
|
|
GITEA_IP=$(hostname -I | awk '{print $1}')
|
|
log "Gitea running locally — connecting via $GITEA_IP"
|
|
else
|
|
# Gitea not running locally — find it on the remote server via Tailscale
|
|
log "Gitea not running locally — checking remote server"
|
|
GITEA_IP=$(resolve_tailscale_ip "${REMOTE_SERVER_NAME}")
|
|
if [[ -n "$GITEA_IP" ]]; then
|
|
echo " Gitea on $REMOTE_SERVER_NAME — connecting via Tailscale $GITEA_IP"
|
|
elif [[ -n "${GITEA_DOMAIN:-}" ]]; then
|
|
warn "Tailscale resolution failed — falling back to $GITEA_DOMAIN"
|
|
GITEA_IP="$GITEA_DOMAIN"
|
|
else
|
|
error "Cannot find Gitea — local: not running, Tailscale: failed, domain: not configured"
|
|
notify "Git pull failed on $(hostname) — cannot locate Gitea container" "Git Sync" "alert"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
REPO_SSH="git@${GITEA_IP}:${GITEA_REPO_PATH}"
|
|
|
|
require_var REPO_SSH
|
|
require_var TARGET_DIR
|
|
require_var GITEA_SSH_KEY
|
|
require_var SSH_PORT
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_NET Repo: $REPO_SSH"
|
|
echo "$ICON_GEAR Target: $TARGET_DIR"
|
|
echo "$ICON_GEAR SSH Key: $GITEA_SSH_KEY"
|
|
echo "$ICON_GEAR SSH Port: $SSH_PORT"
|
|
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
|
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Sparse Checkout Configuration ━━━
|
|
# ==============================================================================================
|
|
# Build the list of host*.conf files that belong to OTHER servers.
|
|
# This server pulls everything EXCEPT those files.
|
|
# MY_ID is set by detect_hosts() — e.g. "HOST1"
|
|
|
|
configure_sparse_checkout() {
|
|
local repo_dir="$1"
|
|
|
|
log "Configuring sparse checkout for $MY_ID..."
|
|
|
|
# Enable sparse checkout
|
|
git -C "$repo_dir" config core.sparseCheckout true 2>/dev/null
|
|
|
|
# Build exclusion list — all host*.conf files except MY_ID's
|
|
local sparse_file="$repo_dir/.git/info/sparse-checkout"
|
|
mkdir -p "$(dirname "$sparse_file")"
|
|
|
|
# Start with: pull everything
|
|
echo "/*" > "$sparse_file"
|
|
|
|
# Exclude each other server's conf file
|
|
# Find all host*.conf files present in the repo
|
|
local excluded=0
|
|
for conf_file in "$repo_dir"/host*.conf; do
|
|
[[ -f "$conf_file" ]] || continue
|
|
local conf_name
|
|
conf_name=$(basename "$conf_file")
|
|
|
|
# Determine which HOST ID owns this conf by grepping its hostname var
|
|
# Pattern: HOST1="unRAID-..." or HOST2="unRAID-..."
|
|
local conf_host_id
|
|
conf_host_id=$(grep -m1 -oP '^\s+HOST[0-9]+(?==)' "$conf_file" 2>/dev/null | tr -d ' ')
|
|
|
|
if [[ -z "$conf_host_id" ]]; then
|
|
log "Cannot determine HOST ID for $conf_name — including in pull (safe default)"
|
|
continue
|
|
fi
|
|
|
|
if [[ "$conf_host_id" != "$MY_ID" ]]; then
|
|
echo "!$conf_name" >> "$sparse_file"
|
|
log "Sparse checkout: excluding $conf_name (belongs to $conf_host_id)"
|
|
((excluded++))
|
|
else
|
|
log "Sparse checkout: including $conf_name (belongs to $MY_ID — this server)"
|
|
fi
|
|
done
|
|
|
|
if [[ "$excluded" -gt 0 ]]; then
|
|
echo " Sparse checkout: excluding $excluded peer conf file(s) — credentials protected"
|
|
else
|
|
log "Sparse checkout: no peer conf files to exclude (single server or first run)"
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Git Sync ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Git Sync ━━━"
|
|
echo "$ICON_NET Repo: $REPO_SSH"
|
|
echo "$ICON_GEAR Target: $TARGET_DIR"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
SYNC_SUCCESS=false
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would sync $REPO_SSH → $TARGET_DIR"
|
|
warn "DRY RUN — would configure sparse checkout for $MY_ID"
|
|
warn "DRY RUN — would exclude peer host*.conf files"
|
|
SYNC_SUCCESS=true
|
|
else
|
|
mkdir -p "$TARGET_DIR"
|
|
git config --global --add safe.directory "$TARGET_DIR"
|
|
cd "$TARGET_DIR" || { error "Cannot cd into $TARGET_DIR"; exit 1; }
|
|
|
|
if [[ -d ".git" ]]; then
|
|
# ── Existing repository ──────────────────────────────────────────────
|
|
echo " Existing repository — updating"
|
|
|
|
# Configure sparse checkout BEFORE pull
|
|
# Uses conf files already present from last pull to determine exclusions
|
|
configure_sparse_checkout "$TARGET_DIR"
|
|
|
|
echo " Pulling latest changes..."
|
|
if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git pull --ff-only; then
|
|
echo " Git pull successful"
|
|
SYNC_SUCCESS=true
|
|
else
|
|
# ff-only fails when local commits or tracked changes exist that can't
|
|
# fast-forward. Fail loudly — never silently destroy local work.
|
|
error "Git pull failed — local changes conflict with remote (will not force-reset)"
|
|
notify "Git pull failed on $(hostname) — local changes conflict, manual resolve needed" "Git Sync" "alert"
|
|
exit 1
|
|
fi
|
|
|
|
else
|
|
# ── Fresh clone ──────────────────────────────────────────────────────
|
|
echo " No repository found — cloning"
|
|
|
|
# Clone first — need the repo to exist before configuring sparse checkout
|
|
if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git clone "$REPO_SSH" .; then
|
|
echo " Clone successful"
|
|
|
|
# Configure sparse checkout after clone
|
|
# Now all host*.conf files are present — can detect exclusions
|
|
configure_sparse_checkout "$TARGET_DIR"
|
|
|
|
# Apply sparse checkout — removes excluded files from working tree
|
|
echo " Applying sparse checkout..."
|
|
git read-tree -mu HEAD
|
|
echo " Sparse checkout applied — peer credentials removed from working tree"
|
|
|
|
SYNC_SUCCESS=true
|
|
else
|
|
error "Clone failed"
|
|
notify "Git clone failed on $(hostname) — check Gitea connectivity" "Git Sync" "alert"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# ── Re-exec if the pull replaced this script ─────────────────────────────
|
|
#
|
|
# git installs an updated file by rename, so the running bash keeps reading the inode it
|
|
# started with. That is the benign half — no torn parse, no garbage — but it means every
|
|
# step below this line runs the version that was on disk when the run began, not the one
|
|
# just pulled. Observed 2026-08-08: a pull carrying a fix to the conf-upgrade call executed
|
|
# the old call anyway, and the fix only took effect on the following run.
|
|
#
|
|
# Re-exec makes the pull self-applying. Guarded three ways:
|
|
# VV_PULL_REEXECED — set before exec, so the new image can never re-exec again. One
|
|
# restart per run, whatever the checksums say.
|
|
# checksum compare — no change means no restart, so the common case costs one md5sum.
|
|
# lock released — exec keeps the PID but does NOT fire the EXIT trap, so the lock file
|
|
# would survive into the new image, which would then find a live PID
|
|
# holding its own name and exit 1 under strict mode. Dropped by hand
|
|
# first; the new image re-acquires it immediately.
|
|
if [[ -z "${VV_PULL_REEXECED:-}" ]]; then
|
|
_vv_self_now="$(md5sum "$_VV_SELF" 2>/dev/null | cut -d' ' -f1)"
|
|
if [[ -n "$_vv_self_now" && -n "$_VV_SELF_SUM" && "$_vv_self_now" != "$_VV_SELF_SUM" ]]; then
|
|
echo " Pull updated this script — restarting at $(git rev-parse --short HEAD 2>/dev/null || echo 'unknown') so the rest of the run uses it"
|
|
_release_all_locks
|
|
export VV_PULL_REEXECED=1
|
|
exec bash "$_VV_SELF" "${_VV_SELF_ARGS[@]}"
|
|
fi
|
|
fi
|
|
|
|
# ── Permissions ──────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Permissions ━━━"
|
|
log "Setting executable permissions on all .sh files..."
|
|
find "$TARGET_DIR" -type f -name "*.sh" -exec chmod +x {} \;
|
|
echo " Permissions set on .sh files"
|
|
|
|
# ── Flash mode: sync Plugin/ to /boot/ so the webUI picks up updates ─────
|
|
# In flash mode SCRIPTS_DIR is in appdata — Plugin/ lives in the repo there
|
|
# but Unraid serves PHP from /boot/. Sync after every pull to keep them in step.
|
|
_BOOT_DIR=$(platform_webui_install_path)
|
|
if [[ "$TARGET_DIR" != "$_BOOT_DIR" ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Flash mode: sync Plugin/ → /boot/ ━━━"
|
|
if rsync -a --delete "$TARGET_DIR/Plugin/" "$_BOOT_DIR/Plugin/" 2>/dev/null; then
|
|
echo " Plugin/ synced to /boot/ ✅"
|
|
else
|
|
warn "Plugin/ sync to /boot/ failed — webUI may be stale until next pull"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Conf Upgrade ━━━
|
|
# ==============================================================================================
|
|
# Merges new conf structure into the live conf files after every pull.
|
|
# New keys → added with template defaults (user fills in once).
|
|
# Removed keys → dropped. Existing values → always preserved.
|
|
# Silent when already up to date — no overhead on unchanged pulls.
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Conf Upgrade ━━━"
|
|
|
|
UPGRADE_SCRIPT="$TARGET_DIR/Deployment/conf_upgrade.sh"
|
|
CONF_DIR="$TARGET_DIR/Configurations"
|
|
DEPLOY_DIR="$TARGET_DIR/Deployment"
|
|
|
|
if [[ ! -f "$UPGRADE_SCRIPT" ]]; then
|
|
log "conf_upgrade.sh not found — skipping (pre-deployment-folder repo)"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would upgrade master.conf and ${MY_ID,,}.conf"
|
|
elif [[ "$SYNC_SUCCESS" == true ]]; then
|
|
_DRY=""
|
|
|
|
# master.conf
|
|
if [[ -f "$DEPLOY_DIR/master.conf.template" && -f "$CONF_DIR/master.conf" ]]; then
|
|
bash "$UPGRADE_SCRIPT" \
|
|
--template "$DEPLOY_DIR/master.conf.template" \
|
|
--target "$CONF_DIR/master.conf" \
|
|
--backup $_DRY
|
|
else
|
|
warn "master.conf.template or master.conf not found — skipping"
|
|
fi
|
|
|
|
# This server's host conf only — sparse checkout ensures we have it
|
|
HOST_CONF="$CONF_DIR/${MY_ID,,}.conf"
|
|
if [[ -f "$DEPLOY_DIR/host.conf.template" && -f "$HOST_CONF" ]]; then
|
|
# The template is slot-generic; conf_upgrade resolves HOSTN/hostn to MY_ID and refuses
|
|
# if that disagrees with the target's own keys. This used to be a local sed that only
|
|
# replaced HOSTN_, which left bare HOSTN in comments — enough to trip conf_upgrade's
|
|
# own guard, so the host conf silently never upgraded — and left lowercase hostn alone,
|
|
# which would have installed an unreferenced hostn-appdata rsync profile.
|
|
bash "$UPGRADE_SCRIPT" \
|
|
--template "$DEPLOY_DIR/host.conf.template" \
|
|
--target "$HOST_CONF" \
|
|
--host-slot "$MY_ID" \
|
|
--backup $_DRY
|
|
else
|
|
warn "${MY_ID,,}.conf or host.conf.template not found — skipping"
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ AI Index Refresh ━━━
|
|
# ==============================================================================================
|
|
#
|
|
# A pull is the only thing that changes tracked files on a server — prod never edits them — so
|
|
# it is the only moment the AI index can go stale. A timer would be the wrong shape: it would
|
|
# do nothing 23 times a day and still drift from the pull that matters.
|
|
#
|
|
# Staleness is invisible in the answers themselves. The index keeps returning the old text with
|
|
# full confidence and correct-looking citations, so a day of drift means the assistant quoting
|
|
# code that no longer exists. That is why this runs here rather than being left to a human.
|
|
#
|
|
# Four gates, any of which skips it: the pull must have succeeded, AI_INDEX_ON_PULL must be true,
|
|
# AI_ENABLED must be true, and this must be the AI owner. ai_index.sh also refuses on its own on
|
|
# both the switch and the owner check, so a node never pays for this even if the flags disagree.
|
|
#
|
|
# The owner gate is the newest and was missing: the mesh shares one AI and the index belongs with
|
|
# the model, but this ran on every node regardless. On a node without a local Ollama it failed
|
|
# every night on the empty URL and said nothing, because the call below discards its output.
|
|
#
|
|
# Never fatal. Indexing is an enhancement; a git pull must not be reported as failed because an
|
|
# embedding call timed out.
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
[[ "${AI_INDEX_ON_PULL:-false}" == "true" ]] && warn "DRY RUN — would refresh the AI index"
|
|
elif [[ "$SYNC_SUCCESS" == true \
|
|
&& "${AI_INDEX_ON_PULL:-false}" == "true" \
|
|
&& "${AI_ENABLED:-false}" == "true" \
|
|
&& "${MY_ID,,}" == "${AI_OWNER_HOST:-host1}" ]]; then
|
|
|
|
_AI_INDEX="$TARGET_DIR/AI/ai_index.sh"
|
|
if [[ -f "$_AI_INDEX" ]]; then
|
|
log "Refreshing AI index (incremental — unchanged files are skipped)..."
|
|
if bash "$_AI_INDEX" >/dev/null 2>&1; then
|
|
echo " AI index refreshed"
|
|
else
|
|
warn "AI index refresh failed — answers may cite outdated code until it is rerun"
|
|
fi
|
|
else
|
|
warn "AI_INDEX_ON_PULL is true but $_AI_INDEX not found — skipping"
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY GIT SYNC SUMMARY ━━━━━"
|
|
echo "$ICON_NET Repo: $REPO_SSH"
|
|
echo "$ICON_GEAR Target: $TARGET_DIR"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_LOCK Excluded: peer host*.conf files"
|
|
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
|
elif [[ "$SYNC_SUCCESS" == true ]]; then
|
|
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
|
notify "Repository synced successfully on $(hostname)" "Git Sync" "normal"
|
|
else
|
|
echo "$ICON_ERROR Status: $ICON_ERROR FAILED"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |