Gmer4Lfe
2fefb56b8c
The restart line named no version, so a log showing one gave no way to tell what the rest of the run actually executed.
456 lines
21 KiB
Bash
Executable File
456 lines
21 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= Git Pull & Execute =========================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Pulls the latest scripts from the Gitea repository via SSH. Lives at the repo
|
|
# root — sources load_config.sh from the same directory.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# 1. detect_hosts() — identifies which server is running the pull (MY_ID)
|
|
# 2. Configure sparse checkout — exclude other servers' credential files
|
|
# 3. Pull or clone latest scripts from Gitea
|
|
# 4. Set executable permissions on all .sh files
|
|
#
|
|
# SPARSE CHECKOUT
|
|
# Each server only receives its own host conf — never peer credentials:
|
|
# HOST1 pulls: master.conf + host1.conf + all scripts
|
|
# HOST1 skips: host2.conf, host3.conf etc.
|
|
# Adding a new server: create host3.conf in the repo — all existing servers
|
|
# automatically exclude it on next pull; new server gets only its own conf.
|
|
#
|
|
# GITEA LOCATION DETECTION
|
|
# Detected at runtime — works through fallback:
|
|
# Gitea local → connects via local IP
|
|
# Gitea remote → connects via Tailscale IP
|
|
# Both fail → falls back to GITEA_DOMAIN if configured
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Credentials Stay Separated
|
|
# Sparse checkout is configured per-server on every pull, not just at clone
|
|
# time. Ensures newly added host confs are automatically excluded on all
|
|
# existing servers without any manual intervention.
|
|
#
|
|
# Runtime Location Detection
|
|
# Gitea's IP is never hardcoded — the script probes whether Gitea is local
|
|
# or remote on every run. Handles Gitea container restarts, migrations, and
|
|
# Tailscale address changes automatically.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root check — git operations on /boot require root
|
|
# acquire_lock — prevents concurrent pulls
|
|
# detect_hosts() — MY_ID required to build correct sparse checkout rules
|
|
# Docker check — Gitea container status probed before any SSH attempt
|
|
# Self re-exec — a pull that replaces this file restarts the run once, so the steps after
|
|
# the sync are the pulled ones rather than the ones already in memory
|
|
#
|
|
# The re-exec is the non-obvious one. This script lives in the repo it pulls, and git installs
|
|
# an updated file by rename — the running bash holds the old inode and finishes the run on the
|
|
# old logic. Nothing looks wrong; the change simply takes effect one run late. Bounded by
|
|
# VV_PULL_REEXECED so it can restart at most once, skipped entirely when the checksum is
|
|
# unchanged, and the lock is released by hand first because exec does not fire EXIT traps.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# GITEA_CONTAINER — Docker container name for Gitea
|
|
# GITEA_REPO_PATH — repo path on Gitea (e.g. Varaverk/varaverk.git)
|
|
# GITEA_DOMAIN — public domain fallback (optional)
|
|
# TARGET_DIR — local path to clone/pull into
|
|
# GITEA_SSH_KEY — SSH key for Gitea authentication
|
|
# SSH_PORT — Gitea SSH port (often 221 or 222)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# git_pull_execute.sh
|
|
# Normal pull.
|
|
#
|
|
# git_pull_execute.sh --dry-run
|
|
# Preview without making changes.
|
|
#
|
|
# git_pull_execute.sh --log
|
|
# Verbose output.
|
|
#
|
|
# git_pull_execute.sh --status
|
|
# Show config and exit.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
# This script lives inside the repo it pulls, so a pull can replace it mid-run. Recorded before
|
|
# anything happens; compared again once the pull lands. See the re-exec block after the sync.
|
|
_VV_SELF="$SCRIPT_DIR/$(basename "${BASH_SOURCE[0]}")"
|
|
_VV_SELF_ARGS=("$@")
|
|
_VV_SELF_SUM="$(md5sum "$_VV_SELF" 2>/dev/null | cut -d' ' -f1)"
|
|
|
|
# Root level script — load_config.sh is in the same directory
|
|
source "$SCRIPT_DIR/load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
acquire_lock
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
# detect_hosts() sets MY_ID — needed for sparse checkout configuration
|
|
detect_hosts
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Locate Gitea ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_CONTAINERS Locate Gitea ━━━"
|
|
|
|
if docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^${GITEA_CONTAINER}$"; then
|
|
# Gitea is running on this server — use local IP
|
|
GITEA_IP=$(hostname -I | awk '{print $1}')
|
|
log "Gitea running locally — connecting via $GITEA_IP"
|
|
else
|
|
# Gitea not running locally — find it on the remote server via Tailscale
|
|
log "Gitea not running locally — checking remote server"
|
|
GITEA_IP=$(resolve_tailscale_ip "${REMOTE_SERVER_NAME}")
|
|
if [[ -n "$GITEA_IP" ]]; then
|
|
echo " Gitea on $REMOTE_SERVER_NAME — connecting via Tailscale $GITEA_IP"
|
|
elif [[ -n "${GITEA_DOMAIN:-}" ]]; then
|
|
warn "Tailscale resolution failed — falling back to $GITEA_DOMAIN"
|
|
GITEA_IP="$GITEA_DOMAIN"
|
|
else
|
|
error "Cannot find Gitea — local: not running, Tailscale: failed, domain: not configured"
|
|
notify "Git pull failed on $(hostname) — cannot locate Gitea container" "Git Sync" "alert"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
REPO_SSH="git@${GITEA_IP}:${GITEA_REPO_PATH}"
|
|
|
|
require_var REPO_SSH
|
|
require_var TARGET_DIR
|
|
require_var GITEA_SSH_KEY
|
|
require_var SSH_PORT
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_NET Repo: $REPO_SSH"
|
|
echo "$ICON_GEAR Target: $TARGET_DIR"
|
|
echo "$ICON_GEAR SSH Key: $GITEA_SSH_KEY"
|
|
echo "$ICON_GEAR SSH Port: $SSH_PORT"
|
|
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_HOST Remote ID: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
|
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Sparse Checkout Configuration ━━━
|
|
# ==============================================================================================
|
|
# Build the list of host*.conf files that belong to OTHER servers.
|
|
# This server pulls everything EXCEPT those files.
|
|
# MY_ID is set by detect_hosts() — e.g. "HOST1"
|
|
|
|
configure_sparse_checkout() {
|
|
local repo_dir="$1"
|
|
|
|
log "Configuring sparse checkout for $MY_ID..."
|
|
|
|
# Enable sparse checkout
|
|
git -C "$repo_dir" config core.sparseCheckout true 2>/dev/null
|
|
|
|
# Build exclusion list — all host*.conf files except MY_ID's
|
|
local sparse_file="$repo_dir/.git/info/sparse-checkout"
|
|
mkdir -p "$(dirname "$sparse_file")"
|
|
|
|
# Start with: pull everything
|
|
echo "/*" > "$sparse_file"
|
|
|
|
# Exclude each other server's conf file
|
|
# Find all host*.conf files present in the repo
|
|
local excluded=0
|
|
for conf_file in "$repo_dir"/host*.conf; do
|
|
[[ -f "$conf_file" ]] || continue
|
|
local conf_name
|
|
conf_name=$(basename "$conf_file")
|
|
|
|
# Determine which HOST ID owns this conf by grepping its hostname var
|
|
# Pattern: HOST1="unRAID-..." or HOST2="unRAID-..."
|
|
local conf_host_id
|
|
conf_host_id=$(grep -m1 -oP '^\s+HOST[0-9]+(?==)' "$conf_file" 2>/dev/null | tr -d ' ')
|
|
|
|
if [[ -z "$conf_host_id" ]]; then
|
|
log "Cannot determine HOST ID for $conf_name — including in pull (safe default)"
|
|
continue
|
|
fi
|
|
|
|
if [[ "$conf_host_id" != "$MY_ID" ]]; then
|
|
echo "!$conf_name" >> "$sparse_file"
|
|
log "Sparse checkout: excluding $conf_name (belongs to $conf_host_id)"
|
|
((excluded++))
|
|
else
|
|
log "Sparse checkout: including $conf_name (belongs to $MY_ID — this server)"
|
|
fi
|
|
done
|
|
|
|
if [[ "$excluded" -gt 0 ]]; then
|
|
echo " Sparse checkout: excluding $excluded peer conf file(s) — credentials protected"
|
|
else
|
|
log "Sparse checkout: no peer conf files to exclude (single server or first run)"
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Git Sync ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Git Sync ━━━"
|
|
echo "$ICON_NET Repo: $REPO_SSH"
|
|
echo "$ICON_GEAR Target: $TARGET_DIR"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
SYNC_SUCCESS=false
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would sync $REPO_SSH → $TARGET_DIR"
|
|
warn "DRY RUN — would configure sparse checkout for $MY_ID"
|
|
warn "DRY RUN — would exclude peer host*.conf files"
|
|
SYNC_SUCCESS=true
|
|
else
|
|
mkdir -p "$TARGET_DIR"
|
|
git config --global --add safe.directory "$TARGET_DIR"
|
|
cd "$TARGET_DIR" || { error "Cannot cd into $TARGET_DIR"; exit 1; }
|
|
|
|
if [[ -d ".git" ]]; then
|
|
# ── Existing repository ──────────────────────────────────────────────
|
|
echo " Existing repository — updating"
|
|
|
|
# Configure sparse checkout BEFORE pull
|
|
# Uses conf files already present from last pull to determine exclusions
|
|
configure_sparse_checkout "$TARGET_DIR"
|
|
|
|
echo " Pulling latest changes..."
|
|
if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git pull --ff-only; then
|
|
echo " Git pull successful"
|
|
SYNC_SUCCESS=true
|
|
else
|
|
# ff-only fails when local commits or tracked changes exist that can't
|
|
# fast-forward. Fail loudly — never silently destroy local work.
|
|
error "Git pull failed — local changes conflict with remote (will not force-reset)"
|
|
notify "Git pull failed on $(hostname) — local changes conflict, manual resolve needed" "Git Sync" "alert"
|
|
exit 1
|
|
fi
|
|
|
|
else
|
|
# ── Fresh clone ──────────────────────────────────────────────────────
|
|
echo " No repository found — cloning"
|
|
|
|
# Clone first — need the repo to exist before configuring sparse checkout
|
|
if GIT_SSH_COMMAND="ssh -i $GITEA_SSH_KEY -p $SSH_PORT" git clone "$REPO_SSH" .; then
|
|
echo " Clone successful"
|
|
|
|
# Configure sparse checkout after clone
|
|
# Now all host*.conf files are present — can detect exclusions
|
|
configure_sparse_checkout "$TARGET_DIR"
|
|
|
|
# Apply sparse checkout — removes excluded files from working tree
|
|
echo " Applying sparse checkout..."
|
|
git read-tree -mu HEAD
|
|
echo " Sparse checkout applied — peer credentials removed from working tree"
|
|
|
|
SYNC_SUCCESS=true
|
|
else
|
|
error "Clone failed"
|
|
notify "Git clone failed on $(hostname) — check Gitea connectivity" "Git Sync" "alert"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# ── Re-exec if the pull replaced this script ─────────────────────────────
|
|
#
|
|
# git installs an updated file by rename, so the running bash keeps reading the inode it
|
|
# started with. That is the benign half — no torn parse, no garbage — but it means every
|
|
# step below this line runs the version that was on disk when the run began, not the one
|
|
# just pulled. Observed 2026-08-08: a pull carrying a fix to the conf-upgrade call executed
|
|
# the old call anyway, and the fix only took effect on the following run.
|
|
#
|
|
# Re-exec makes the pull self-applying. Guarded three ways:
|
|
# VV_PULL_REEXECED — set before exec, so the new image can never re-exec again. One
|
|
# restart per run, whatever the checksums say.
|
|
# checksum compare — no change means no restart, so the common case costs one md5sum.
|
|
# lock released — exec keeps the PID but does NOT fire the EXIT trap, so the lock file
|
|
# would survive into the new image, which would then find a live PID
|
|
# holding its own name and exit 1 under strict mode. Dropped by hand
|
|
# first; the new image re-acquires it immediately.
|
|
if [[ -z "${VV_PULL_REEXECED:-}" ]]; then
|
|
_vv_self_now="$(md5sum "$_VV_SELF" 2>/dev/null | cut -d' ' -f1)"
|
|
if [[ -n "$_vv_self_now" && -n "$_VV_SELF_SUM" && "$_vv_self_now" != "$_VV_SELF_SUM" ]]; then
|
|
echo " Pull updated this script — restarting at $(git rev-parse --short HEAD 2>/dev/null || echo 'unknown') so the rest of the run uses it"
|
|
_release_all_locks
|
|
export VV_PULL_REEXECED=1
|
|
exec bash "$_VV_SELF" "${_VV_SELF_ARGS[@]}"
|
|
fi
|
|
fi
|
|
|
|
# ── Permissions ──────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Permissions ━━━"
|
|
log "Setting executable permissions on all .sh files..."
|
|
find "$TARGET_DIR" -type f -name "*.sh" -exec chmod +x {} \;
|
|
echo " Permissions set on .sh files"
|
|
|
|
# ── Flash mode: sync Plugin/ to /boot/ so the webUI picks up updates ─────
|
|
# In flash mode SCRIPTS_DIR is in appdata — Plugin/ lives in the repo there
|
|
# but Unraid serves PHP from /boot/. Sync after every pull to keep them in step.
|
|
_BOOT_DIR=$(platform_webui_install_path)
|
|
if [[ "$TARGET_DIR" != "$_BOOT_DIR" ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Flash mode: sync Plugin/ → /boot/ ━━━"
|
|
if rsync -a --delete "$TARGET_DIR/Plugin/" "$_BOOT_DIR/Plugin/" 2>/dev/null; then
|
|
echo " Plugin/ synced to /boot/ ✅"
|
|
else
|
|
warn "Plugin/ sync to /boot/ failed — webUI may be stale until next pull"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Conf Upgrade ━━━
|
|
# ==============================================================================================
|
|
# Merges new conf structure into the live conf files after every pull.
|
|
# New keys → added with template defaults (user fills in once).
|
|
# Removed keys → dropped. Existing values → always preserved.
|
|
# Silent when already up to date — no overhead on unchanged pulls.
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Conf Upgrade ━━━"
|
|
|
|
UPGRADE_SCRIPT="$TARGET_DIR/Deployment/conf_upgrade.sh"
|
|
CONF_DIR="$TARGET_DIR/Configurations"
|
|
DEPLOY_DIR="$TARGET_DIR/Deployment"
|
|
|
|
if [[ ! -f "$UPGRADE_SCRIPT" ]]; then
|
|
log "conf_upgrade.sh not found — skipping (pre-deployment-folder repo)"
|
|
elif [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would upgrade master.conf and ${MY_ID,,}.conf"
|
|
elif [[ "$SYNC_SUCCESS" == true ]]; then
|
|
_DRY=""
|
|
|
|
# master.conf
|
|
if [[ -f "$DEPLOY_DIR/master.conf.template" && -f "$CONF_DIR/master.conf" ]]; then
|
|
bash "$UPGRADE_SCRIPT" \
|
|
--template "$DEPLOY_DIR/master.conf.template" \
|
|
--target "$CONF_DIR/master.conf" \
|
|
--backup $_DRY
|
|
else
|
|
warn "master.conf.template or master.conf not found — skipping"
|
|
fi
|
|
|
|
# This server's host conf only — sparse checkout ensures we have it
|
|
HOST_CONF="$CONF_DIR/${MY_ID,,}.conf"
|
|
if [[ -f "$DEPLOY_DIR/host.conf.template" && -f "$HOST_CONF" ]]; then
|
|
# The template is slot-generic; conf_upgrade resolves HOSTN/hostn to MY_ID and refuses
|
|
# if that disagrees with the target's own keys. This used to be a local sed that only
|
|
# replaced HOSTN_, which left bare HOSTN in comments — enough to trip conf_upgrade's
|
|
# own guard, so the host conf silently never upgraded — and left lowercase hostn alone,
|
|
# which would have installed an unreferenced hostn-appdata rsync profile.
|
|
bash "$UPGRADE_SCRIPT" \
|
|
--template "$DEPLOY_DIR/host.conf.template" \
|
|
--target "$HOST_CONF" \
|
|
--host-slot "$MY_ID" \
|
|
--backup $_DRY
|
|
else
|
|
warn "${MY_ID,,}.conf or host.conf.template not found — skipping"
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ AI Index Refresh ━━━
|
|
# ==============================================================================================
|
|
#
|
|
# A pull is the only thing that changes tracked files on a server — prod never edits them — so
|
|
# it is the only moment the AI index can go stale. A timer would be the wrong shape: it would
|
|
# do nothing 23 times a day and still drift from the pull that matters.
|
|
#
|
|
# Staleness is invisible in the answers themselves. The index keeps returning the old text with
|
|
# full confidence and correct-looking citations, so a day of drift means the assistant quoting
|
|
# code that no longer exists. That is why this runs here rather than being left to a human.
|
|
#
|
|
# Three gates, any of which skips it: the pull must have succeeded, AI_INDEX_ON_PULL must be
|
|
# true, and AI_ENABLED must be true. ai_index.sh also refuses on its own unless AI_ENABLED is
|
|
# exactly "true", so a node with AI off never pays for this even if the flags disagree.
|
|
#
|
|
# Never fatal. Indexing is an enhancement; a git pull must not be reported as failed because an
|
|
# embedding call timed out.
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
[[ "${AI_INDEX_ON_PULL:-false}" == "true" ]] && warn "DRY RUN — would refresh the AI index"
|
|
elif [[ "$SYNC_SUCCESS" == true \
|
|
&& "${AI_INDEX_ON_PULL:-false}" == "true" \
|
|
&& "${AI_ENABLED:-false}" == "true" ]]; then
|
|
|
|
_AI_INDEX="$TARGET_DIR/AI/ai_index.sh"
|
|
if [[ -f "$_AI_INDEX" ]]; then
|
|
log "Refreshing AI index (incremental — unchanged files are skipped)..."
|
|
if bash "$_AI_INDEX" >/dev/null 2>&1; then
|
|
echo " AI index refreshed"
|
|
else
|
|
warn "AI index refresh failed — answers may cite outdated code until it is rerun"
|
|
fi
|
|
else
|
|
warn "AI_INDEX_ON_PULL is true but $_AI_INDEX not found — skipping"
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY GIT SYNC SUMMARY ━━━━━"
|
|
echo "$ICON_NET Repo: $REPO_SSH"
|
|
echo "$ICON_GEAR Target: $TARGET_DIR"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_LOCK Excluded: peer host*.conf files"
|
|
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
|
elif [[ "$SYNC_SUCCESS" == true ]]; then
|
|
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
|
notify "Repository synced successfully on $(hostname)" "Git Sync" "normal"
|
|
else
|
|
echo "$ICON_ERROR Status: $ICON_ERROR FAILED"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" |