From 427a31cdffc7688d94e8097a6d95f8cd7f8bf775 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Mon, 17 Aug 2026 10:12:10 -0400 Subject: [PATCH] Sync the auth stack's appdata to the mirror before the containers that read it are created Deploying first meant Authelia, Lldap, NPM and both databases initialised themselves against empty directories. The gate arming moves to Step 1d for the same reason: the provisioning sync is an rsync, and Tier 1 stops every rsync. Tier 2 now ends the onboard off rather than as-found. --- Deployment/host.conf.template | 10 ++ Partnership/partnership_onboard.sh | 170 +++++++++++++++++++---------- common.sh | 2 + 3 files changed, 124 insertions(+), 58 deletions(-) diff --git a/Deployment/host.conf.template b/Deployment/host.conf.template index 61a047d..6d5cbb4 100644 --- a/Deployment/host.conf.template +++ b/Deployment/host.conf.template @@ -155,6 +155,16 @@ # "Authelia|9091" ) +# Shares rsynced to the mirror during onboard Step 1e, BEFORE the auth containers are created. +# This is the only rsync an onboard performs — media is never seeded here. +# Profile is inferred from the directory basename, so Critical-Data resolves to critical-data: +# a clean copy with the auth containers stopped on both sides. Do not point this at a share +# whose profile keeps databases running; a dirty copy of MariaDB or Redis is worse than none, +# because the container starts, reads Up, and restarts a dead database behind it. + HOSTN_PARTNERSHIP_PROVISION_SHARES=( + "/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — the auth stack + ) + # XML templates pushed to mirror during onboard — auth stack. # Dependencies (databases) must come before apps that depend on them. HOSTN_PARTNERSHIP_AUTH_STACK=( diff --git a/Partnership/partnership_onboard.sh b/Partnership/partnership_onboard.sh index 1baeefa..6064901 100755 --- a/Partnership/partnership_onboard.sh +++ b/Partnership/partnership_onboard.sh @@ -22,6 +22,10 @@ # Step 1: SSH key setup — generate keypair, install on mirror, update conf # Step 1b: Docker network — ensure varaverk docker network exists on mirror # Step 1c: Share setup — create missing Unraid shares on mirror (pool-aware, idempotent) +# Step 1d: Sync gates — Tier 1 + CONF_SYNC + ARR_SYNC on, every Tier 2 rsync gate +# off. Here, not at the end, because Step 1e is an rsync +# Step 1e: Auth appdata — rsync PARTNERSHIP_PROVISION_SHARES to the mirror BEFORE the +# containers that read it exist. The only rsync an onboard does # Step 2: Stop mirror auth — stop mirror's existing auth containers before replacing # Step 3: Deploy auth stack — push XMLs, pull images, create + start on mirror # Mariadb/Redis health-checked before Authelia deploys @@ -32,10 +36,6 @@ # Step 8: Partnership onboard — configure WebUIs → owner IP, write state, Emby # Step 9: Arr bootstrap — bidirectional library sync (arr_sync.sh) # Step 9b: Webhook setup — register download webhook in arrs on both servers -# Step 9c: Arm sync gates — open RSYNC/CONF_SYNC/ARR_SYNC in master.conf, which the -# template ships closed so a fresh node cannot sync early. -# Still ahead of the Step 13 seed, which is itself an rsync -# and dies at the Tier 1 gate if this has not run # Step 9e: Webhook listener — start listener on mirror (runs continuously, no reboot needed) # Step 10: Conf push — push master.conf + setup state to all listed hosts # Step 11: Service discovery — conf_populate.sh on the mirror, last, once the stacks it @@ -159,6 +159,10 @@ # Partnership/partnership_onboard.sh --skip-webhook-setup # Skip webhook registration in arrs (Step 9b) # +# Partnership/partnership_onboard.sh --skip-appdata-provision +# Skip the pre-container auth appdata rsync (Step 1e) +# The mirror's auth containers then start against whatever is already there +# # Partnership/partnership_onboard.sh --skip-media-seed # Skip dispatching the background media seed (Step 13) # Use when mirror already has files or you want to seed manually @@ -168,7 +172,7 @@ # Listener will start automatically on next array restart # # Partnership/partnership_onboard.sh --no-arm -# Leave RSYNC_ENABLED / CONF_SYNC_ENABLED / ARR_SYNC_ENABLED as they are (Step 9c). +# Leave every sync gate as it is (Step 1d). Step 1e then has no Tier 1 and cannot provision. # Use when onboarding a node you want to keep inert — a rebuild test, or a mirror whose # shares are not populated yet. # @@ -204,7 +208,8 @@ SKIP_WEBHOOK_LISTENER=false PHASE1_ONLY=false # OWNER: SSH + conf push only (HOST2 not yet installed) # MIRROR: SSH key install only, no owner notification PHASE2_ONLY=false # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards) -SKIP_ARM=false # leave the sync gates as they are — see Step 9c +SKIP_ARM=false # leave the sync gates as they are — see Step 1d +SKIP_APPDATA_PROVISION=false # skip the pre-container auth appdata sync — see Step 1e FILTERED_ARGS=() for arg in "$@"; do @@ -217,6 +222,7 @@ for arg in "$@"; do --skip-arr-sync) SKIP_ARR_SYNC=true ;; --skip-webhook-setup) SKIP_WEBHOOK_SETUP=true ;; --skip-media-seed) SKIP_MEDIA_SEED=true ;; + --skip-appdata-provision) SKIP_APPDATA_PROVISION=true ;; --skip-webhook-listener) SKIP_WEBHOOK_LISTENER=true ;; --phase1-only) PHASE1_ONLY=true ;; --phase2-only) PHASE2_ONLY=true; SKIP_SSH=true ;; @@ -615,7 +621,7 @@ if [[ "$PHASE1_ONLY" == true ]]; then fi # ── Phase 1 — Partner conf cache ────────────────────────────────────────────────────────── - # CONF_SYNC_ENABLED is armed here rather than waiting for Step 9c. It is the safe one of the + # CONF_SYNC_ENABLED is armed here rather than waiting for Step 1d. It is the safe one of the # three gates — it moves no data, it copies each side's host*.conf into the other's tmpfs so # partner vars resolve — and the moment SSH works is the moment that should start. Leaving it # until Phase 2 meant conf_sync.sh, which sources the conf fresh in its own process, exited @@ -745,6 +751,102 @@ else bash "$SCRIPT_DIR/share_setup.sh" fi +# ── Step 1d: Sync gates ─────────────────────────────────────────────────────────────────────── +# This was Step 9c, after the containers were already deployed. It has to run here instead, +# because Step 1e below is itself an rsync and Tier 1 stops every rsync — arming afterwards +# meant the provisioning sync exited cleanly having moved nothing. +# +# master.conf.template ships a fresh node inert. A successful Phase 1 is what makes Tier 1, +# CONF_SYNC and ARR_SYNC true; without this the defaults were a one-way door and somebody had +# to remember to hand-edit three toggles on the machine where forgetting looks exactly like +# everything working. +# +# Tier 2 is now set false rather than "left as configured". Tier 1 opening is what makes the +# scheduled jobs live, and a node that was onboarded ten seconds ago is not a node anyone has +# checked yet — free space, share layout, what the partner actually holds. Arriving connected +# and idle is the useful state; the operator arms each orchestrator when they mean to. +# MEDIA_SEED_ENABLED is not touched here at all — see Step 13. +# +# Owner only — the mirror receives these values in the Step 10 push rather than deciding. +ARM_OK=true +_VV_ARM_ON=(RSYNC_ENABLED CONF_SYNC_ENABLED ARR_SYNC_ENABLED) +_VV_ARM_OFF=(CRITICAL_RSYNC_ENABLED INTERMEDIATE_RSYNC_ENABLED DAILY_RSYNC_ENABLED + WEEKLY_RSYNC_ENABLED MONTHLY_RSYNC_ENABLED FALLBACK_RSYNC_ENABLED) +echo "" +echo "━━━ $ICON_GEAR Step 1d — Sync Gates ━━━" + +if [[ "$SKIP_ARM" == true ]]; then + echo " --no-arm — leaving the sync gates as they are" + warn " Step 1e needs Tier 1 open; with it closed the auth appdata will not be provisioned" +elif [[ "$AM_OWNER" != true ]]; then + echo " mirror — the owner's push decides these" +elif [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would set ${_VV_ARM_ON[*]} true and ${_VV_ARM_OFF[*]} false" +else + _master_conf="$SCRIPTS_ROOT/Configurations/master.conf" + if [[ ! -f "$_master_conf" ]]; then + warn "master.conf not found at $_master_conf — gates left closed" + ARM_OK=false + else + cp -a "$_master_conf" "${_master_conf}.bak-arm-$(date +%Y%m%d-%H%M%S)" + for _gate in "${_VV_ARM_ON[@]}"; do set_conf_bool "$_gate" "true" "$_master_conf" || ARM_OK=false; done + for _gate in "${_VV_ARM_OFF[@]}"; do set_conf_bool "$_gate" "false" "$_master_conf" || ARM_OK=false; done + unset _gate + echo " Tier 1 open: ${_VV_ARM_ON[*]}" + echo " Tier 2 off: every scheduled rsync — arm them yourself when you are ready" + fi + unset _master_conf +fi + +# ── Step 1e: Provision the auth stack's appdata ─────────────────────────────────────────────── +# The one rsync an onboard actually needs, and it has to land before Step 3 creates the +# containers that read it. Deploying first meant Authelia, Lldap, NPM and both databases came up +# on the mirror against empty directories and initialised themselves from nothing — so the +# mirror's auth stack was a fresh install wearing the owner's container names. +# +# The critical-data profile, not critical-fallback. critical-data stops the auth containers on +# both sides for the duration, which costs the owner its auth for about a minute at 588 MB, and +# buys a consistent copy. critical-fallback is the dirty variant that keeps auth running, and a +# dirty copy is exactly what broke Redis and MariaDB on the mirror before: the manifest and the +# binlog index were copied without the files they name, and MariaDB hides that — the container +# reads Up while mysqld_safe restarts the dead database every few seconds. +# +# Media is not seeded here or anywhere in Phase 2. This share is 588 MB and the containers do +# not start correctly without it; a media library is terabytes and nothing waits on it. +echo "" +echo "━━━ $ICON_SYNC Step 1e — Auth Appdata Provision ━━━" + +APPDATA_PROVISION_OK=false +if [[ "$SKIP_APPDATA_PROVISION" == true ]]; then + warn "Skipping (--skip-appdata-provision) — the mirror's auth stack will start empty" +elif [[ "${#PARTNERSHIP_PROVISION_SHARES[@]}" -eq 0 ]]; then + warn "PARTNERSHIP_PROVISION_SHARES is empty — nothing to provision" + warn " Set it in host${MY_ID: -1}.conf, or the mirror's auth stack starts from nothing" +elif [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would sync ${PARTNERSHIP_PROVISION_SHARES[*]} to $MIRROR" + APPDATA_PROVISION_OK=true +else + _prov_rc=0 + for _share in "${PARTNERSHIP_PROVISION_SHARES[@]}"; do + echo " Provisioning: $_share" + _prov_flags=() + [[ "$ENABLE_LOGGING" == true ]] && _prov_flags+=(--log) + if ! bash "$SCRIPTS_ROOT/Rsync/rsync.sh" "$_share" "${_prov_flags[@]}"; then + warn " Failed: $_share" + _prov_rc=1 + fi + done + unset _share _prov_flags + if [[ "$_prov_rc" -eq 0 ]]; then + echo "Auth appdata provisioned to $MIRROR ✅" + APPDATA_PROVISION_OK=true + else + warn "Auth appdata NOT fully provisioned — the containers below will start against" + warn " whatever is already on $MIRROR, which on a fresh node is nothing" + fi + unset _prov_rc +fi + # ── Step 2: Stop mirror's existing auth stack ───────────────────────────────────────────────── echo "" echo "━━━ Step 2 — Stop Mirror Auth Stack ━━━" @@ -881,55 +983,6 @@ else fi unset _webhook_script -# ── Step 9c: Arm the sync gates ─────────────────────────────────────────────────────────────── -# master.conf.template ships a fresh node inert — RSYNC_ENABLED, CONF_SYNC_ENABLED and -# ARR_SYNC_ENABLED all false — because a node that has just been seeded has empty shares and no -# verified partner, and those two facts are what make unattended syncing safe to do. -# -# A successful onboard is the event that makes them true. Without this step the defaults were a -# one-way door: the node stayed inert for ever and somebody had to remember to hand-edit three -# toggles, on the machine where forgetting looks exactly like everything working. -# -# Ahead of the media seed, which it used to follow. The seed calls Rsync/rsync.sh, and Tier 1 -# stops *every* rsync — so with the gates still closed all of it exited cleanly having moved -# nothing, and, because a clean exit is exit 0, the seed counted each share as a success and -# reported "14/14 shares ✅" over an empty mirror. -# -# Still ahead of Step 10 on purpose. The push carries the owner's master.conf to every listed -# host, so arming before it means both sides come up agreeing about whether sync is on; arming -# after the push would leave the mirror a version behind until the next conf save. -# -# Owner only — the mirror receives these values in the push rather than deciding for itself. -ARM_OK=true -echo "" -echo "━━━ $ICON_GEAR Step 9c — Arm Sync Gates ━━━" - -if [[ "$ONBOARD_OK" == false ]]; then - warn "Skipping — onboard did not complete, leaving the gates closed" - ARM_OK=false -elif [[ "$SKIP_ARM" == true ]]; then - echo " --no-arm — leaving the sync gates as they are" -elif [[ "$AM_OWNER" != true ]]; then - echo " mirror — the owner's push decides these" -elif [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would set RSYNC_ENABLED, CONF_SYNC_ENABLED, ARR_SYNC_ENABLED to true" -else - _master_conf="$SCRIPTS_ROOT/Configurations/master.conf" - if [[ ! -f "$_master_conf" ]]; then - warn "master.conf not found at $_master_conf — gates left closed" - ARM_OK=false - else - cp -a "$_master_conf" "${_master_conf}.bak-arm-$(date +%Y%m%d-%H%M%S)" - for _gate in RSYNC_ENABLED CONF_SYNC_ENABLED ARR_SYNC_ENABLED; do - set_conf_bool "$_gate" "true" "$_master_conf" || ARM_OK=false - done - # Tier 1 is now open, so say what that actually turned on rather than leaving it implied. - echo " rsync tier 2 gates were left as configured — check them before the first run" - unset _gate - fi - unset _master_conf -fi - # ── Step 9e: Start webhook listener on mirror ───────────────────────────────────────────────── # Listener is in ARRAY_START_SCRIPTS so it starts on next boot, but the mirror's array is # already running — kick it now so events are captured immediately after onboard. @@ -1089,7 +1142,7 @@ _seed_job="Rsync/media_seed.sh" _seed_script="$SCRIPTS_ROOT/Rsync/media_seed.sh" _runner="$SCRIPTS_ROOT/Plugin/$PLATFORM/run_job.sh" -# Read from disk, not from the sourced value: Step 9c rewrote master.conf a few steps ago. +# Read from disk, not from the sourced value: Step 1d rewrote master.conf a few steps ago. # Unset reads as on — the toggle postdates the seed, so a conf that has not been through a # conf_upgrade must keep the behaviour it had. media_seed.sh checks this again itself; the # check here exists so the summary can say "disabled" instead of dispatching a job whose only @@ -1155,6 +1208,8 @@ _skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; } echo " Step 1 — SSH keys: $(_skip "$SKIP_SSH" "$STEP_SSH_OK")" echo " Step 1b — Docker network: $(_ok "$STEP_NETWORK_OK")" +echo " Step 1d — Sync gates: $( [[ "$SKIP_ARM" == true ]] && echo "skipped (--no-arm)" || { [[ "$AM_OWNER" != true ]] && echo "mirror — set by owner" || _ok "$ARM_OK"; } )" +echo " Step 1e — Auth appdata: $( [[ "$SKIP_APPDATA_PROVISION" == true ]] && echo "skipped" || _ok "$APPDATA_PROVISION_OK" )" echo " Step 2 — Stop auth: $(_skip "$SKIP_AUTH_STACK" "$STEP_STOP_AUTH_OK")" echo " Step 3 — Auth stack: $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )" echo " Step 4 — Stop arr: $(_skip "$SKIP_ARR_STACK" "$STEP_STOP_ARR_OK")" @@ -1164,7 +1219,6 @@ echo " Step 7 — Services stack: $( [[ "$SKIP_SERVICES_STACK" == true ]] && echo " Step 8 — Onboard: $(_ok "$ONBOARD_OK")" echo " Step 9 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )" echo " Step 9b — Webhook setup: $(_skip "$SKIP_WEBHOOK_SETUP" "$WEBHOOK_SETUP_OK")" -echo " Step 9c — Arm sync gates: $( [[ "$SKIP_ARM" == true ]] && echo "skipped (--no-arm)" || { [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARM_OK")"; } )" echo " Step 9e — Webhook listener: $(_skip "$SKIP_WEBHOOK_LISTENER" "$WEBHOOK_LISTENER_OK")" echo " Step 10 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )" echo " Step 11 — Discovery: $( [[ "$POPULATE_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$POPULATE_OK" )" diff --git a/common.sh b/common.sh index 2485ff8..f9ef3f0 100755 --- a/common.sh +++ b/common.sh @@ -501,6 +501,7 @@ is_in_list() { # # Also sets aliases for all host-specific arrays so scripts use unprefixed names: # DAILY_SYNC_SHARES ← HOST*_DAILY_SYNC_SHARES +# PARTNERSHIP_PROVISION_SHARES ← HOST*_PARTNERSHIP_PROVISION_SHARES # INTERMEDIATE_SYNC_SHARES ← HOST*_INTERMEDIATE_SYNC_SHARES # WEEKLY_SYNC_SHARES ← HOST*_WEEKLY_SYNC_SHARES # CRITICAL_SYNC_SHARES ← HOST*_CRITICAL_SYNC_SHARES @@ -729,6 +730,7 @@ detect_hosts() { _alias_array "INTERMEDIATE_SYNC_SHARES" _alias_array "WEEKLY_SYNC_SHARES" _alias_array "CRITICAL_SYNC_SHARES" + _alias_array "PARTNERSHIP_PROVISION_SHARES" _alias_array "BACKUP_VERIFY_SHARES" _alias_array "DAILY_RESTART_CONTAINERS" _alias_array "WEEKLY_RESTART_CONTAINERS"