diff --git a/Deployment/host.conf.template b/Deployment/host.conf.template index 61a047d..6d5cbb4 100644 --- a/Deployment/host.conf.template +++ b/Deployment/host.conf.template @@ -155,6 +155,16 @@ # "Authelia|9091" ) +# Shares rsynced to the mirror during onboard Step 1e, BEFORE the auth containers are created. +# This is the only rsync an onboard performs — media is never seeded here. +# Profile is inferred from the directory basename, so Critical-Data resolves to critical-data: +# a clean copy with the auth containers stopped on both sides. Do not point this at a share +# whose profile keeps databases running; a dirty copy of MariaDB or Redis is worse than none, +# because the container starts, reads Up, and restarts a dead database behind it. + HOSTN_PARTNERSHIP_PROVISION_SHARES=( + "/mnt/user/appdata-Fallback/Critical-Data" # critical-data profile — the auth stack + ) + # XML templates pushed to mirror during onboard — auth stack. # Dependencies (databases) must come before apps that depend on them. HOSTN_PARTNERSHIP_AUTH_STACK=( diff --git a/Partnership/partnership_onboard.sh b/Partnership/partnership_onboard.sh index 1baeefa..6064901 100755 --- a/Partnership/partnership_onboard.sh +++ b/Partnership/partnership_onboard.sh @@ -22,6 +22,10 @@ # Step 1: SSH key setup — generate keypair, install on mirror, update conf # Step 1b: Docker network — ensure varaverk docker network exists on mirror # Step 1c: Share setup — create missing Unraid shares on mirror (pool-aware, idempotent) +# Step 1d: Sync gates — Tier 1 + CONF_SYNC + ARR_SYNC on, every Tier 2 rsync gate +# off. Here, not at the end, because Step 1e is an rsync +# Step 1e: Auth appdata — rsync PARTNERSHIP_PROVISION_SHARES to the mirror BEFORE the +# containers that read it exist. The only rsync an onboard does # Step 2: Stop mirror auth — stop mirror's existing auth containers before replacing # Step 3: Deploy auth stack — push XMLs, pull images, create + start on mirror # Mariadb/Redis health-checked before Authelia deploys @@ -32,10 +36,6 @@ # Step 8: Partnership onboard — configure WebUIs → owner IP, write state, Emby # Step 9: Arr bootstrap — bidirectional library sync (arr_sync.sh) # Step 9b: Webhook setup — register download webhook in arrs on both servers -# Step 9c: Arm sync gates — open RSYNC/CONF_SYNC/ARR_SYNC in master.conf, which the -# template ships closed so a fresh node cannot sync early. -# Still ahead of the Step 13 seed, which is itself an rsync -# and dies at the Tier 1 gate if this has not run # Step 9e: Webhook listener — start listener on mirror (runs continuously, no reboot needed) # Step 10: Conf push — push master.conf + setup state to all listed hosts # Step 11: Service discovery — conf_populate.sh on the mirror, last, once the stacks it @@ -159,6 +159,10 @@ # Partnership/partnership_onboard.sh --skip-webhook-setup # Skip webhook registration in arrs (Step 9b) # +# Partnership/partnership_onboard.sh --skip-appdata-provision +# Skip the pre-container auth appdata rsync (Step 1e) +# The mirror's auth containers then start against whatever is already there +# # Partnership/partnership_onboard.sh --skip-media-seed # Skip dispatching the background media seed (Step 13) # Use when mirror already has files or you want to seed manually @@ -168,7 +172,7 @@ # Listener will start automatically on next array restart # # Partnership/partnership_onboard.sh --no-arm -# Leave RSYNC_ENABLED / CONF_SYNC_ENABLED / ARR_SYNC_ENABLED as they are (Step 9c). +# Leave every sync gate as it is (Step 1d). Step 1e then has no Tier 1 and cannot provision. # Use when onboarding a node you want to keep inert — a rebuild test, or a mirror whose # shares are not populated yet. # @@ -204,7 +208,8 @@ SKIP_WEBHOOK_LISTENER=false PHASE1_ONLY=false # OWNER: SSH + conf push only (HOST2 not yet installed) # MIRROR: SSH key install only, no owner notification PHASE2_ONLY=false # OWNER: containers/arr/onboard only (triggered by HOST2 after it onboards) -SKIP_ARM=false # leave the sync gates as they are — see Step 9c +SKIP_ARM=false # leave the sync gates as they are — see Step 1d +SKIP_APPDATA_PROVISION=false # skip the pre-container auth appdata sync — see Step 1e FILTERED_ARGS=() for arg in "$@"; do @@ -217,6 +222,7 @@ for arg in "$@"; do --skip-arr-sync) SKIP_ARR_SYNC=true ;; --skip-webhook-setup) SKIP_WEBHOOK_SETUP=true ;; --skip-media-seed) SKIP_MEDIA_SEED=true ;; + --skip-appdata-provision) SKIP_APPDATA_PROVISION=true ;; --skip-webhook-listener) SKIP_WEBHOOK_LISTENER=true ;; --phase1-only) PHASE1_ONLY=true ;; --phase2-only) PHASE2_ONLY=true; SKIP_SSH=true ;; @@ -615,7 +621,7 @@ if [[ "$PHASE1_ONLY" == true ]]; then fi # ── Phase 1 — Partner conf cache ────────────────────────────────────────────────────────── - # CONF_SYNC_ENABLED is armed here rather than waiting for Step 9c. It is the safe one of the + # CONF_SYNC_ENABLED is armed here rather than waiting for Step 1d. It is the safe one of the # three gates — it moves no data, it copies each side's host*.conf into the other's tmpfs so # partner vars resolve — and the moment SSH works is the moment that should start. Leaving it # until Phase 2 meant conf_sync.sh, which sources the conf fresh in its own process, exited @@ -745,6 +751,102 @@ else bash "$SCRIPT_DIR/share_setup.sh" fi +# ── Step 1d: Sync gates ─────────────────────────────────────────────────────────────────────── +# This was Step 9c, after the containers were already deployed. It has to run here instead, +# because Step 1e below is itself an rsync and Tier 1 stops every rsync — arming afterwards +# meant the provisioning sync exited cleanly having moved nothing. +# +# master.conf.template ships a fresh node inert. A successful Phase 1 is what makes Tier 1, +# CONF_SYNC and ARR_SYNC true; without this the defaults were a one-way door and somebody had +# to remember to hand-edit three toggles on the machine where forgetting looks exactly like +# everything working. +# +# Tier 2 is now set false rather than "left as configured". Tier 1 opening is what makes the +# scheduled jobs live, and a node that was onboarded ten seconds ago is not a node anyone has +# checked yet — free space, share layout, what the partner actually holds. Arriving connected +# and idle is the useful state; the operator arms each orchestrator when they mean to. +# MEDIA_SEED_ENABLED is not touched here at all — see Step 13. +# +# Owner only — the mirror receives these values in the Step 10 push rather than deciding. +ARM_OK=true +_VV_ARM_ON=(RSYNC_ENABLED CONF_SYNC_ENABLED ARR_SYNC_ENABLED) +_VV_ARM_OFF=(CRITICAL_RSYNC_ENABLED INTERMEDIATE_RSYNC_ENABLED DAILY_RSYNC_ENABLED + WEEKLY_RSYNC_ENABLED MONTHLY_RSYNC_ENABLED FALLBACK_RSYNC_ENABLED) +echo "" +echo "━━━ $ICON_GEAR Step 1d — Sync Gates ━━━" + +if [[ "$SKIP_ARM" == true ]]; then + echo " --no-arm — leaving the sync gates as they are" + warn " Step 1e needs Tier 1 open; with it closed the auth appdata will not be provisioned" +elif [[ "$AM_OWNER" != true ]]; then + echo " mirror — the owner's push decides these" +elif [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would set ${_VV_ARM_ON[*]} true and ${_VV_ARM_OFF[*]} false" +else + _master_conf="$SCRIPTS_ROOT/Configurations/master.conf" + if [[ ! -f "$_master_conf" ]]; then + warn "master.conf not found at $_master_conf — gates left closed" + ARM_OK=false + else + cp -a "$_master_conf" "${_master_conf}.bak-arm-$(date +%Y%m%d-%H%M%S)" + for _gate in "${_VV_ARM_ON[@]}"; do set_conf_bool "$_gate" "true" "$_master_conf" || ARM_OK=false; done + for _gate in "${_VV_ARM_OFF[@]}"; do set_conf_bool "$_gate" "false" "$_master_conf" || ARM_OK=false; done + unset _gate + echo " Tier 1 open: ${_VV_ARM_ON[*]}" + echo " Tier 2 off: every scheduled rsync — arm them yourself when you are ready" + fi + unset _master_conf +fi + +# ── Step 1e: Provision the auth stack's appdata ─────────────────────────────────────────────── +# The one rsync an onboard actually needs, and it has to land before Step 3 creates the +# containers that read it. Deploying first meant Authelia, Lldap, NPM and both databases came up +# on the mirror against empty directories and initialised themselves from nothing — so the +# mirror's auth stack was a fresh install wearing the owner's container names. +# +# The critical-data profile, not critical-fallback. critical-data stops the auth containers on +# both sides for the duration, which costs the owner its auth for about a minute at 588 MB, and +# buys a consistent copy. critical-fallback is the dirty variant that keeps auth running, and a +# dirty copy is exactly what broke Redis and MariaDB on the mirror before: the manifest and the +# binlog index were copied without the files they name, and MariaDB hides that — the container +# reads Up while mysqld_safe restarts the dead database every few seconds. +# +# Media is not seeded here or anywhere in Phase 2. This share is 588 MB and the containers do +# not start correctly without it; a media library is terabytes and nothing waits on it. +echo "" +echo "━━━ $ICON_SYNC Step 1e — Auth Appdata Provision ━━━" + +APPDATA_PROVISION_OK=false +if [[ "$SKIP_APPDATA_PROVISION" == true ]]; then + warn "Skipping (--skip-appdata-provision) — the mirror's auth stack will start empty" +elif [[ "${#PARTNERSHIP_PROVISION_SHARES[@]}" -eq 0 ]]; then + warn "PARTNERSHIP_PROVISION_SHARES is empty — nothing to provision" + warn " Set it in host${MY_ID: -1}.conf, or the mirror's auth stack starts from nothing" +elif [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would sync ${PARTNERSHIP_PROVISION_SHARES[*]} to $MIRROR" + APPDATA_PROVISION_OK=true +else + _prov_rc=0 + for _share in "${PARTNERSHIP_PROVISION_SHARES[@]}"; do + echo " Provisioning: $_share" + _prov_flags=() + [[ "$ENABLE_LOGGING" == true ]] && _prov_flags+=(--log) + if ! bash "$SCRIPTS_ROOT/Rsync/rsync.sh" "$_share" "${_prov_flags[@]}"; then + warn " Failed: $_share" + _prov_rc=1 + fi + done + unset _share _prov_flags + if [[ "$_prov_rc" -eq 0 ]]; then + echo "Auth appdata provisioned to $MIRROR ✅" + APPDATA_PROVISION_OK=true + else + warn "Auth appdata NOT fully provisioned — the containers below will start against" + warn " whatever is already on $MIRROR, which on a fresh node is nothing" + fi + unset _prov_rc +fi + # ── Step 2: Stop mirror's existing auth stack ───────────────────────────────────────────────── echo "" echo "━━━ Step 2 — Stop Mirror Auth Stack ━━━" @@ -881,55 +983,6 @@ else fi unset _webhook_script -# ── Step 9c: Arm the sync gates ─────────────────────────────────────────────────────────────── -# master.conf.template ships a fresh node inert — RSYNC_ENABLED, CONF_SYNC_ENABLED and -# ARR_SYNC_ENABLED all false — because a node that has just been seeded has empty shares and no -# verified partner, and those two facts are what make unattended syncing safe to do. -# -# A successful onboard is the event that makes them true. Without this step the defaults were a -# one-way door: the node stayed inert for ever and somebody had to remember to hand-edit three -# toggles, on the machine where forgetting looks exactly like everything working. -# -# Ahead of the media seed, which it used to follow. The seed calls Rsync/rsync.sh, and Tier 1 -# stops *every* rsync — so with the gates still closed all of it exited cleanly having moved -# nothing, and, because a clean exit is exit 0, the seed counted each share as a success and -# reported "14/14 shares ✅" over an empty mirror. -# -# Still ahead of Step 10 on purpose. The push carries the owner's master.conf to every listed -# host, so arming before it means both sides come up agreeing about whether sync is on; arming -# after the push would leave the mirror a version behind until the next conf save. -# -# Owner only — the mirror receives these values in the push rather than deciding for itself. -ARM_OK=true -echo "" -echo "━━━ $ICON_GEAR Step 9c — Arm Sync Gates ━━━" - -if [[ "$ONBOARD_OK" == false ]]; then - warn "Skipping — onboard did not complete, leaving the gates closed" - ARM_OK=false -elif [[ "$SKIP_ARM" == true ]]; then - echo " --no-arm — leaving the sync gates as they are" -elif [[ "$AM_OWNER" != true ]]; then - echo " mirror — the owner's push decides these" -elif [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would set RSYNC_ENABLED, CONF_SYNC_ENABLED, ARR_SYNC_ENABLED to true" -else - _master_conf="$SCRIPTS_ROOT/Configurations/master.conf" - if [[ ! -f "$_master_conf" ]]; then - warn "master.conf not found at $_master_conf — gates left closed" - ARM_OK=false - else - cp -a "$_master_conf" "${_master_conf}.bak-arm-$(date +%Y%m%d-%H%M%S)" - for _gate in RSYNC_ENABLED CONF_SYNC_ENABLED ARR_SYNC_ENABLED; do - set_conf_bool "$_gate" "true" "$_master_conf" || ARM_OK=false - done - # Tier 1 is now open, so say what that actually turned on rather than leaving it implied. - echo " rsync tier 2 gates were left as configured — check them before the first run" - unset _gate - fi - unset _master_conf -fi - # ── Step 9e: Start webhook listener on mirror ───────────────────────────────────────────────── # Listener is in ARRAY_START_SCRIPTS so it starts on next boot, but the mirror's array is # already running — kick it now so events are captured immediately after onboard. @@ -1089,7 +1142,7 @@ _seed_job="Rsync/media_seed.sh" _seed_script="$SCRIPTS_ROOT/Rsync/media_seed.sh" _runner="$SCRIPTS_ROOT/Plugin/$PLATFORM/run_job.sh" -# Read from disk, not from the sourced value: Step 9c rewrote master.conf a few steps ago. +# Read from disk, not from the sourced value: Step 1d rewrote master.conf a few steps ago. # Unset reads as on — the toggle postdates the seed, so a conf that has not been through a # conf_upgrade must keep the behaviour it had. media_seed.sh checks this again itself; the # check here exists so the summary can say "disabled" instead of dispatching a job whose only @@ -1155,6 +1208,8 @@ _skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; } echo " Step 1 — SSH keys: $(_skip "$SKIP_SSH" "$STEP_SSH_OK")" echo " Step 1b — Docker network: $(_ok "$STEP_NETWORK_OK")" +echo " Step 1d — Sync gates: $( [[ "$SKIP_ARM" == true ]] && echo "skipped (--no-arm)" || { [[ "$AM_OWNER" != true ]] && echo "mirror — set by owner" || _ok "$ARM_OK"; } )" +echo " Step 1e — Auth appdata: $( [[ "$SKIP_APPDATA_PROVISION" == true ]] && echo "skipped" || _ok "$APPDATA_PROVISION_OK" )" echo " Step 2 — Stop auth: $(_skip "$SKIP_AUTH_STACK" "$STEP_STOP_AUTH_OK")" echo " Step 3 — Auth stack: $( [[ "$SKIP_AUTH_STACK" == true ]] && echo "skipped" || echo "${AUTH_DEPLOYED} deployed, ${AUTH_FAILED} failed" )" echo " Step 4 — Stop arr: $(_skip "$SKIP_ARR_STACK" "$STEP_STOP_ARR_OK")" @@ -1164,7 +1219,6 @@ echo " Step 7 — Services stack: $( [[ "$SKIP_SERVICES_STACK" == true ]] && echo " Step 8 — Onboard: $(_ok "$ONBOARD_OK")" echo " Step 9 — Arr bootstrap: $( [[ "$SKIP_ARR_SYNC" == true || "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARR_SYNC_OK")" )" echo " Step 9b — Webhook setup: $(_skip "$SKIP_WEBHOOK_SETUP" "$WEBHOOK_SETUP_OK")" -echo " Step 9c — Arm sync gates: $( [[ "$SKIP_ARM" == true ]] && echo "skipped (--no-arm)" || { [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$ARM_OK")"; } )" echo " Step 9e — Webhook listener: $(_skip "$SKIP_WEBHOOK_LISTENER" "$WEBHOOK_LISTENER_OK")" echo " Step 10 — Conf push: $( [[ "$ONBOARD_OK" == false ]] && echo "skipped" || echo "$(_ok "$MASTER_PUSH_OK")" )" echo " Step 11 — Discovery: $( [[ "$POPULATE_OK" == skipped ]] && echo "skipped (unreachable)" || _ok "$POPULATE_OK" )" diff --git a/common.sh b/common.sh index 2485ff8..f9ef3f0 100755 --- a/common.sh +++ b/common.sh @@ -501,6 +501,7 @@ is_in_list() { # # Also sets aliases for all host-specific arrays so scripts use unprefixed names: # DAILY_SYNC_SHARES ← HOST*_DAILY_SYNC_SHARES +# PARTNERSHIP_PROVISION_SHARES ← HOST*_PARTNERSHIP_PROVISION_SHARES # INTERMEDIATE_SYNC_SHARES ← HOST*_INTERMEDIATE_SYNC_SHARES # WEEKLY_SYNC_SHARES ← HOST*_WEEKLY_SYNC_SHARES # CRITICAL_SYNC_SHARES ← HOST*_CRITICAL_SYNC_SHARES @@ -729,6 +730,7 @@ detect_hosts() { _alias_array "INTERMEDIATE_SYNC_SHARES" _alias_array "WEEKLY_SYNC_SHARES" _alias_array "CRITICAL_SYNC_SHARES" + _alias_array "PARTNERSHIP_PROVISION_SHARES" _alias_array "BACKUP_VERIFY_SHARES" _alias_array "DAILY_RESTART_CONTAINERS" _alias_array "WEEKLY_RESTART_CONTAINERS"