From fda853eed8d6504764642cb59f96e6a2fde62940 Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Sat, 23 May 2026 09:24:34 -0400 Subject: [PATCH] fix 9 bugs in partnership offboard flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit partnership_manager.sh: - fix auto-offboard threshold: × 96 → × 48 (was triggering at 2× configured days) - fix do_final_sync(): use CRITICAL_SYNC_SHARES if configured; fall back to hardcoded paths - replace inline --check offboard blocks with background partnership_offboard.sh calls: • owner path was missing remote container cleanup (steps 8+9) and blocklist • mirror path was missing blocklist, SSH revocation, CRITICAL_RSYNC_ENABLED=false • local grace_seconds outside a function (bash error) eliminated with the block removal • 6hr Tailscale grace sleep was blocking the cron for the full grace window partnership_offboard.sh: - reorder owner path: state write moved from step 5 to step 10 (last) so re-running after a mid-offboard crash restarts from scratch instead of exiting as INACTIVE; state is now written before the Tailscale grace sleep so --check stops re-triggering - add FolderView3 remote folder cleanup in step 7 (mirror's fallback folder persisted) - track step 1+2 outcome with STEP_STOP_OK / STEP_SYNC_OK; summary now reflects actual result instead of hardcoding ✅ regardless of dry-run or failure - update header step list to match new step order --- Partnership/partnership_manager.sh | 91 +++++++---------- Partnership/partnership_offboard.sh | 148 ++++++++++++++++------------ 2 files changed, 121 insertions(+), 118 deletions(-) diff --git a/Partnership/partnership_manager.sh b/Partnership/partnership_manager.sh index bec32f5..a33a13f 100644 --- a/Partnership/partnership_manager.sh +++ b/Partnership/partnership_manager.sh @@ -1123,18 +1123,30 @@ check_both_healthy() { } do_final_sync() { - log "Running final Critical-Data and Emby sync..." + log "Running final critical sync..." if [[ "$DRY_RUN" == false ]]; then - # Use CRITICAL_SYNC_SHARES from HOST* conf if available - # Falls back to known paths — these are the critical ones - bash "$SCRIPT_DIR/../Rsync/rsync.sh" \ - "/mnt/user/appdata-Fallback/Critical-Data" \ - --profile=critical-fallback --log - bash "$SCRIPT_DIR/../Rsync/rsync.sh" \ - "/mnt/user/Media_Server/Emby" \ - --profile=emby-fallback --log + if [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then + for _share in "${CRITICAL_SYNC_SHARES[@]}"; do + [[ -z "$_share" ]] && continue + local _path="${_share%%|*}" + local _profile="${_share##*|}" + if [[ "$_path" == "$_profile" ]]; then + bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" --log + else + bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" \ + --profile="$_profile" --log + fi + done + else + bash "$SCRIPT_DIR/../Rsync/rsync.sh" \ + "/mnt/user/appdata-Fallback/Critical-Data" \ + --profile=critical-fallback --log + bash "$SCRIPT_DIR/../Rsync/rsync.sh" \ + "/mnt/user/Media_Server/Emby" \ + --profile=emby-fallback --log + fi else - warn "DRY RUN — would run final Critical-Data and Emby sync" + warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]:-hardcoded} shares)" fi warn "Final sync complete — mirror has current state ✅" } @@ -1317,7 +1329,7 @@ if [[ "$MODE" == "check" ]]; then echo "$OFFLINE_COUNT" > "$OFFLINE_COUNTER" # Auto-offboard threshold: threshold_days × 48 intervals/day (every 30min) - THRESHOLD_INTERVALS=$(( ${PARTNERSHIP_OFFLINE_THRESHOLD:-30} * 96 )) + THRESHOLD_INTERVALS=$(( ${PARTNERSHIP_OFFLINE_THRESHOLD:-30} * 48 )) if [[ "$OFFLINE_COUNT" -ge "$THRESHOLD_INTERVALS" ]]; then warn "Remote offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days — triggering auto-offboard" notify "Partnership auto-offboard on $(hostname) — $REMOTE_SERVER_NAME offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days" \ @@ -1360,56 +1372,21 @@ if [[ "$MODE" == "check" ]]; then exit 0 fi - # Remote requested offboard + # Remote requested offboard — run full offboard in background so cron is not blocked if [[ "$REMOTE_STATE" == "INACTIVE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then warn "Partnership check — $REMOTE_SERVER_NAME requested offboard" - if [[ "$AM_OWNER" == true ]]; then - warn "Owner finalising offboard request from mirror..." - do_final_sync - - # Remove owner's fallback containers, restart own stack - cleanup_partner_containers "$(derive_partner_folder_name "$MIRROR")" - start_own_stack - - NOW=$(date '+%Y-%m-%d %H:%M:%S') - write_state_file "$LOCAL_STATE_FILE" \ - "INACTIVE" "" "$NOW" "$REMOTE_SERVER_NAME" "mirror-requested" - - if [[ "$DRY_RUN" == false ]]; then - update_master_conf "CRITICAL_RSYNC_ENABLED" "false" - fi - - # SSH key revocation — mutual, both directions - # REMOTE_IP already resolved above — this is the last SSH operation - SSH_REVOKE_REMOTE_OK=false - SSH_REVOKE_LOCAL_OK=false - do_ssh_key_revocation "$REMOTE_IP" - - if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then - local grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 )) - warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace period..." - # Grace sleep — interruptible via SIGTERM trap - if [[ "$DRY_RUN" == false ]]; then - trap 'warn "Partnership check interrupted during grace sleep"; exit 0' SIGTERM SIGINT - sleep "$grace_seconds" - trap - SIGTERM SIGINT - fi - remove_tailscale_device "$MIRROR" - fi - notify "Partnership offboard finalised — $MIRROR requested separation" \ - "Partnership" "normal" + warn "Owner finalising offboard — spawning partnership_offboard.sh in background" + nohup bash "$SCRIPT_DIR/partnership_offboard.sh" \ + --reason=mirror-requested \ + >> /var/log/partnership_offboard.log 2>&1 & + warn "Offboard PID $! running — tail /var/log/partnership_offboard.log to follow" else - # Mirror sees owner is INACTIVE — clean up own side - warn "Owner has offboarded — cleaning up mirror side..." - reconfigure_local_webuis "localhost" - cleanup_partner_containers "$(derive_partner_folder_name "$OWNER")" - start_own_stack - NOW=$(date '+%Y-%m-%d %H:%M:%S') - write_state_file "$LOCAL_STATE_FILE" \ - "INACTIVE" "" "$NOW" "$OWNER" "owner-offboarded" - notify "Partnership ended — $OWNER offboarded. Auth WebUIs → localhost." \ - "Partnership" "normal" + warn "Owner offboarded — spawning mirror-side cleanup in background" + nohup bash "$SCRIPT_DIR/partnership_offboard.sh" \ + --reason=owner-offboarded \ + >> /var/log/partnership_offboard.log 2>&1 & + warn "Offboard PID $! running — tail /var/log/partnership_offboard.log to follow" fi exit 0 fi diff --git a/Partnership/partnership_offboard.sh b/Partnership/partnership_offboard.sh index 7351aad..e6d832a 100755 --- a/Partnership/partnership_offboard.sh +++ b/Partnership/partnership_offboard.sh @@ -18,12 +18,13 @@ # Step 2: Final sync — mirror leaves with current Critical-Data state # Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost # Step 4: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf -# Step 5: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted -# Step 6: Local cleanup — remove fallback coverage containers + appdata -# Step 7: Restart own stack — bring up owner's own parked containers -# Step 8: Remote cleanup — remove auth/arr stack + fallback containers from mirror -# Step 9: Restart mirror — bring up mirror's own parked containers -# Step 10: Revocation — Emby admin, SSH keys, Tailscale device +# Step 5: Local cleanup — remove fallback coverage containers + appdata +# Step 6: Restart own stack — bring up owner's own parked containers +# Step 7: Remote cleanup — remove auth/arr stack + fallback containers from mirror +# Step 8: Restart mirror — bring up mirror's own parked containers +# Step 9: Revocation — Emby admin, SSH keys +# Step 10: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted +# Tailscale — grace window then device removal (after state written) # # MIRROR PATH (8 steps) # Step 1: Stop rsync — halt any running sync @@ -499,6 +500,8 @@ if [[ "$DRY_RUN" == false ]]; then fi WEBUI_FAILURES=0 +STEP_STOP_OK=true +STEP_SYNC_OK=true SSH_REVOKE_REMOTE_OK=false SSH_REVOKE_LOCAL_OK=false @@ -507,7 +510,7 @@ echo "" echo "━━━ $ICON_STOP Step 1/10 — Stop Rsync ━━━" if [[ "$DRY_RUN" == false ]]; then - bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true + bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || STEP_STOP_OK=false log "Rsync stopped ✅" else warn "DRY RUN — would stop rsync" @@ -517,7 +520,7 @@ fi echo "" echo "━━━ $ICON_SYNC Step 2/10 — Final Sync ━━━" -do_final_sync +do_final_sync || STEP_SYNC_OK=false # ── Step 3: Reconfigure mirror WebUIs → localhost ───────────────────────────────────────────── echo "" @@ -552,9 +555,72 @@ else warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false" fi -# ── Step 5: Write state, push to mirror, blocklist ──────────────────────────────────────────── +# ── Step 5: Local container cleanup ─────────────────────────────────────────────────────────── echo "" -echo "━━━ $ICON_GEAR Step 5/10 — Write State ━━━" +echo "━━━ $ICON_CONTAINERS Step 5/10 — Local Container Cleanup ━━━" + +PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR") +cleanup_partner_containers "$PARTNER_FOLDER_NAME" + +# ── Step 6: Restart own stack ───────────────────────────────────────────────────────────────── +start_own_stack + +# ── Step 7: Remote container cleanup ────────────────────────────────────────────────────────── +echo "" +echo "━━━ $ICON_CONTAINERS Step 7/10 — Remote Container Cleanup ━━━" + +if [[ "$MIRROR_REACHABLE" == true ]]; then + # Remove auth/arr stack containers deployed during onboard (by config array) + cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY" + # Remove fallback coverage containers (by *-owner_short naming pattern) + cleanup_owner_containers_on_mirror "$MIRROR_IP" + # Remove mirror's FolderView3 fallback folder (owner's containers were hosted there) + if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then + OWNER_FOLDER_ON_MIRROR=$(derive_partner_folder_name "$OWNER") + log "Removing FolderView3 folder '$OWNER_FOLDER_ON_MIRROR' from $MIRROR..." + if [[ "$DRY_RUN" == false ]]; then + timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \ + -o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \ + "fv3='/boot/config/plugins/folder.view3/docker.json' + [[ -f \"\$fv3\" ]] && command -v jq >/dev/null 2>&1 && \ + jq --arg n '$OWNER_FOLDER_ON_MIRROR' \ + 'with_entries(select(.value.name != \$n))' \ + \"\$fv3\" > \"\${fv3}.tmp\" && \ + mv \"\${fv3}.tmp\" \"\$fv3\" && echo removed" 2>/dev/null | \ + grep -q removed && \ + log "FolderView3 '$OWNER_FOLDER_ON_MIRROR' removed from $MIRROR ✅" || \ + warn "FolderView3 folder not found on $MIRROR or jq unavailable — skipping" + else + warn "DRY RUN — would remove FolderView3 folder '$OWNER_FOLDER_ON_MIRROR' from $MIRROR" + fi + fi +else + warn "$MIRROR unreachable — remote container cleanup skipped" + warn "Run 'partnership_offboard.sh' on $MIRROR to clean up manually" +fi + +# ── Step 8: Restart mirror's own stack ──────────────────────────────────────────────────────── +echo "" +echo "━━━ $ICON_START Step 8/10 — Restart Mirror Stack ━━━" + +[[ "$MIRROR_REACHABLE" == true ]] && start_mirror_own_stack "$MIRROR_IP" + +# ── Step 9: Revocation (Emby + SSH) ────────────────────────────────────────────────────────── +echo "" +echo "━━━ $ICON_SHIELD Step 9/10 — Revocation ━━━" + +# Emby admin — before SSH key revocation while Emby still reachable +[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP" + +# SSH key revocation — mutual, both directions; must run while Tailscale still active +do_ssh_key_revocation "${MIRROR_IP:-}" + +# ── Step 10: Write state, push to mirror, blocklist ─────────────────────────────────────────── +# State is written after container cleanup and SSH revocation so that: +# • Re-running after a crash between steps 5–9 restarts from scratch (no early-exit on INACTIVE) +# • --check sees INACTIVE during the Tailscale grace sleep and does not re-trigger offboard +echo "" +echo "━━━ $ICON_GEAR Step 10/10 — Write State ━━━" NOW=$(date '+%Y-%m-%d %H:%M:%S') @@ -569,47 +635,7 @@ else warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote" fi -# ── Step 6: Local container cleanup ─────────────────────────────────────────────────────────── -echo "" -echo "━━━ $ICON_CONTAINERS Step 6/10 — Local Container Cleanup ━━━" - -PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR") -cleanup_partner_containers "$PARTNER_FOLDER_NAME" - -# ── Step 7: Restart own stack ───────────────────────────────────────────────────────────────── -start_own_stack - -# ── Step 8: Remote container cleanup ────────────────────────────────────────────────────────── -echo "" -echo "━━━ $ICON_CONTAINERS Step 8/10 — Remote Container Cleanup ━━━" - -if [[ "$MIRROR_REACHABLE" == true ]]; then - # Remove auth/arr stack containers deployed during onboard (by config array) - cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY" - # Remove fallback coverage containers (by *-owner_short naming pattern) - cleanup_owner_containers_on_mirror "$MIRROR_IP" -else - warn "$MIRROR unreachable — remote container cleanup skipped" - warn "Run 'partnership_manager.sh --offboard' on $MIRROR to clean up manually" -fi - -# ── Step 9: Restart mirror's own stack ──────────────────────────────────────────────────────── -echo "" -echo "━━━ $ICON_START Step 9/10 — Restart Mirror Stack ━━━" - -[[ "$MIRROR_REACHABLE" == true ]] && start_mirror_own_stack "$MIRROR_IP" - -# ── Step 10: Revocation ─────────────────────────────────────────────────────────────────────── -echo "" -echo "━━━ $ICON_SHIELD Step 10/10 — Revocation ━━━" - -# Emby admin — before SSH key revocation while Emby still reachable -[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP" - -# SSH key revocation — mutual, both directions; must run while Tailscale still active -do_ssh_key_revocation "${MIRROR_IP:-}" - -# Tailscale removal — after SSH revocation, guard with grace window +# Tailscale removal — after state written so --check does not re-trigger offboard during grace sleep if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then echo "" echo "━━━ $ICON_NET Tailscale Separation ━━━" @@ -658,20 +684,20 @@ _revoke_status() { fi } -echo " Step 1 — Stop rsync: ✅" -echo " Step 2 — Final sync: ✅" +echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_OK")" +echo " Step 2 — Final sync: $(_ok "$STEP_SYNC_OK")" echo " Step 3 — WebUI failures: $WEBUI_FAILURES" -echo " Step 4 — Disable sync: ✅" -echo " Step 5 — State: INACTIVE ✅" -echo " Step 6 — Local cleanup: ✅" -echo " Step 7 — Own stack: started" -echo " Step 8 — Remote cleanup: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "✅" || echo "skipped (unreachable)" )" -echo " Step 9 — Mirror stack: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "started" || echo "skipped (unreachable)" )" -echo " Step 10 — Keys revoked: $(_revoke_status)" +echo " Step 4 — Disable sync: ✅" +echo " Step 5 — Local cleanup: ✅" +echo " Step 6 — Own stack: started" +echo " Step 7 — Remote cleanup: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "✅" || echo "skipped (unreachable)" )" +echo " Step 8 — Mirror stack: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "started" || echo "skipped (unreachable)" )" +echo " Step 9 — Keys revoked: $(_revoke_status)" +echo " Step 10 — State: INACTIVE ✅" echo "" echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅" [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]] && \ - echo " FolderView3: ${PARTNER_FOLDER_NAME:-} cleaned ✅" + echo " FolderView3: ${PARTNER_FOLDER_NAME:-} (local) + mirror remote cleaned ✅" [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \ echo " Tailscale: $MIRROR removed ✅" echo ""