fix 9 bugs in partnership offboard flow
partnership_manager.sh:
- fix auto-offboard threshold: × 96 → × 48 (was triggering at 2× configured days)
- fix do_final_sync(): use CRITICAL_SYNC_SHARES if configured; fall back to hardcoded paths
- replace inline --check offboard blocks with background partnership_offboard.sh calls:
• owner path was missing remote container cleanup (steps 8+9) and blocklist
• mirror path was missing blocklist, SSH revocation, CRITICAL_RSYNC_ENABLED=false
• local grace_seconds outside a function (bash error) eliminated with the block removal
• 6hr Tailscale grace sleep was blocking the cron for the full grace window
partnership_offboard.sh:
- reorder owner path: state write moved from step 5 to step 10 (last) so re-running
after a mid-offboard crash restarts from scratch instead of exiting as INACTIVE;
state is now written before the Tailscale grace sleep so --check stops re-triggering
- add FolderView3 remote folder cleanup in step 7 (mirror's fallback folder persisted)
- track step 1+2 outcome with STEP_STOP_OK / STEP_SYNC_OK; summary now reflects actual
result instead of hardcoding ✅ regardless of dry-run or failure
- update header step list to match new step order
This commit is contained in:
Gmer4Lfe
@@ -1123,18 +1123,30 @@ check_both_healthy() {
|
||||
}
|
||||
|
||||
do_final_sync() {
|
||||
log "Running final Critical-Data and Emby sync..."
|
||||
log "Running final critical sync..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
# Use CRITICAL_SYNC_SHARES from HOST* conf if available
|
||||
# Falls back to known paths — these are the critical ones
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" \
|
||||
--profile=critical-fallback --log
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
|
||||
"/mnt/user/Media_Server/Emby" \
|
||||
--profile=emby-fallback --log
|
||||
if [[ "${#CRITICAL_SYNC_SHARES[@]}" -gt 0 ]]; then
|
||||
for _share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
||||
[[ -z "$_share" ]] && continue
|
||||
local _path="${_share%%|*}"
|
||||
local _profile="${_share##*|}"
|
||||
if [[ "$_path" == "$_profile" ]]; then
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" --log
|
||||
else
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$_path" \
|
||||
--profile="$_profile" --log
|
||||
fi
|
||||
done
|
||||
else
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
|
||||
"/mnt/user/appdata-Fallback/Critical-Data" \
|
||||
--profile=critical-fallback --log
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
|
||||
"/mnt/user/Media_Server/Emby" \
|
||||
--profile=emby-fallback --log
|
||||
fi
|
||||
else
|
||||
warn "DRY RUN — would run final Critical-Data and Emby sync"
|
||||
warn "DRY RUN — would run final critical sync (${#CRITICAL_SYNC_SHARES[@]:-hardcoded} shares)"
|
||||
fi
|
||||
warn "Final sync complete — mirror has current state ✅"
|
||||
}
|
||||
@@ -1317,7 +1329,7 @@ if [[ "$MODE" == "check" ]]; then
|
||||
echo "$OFFLINE_COUNT" > "$OFFLINE_COUNTER"
|
||||
|
||||
# Auto-offboard threshold: threshold_days × 48 intervals/day (every 30min)
|
||||
THRESHOLD_INTERVALS=$(( ${PARTNERSHIP_OFFLINE_THRESHOLD:-30} * 96 ))
|
||||
THRESHOLD_INTERVALS=$(( ${PARTNERSHIP_OFFLINE_THRESHOLD:-30} * 48 ))
|
||||
if [[ "$OFFLINE_COUNT" -ge "$THRESHOLD_INTERVALS" ]]; then
|
||||
warn "Remote offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days — triggering auto-offboard"
|
||||
notify "Partnership auto-offboard on $(hostname) — $REMOTE_SERVER_NAME offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days" \
|
||||
@@ -1360,56 +1372,21 @@ if [[ "$MODE" == "check" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Remote requested offboard
|
||||
# Remote requested offboard — run full offboard in background so cron is not blocked
|
||||
if [[ "$REMOTE_STATE" == "INACTIVE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then
|
||||
warn "Partnership check — $REMOTE_SERVER_NAME requested offboard"
|
||||
|
||||
if [[ "$AM_OWNER" == true ]]; then
|
||||
warn "Owner finalising offboard request from mirror..."
|
||||
do_final_sync
|
||||
|
||||
# Remove owner's fallback containers, restart own stack
|
||||
cleanup_partner_containers "$(derive_partner_folder_name "$MIRROR")"
|
||||
start_own_stack
|
||||
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$REMOTE_SERVER_NAME" "mirror-requested"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false"
|
||||
fi
|
||||
|
||||
# SSH key revocation — mutual, both directions
|
||||
# REMOTE_IP already resolved above — this is the last SSH operation
|
||||
SSH_REVOKE_REMOTE_OK=false
|
||||
SSH_REVOKE_LOCAL_OK=false
|
||||
do_ssh_key_revocation "$REMOTE_IP"
|
||||
|
||||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||||
local grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 ))
|
||||
warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace period..."
|
||||
# Grace sleep — interruptible via SIGTERM trap
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
trap 'warn "Partnership check interrupted during grace sleep"; exit 0' SIGTERM SIGINT
|
||||
sleep "$grace_seconds"
|
||||
trap - SIGTERM SIGINT
|
||||
fi
|
||||
remove_tailscale_device "$MIRROR"
|
||||
fi
|
||||
notify "Partnership offboard finalised — $MIRROR requested separation" \
|
||||
"Partnership" "normal"
|
||||
warn "Owner finalising offboard — spawning partnership_offboard.sh in background"
|
||||
nohup bash "$SCRIPT_DIR/partnership_offboard.sh" \
|
||||
--reason=mirror-requested \
|
||||
>> /var/log/partnership_offboard.log 2>&1 &
|
||||
warn "Offboard PID $! running — tail /var/log/partnership_offboard.log to follow"
|
||||
else
|
||||
# Mirror sees owner is INACTIVE — clean up own side
|
||||
warn "Owner has offboarded — cleaning up mirror side..."
|
||||
reconfigure_local_webuis "localhost"
|
||||
cleanup_partner_containers "$(derive_partner_folder_name "$OWNER")"
|
||||
start_own_stack
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$OWNER" "owner-offboarded"
|
||||
notify "Partnership ended — $OWNER offboarded. Auth WebUIs → localhost." \
|
||||
"Partnership" "normal"
|
||||
warn "Owner offboarded — spawning mirror-side cleanup in background"
|
||||
nohup bash "$SCRIPT_DIR/partnership_offboard.sh" \
|
||||
--reason=owner-offboarded \
|
||||
>> /var/log/partnership_offboard.log 2>&1 &
|
||||
warn "Offboard PID $! running — tail /var/log/partnership_offboard.log to follow"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -18,12 +18,13 @@
|
||||
# Step 2: Final sync — mirror leaves with current Critical-Data state
|
||||
# Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost
|
||||
# Step 4: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf
|
||||
# Step 5: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted
|
||||
# Step 6: Local cleanup — remove fallback coverage containers + appdata
|
||||
# Step 7: Restart own stack — bring up owner's own parked containers
|
||||
# Step 8: Remote cleanup — remove auth/arr stack + fallback containers from mirror
|
||||
# Step 9: Restart mirror — bring up mirror's own parked containers
|
||||
# Step 10: Revocation — Emby admin, SSH keys, Tailscale device
|
||||
# Step 5: Local cleanup — remove fallback coverage containers + appdata
|
||||
# Step 6: Restart own stack — bring up owner's own parked containers
|
||||
# Step 7: Remote cleanup — remove auth/arr stack + fallback containers from mirror
|
||||
# Step 8: Restart mirror — bring up mirror's own parked containers
|
||||
# Step 9: Revocation — Emby admin, SSH keys
|
||||
# Step 10: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted
|
||||
# Tailscale — grace window then device removal (after state written)
|
||||
#
|
||||
# MIRROR PATH (8 steps)
|
||||
# Step 1: Stop rsync — halt any running sync
|
||||
@@ -499,6 +500,8 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
fi
|
||||
|
||||
WEBUI_FAILURES=0
|
||||
STEP_STOP_OK=true
|
||||
STEP_SYNC_OK=true
|
||||
SSH_REVOKE_REMOTE_OK=false
|
||||
SSH_REVOKE_LOCAL_OK=false
|
||||
|
||||
@@ -507,7 +510,7 @@ echo ""
|
||||
echo "━━━ $ICON_STOP Step 1/10 — Stop Rsync ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true
|
||||
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || STEP_STOP_OK=false
|
||||
log "Rsync stopped ✅"
|
||||
else
|
||||
warn "DRY RUN — would stop rsync"
|
||||
@@ -517,7 +520,7 @@ fi
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Step 2/10 — Final Sync ━━━"
|
||||
|
||||
do_final_sync
|
||||
do_final_sync || STEP_SYNC_OK=false
|
||||
|
||||
# ── Step 3: Reconfigure mirror WebUIs → localhost ─────────────────────────────────────────────
|
||||
echo ""
|
||||
@@ -552,9 +555,72 @@ else
|
||||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
||||
fi
|
||||
|
||||
# ── Step 5: Write state, push to mirror, blocklist ────────────────────────────────────────────
|
||||
# ── Step 5: Local container cleanup ───────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 5/10 — Write State ━━━"
|
||||
echo "━━━ $ICON_CONTAINERS Step 5/10 — Local Container Cleanup ━━━"
|
||||
|
||||
PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR")
|
||||
cleanup_partner_containers "$PARTNER_FOLDER_NAME"
|
||||
|
||||
# ── Step 6: Restart own stack ─────────────────────────────────────────────────────────────────
|
||||
start_own_stack
|
||||
|
||||
# ── Step 7: Remote container cleanup ──────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 7/10 — Remote Container Cleanup ━━━"
|
||||
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
# Remove auth/arr stack containers deployed during onboard (by config array)
|
||||
cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||||
# Remove fallback coverage containers (by *-owner_short naming pattern)
|
||||
cleanup_owner_containers_on_mirror "$MIRROR_IP"
|
||||
# Remove mirror's FolderView3 fallback folder (owner's containers were hosted there)
|
||||
if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then
|
||||
OWNER_FOLDER_ON_MIRROR=$(derive_partner_folder_name "$OWNER")
|
||||
log "Removing FolderView3 folder '$OWNER_FOLDER_ON_MIRROR' from $MIRROR..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$MIRROR_IP" \
|
||||
"fv3='/boot/config/plugins/folder.view3/docker.json'
|
||||
[[ -f \"\$fv3\" ]] && command -v jq >/dev/null 2>&1 && \
|
||||
jq --arg n '$OWNER_FOLDER_ON_MIRROR' \
|
||||
'with_entries(select(.value.name != \$n))' \
|
||||
\"\$fv3\" > \"\${fv3}.tmp\" && \
|
||||
mv \"\${fv3}.tmp\" \"\$fv3\" && echo removed" 2>/dev/null | \
|
||||
grep -q removed && \
|
||||
log "FolderView3 '$OWNER_FOLDER_ON_MIRROR' removed from $MIRROR ✅" || \
|
||||
warn "FolderView3 folder not found on $MIRROR or jq unavailable — skipping"
|
||||
else
|
||||
warn "DRY RUN — would remove FolderView3 folder '$OWNER_FOLDER_ON_MIRROR' from $MIRROR"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
warn "$MIRROR unreachable — remote container cleanup skipped"
|
||||
warn "Run 'partnership_offboard.sh' on $MIRROR to clean up manually"
|
||||
fi
|
||||
|
||||
# ── Step 8: Restart mirror's own stack ────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Step 8/10 — Restart Mirror Stack ━━━"
|
||||
|
||||
[[ "$MIRROR_REACHABLE" == true ]] && start_mirror_own_stack "$MIRROR_IP"
|
||||
|
||||
# ── Step 9: Revocation (Emby + SSH) ──────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Step 9/10 — Revocation ━━━"
|
||||
|
||||
# Emby admin — before SSH key revocation while Emby still reachable
|
||||
[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP"
|
||||
|
||||
# SSH key revocation — mutual, both directions; must run while Tailscale still active
|
||||
do_ssh_key_revocation "${MIRROR_IP:-}"
|
||||
|
||||
# ── Step 10: Write state, push to mirror, blocklist ───────────────────────────────────────────
|
||||
# State is written after container cleanup and SSH revocation so that:
|
||||
# • Re-running after a crash between steps 5–9 restarts from scratch (no early-exit on INACTIVE)
|
||||
# • --check sees INACTIVE during the Tailscale grace sleep and does not re-trigger offboard
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 10/10 — Write State ━━━"
|
||||
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
@@ -569,47 +635,7 @@ else
|
||||
warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote"
|
||||
fi
|
||||
|
||||
# ── Step 6: Local container cleanup ───────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 6/10 — Local Container Cleanup ━━━"
|
||||
|
||||
PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR")
|
||||
cleanup_partner_containers "$PARTNER_FOLDER_NAME"
|
||||
|
||||
# ── Step 7: Restart own stack ─────────────────────────────────────────────────────────────────
|
||||
start_own_stack
|
||||
|
||||
# ── Step 8: Remote container cleanup ──────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 8/10 — Remote Container Cleanup ━━━"
|
||||
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
# Remove auth/arr stack containers deployed during onboard (by config array)
|
||||
cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||||
# Remove fallback coverage containers (by *-owner_short naming pattern)
|
||||
cleanup_owner_containers_on_mirror "$MIRROR_IP"
|
||||
else
|
||||
warn "$MIRROR unreachable — remote container cleanup skipped"
|
||||
warn "Run 'partnership_manager.sh --offboard' on $MIRROR to clean up manually"
|
||||
fi
|
||||
|
||||
# ── Step 9: Restart mirror's own stack ────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Step 9/10 — Restart Mirror Stack ━━━"
|
||||
|
||||
[[ "$MIRROR_REACHABLE" == true ]] && start_mirror_own_stack "$MIRROR_IP"
|
||||
|
||||
# ── Step 10: Revocation ───────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Step 10/10 — Revocation ━━━"
|
||||
|
||||
# Emby admin — before SSH key revocation while Emby still reachable
|
||||
[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP"
|
||||
|
||||
# SSH key revocation — mutual, both directions; must run while Tailscale still active
|
||||
do_ssh_key_revocation "${MIRROR_IP:-}"
|
||||
|
||||
# Tailscale removal — after SSH revocation, guard with grace window
|
||||
# Tailscale removal — after state written so --check does not re-trigger offboard during grace sleep
|
||||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET Tailscale Separation ━━━"
|
||||
@@ -658,20 +684,20 @@ _revoke_status() {
|
||||
fi
|
||||
}
|
||||
|
||||
echo " Step 1 — Stop rsync: ✅"
|
||||
echo " Step 2 — Final sync: ✅"
|
||||
echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_OK")"
|
||||
echo " Step 2 — Final sync: $(_ok "$STEP_SYNC_OK")"
|
||||
echo " Step 3 — WebUI failures: $WEBUI_FAILURES"
|
||||
echo " Step 4 — Disable sync: ✅"
|
||||
echo " Step 5 — State: INACTIVE ✅"
|
||||
echo " Step 6 — Local cleanup: ✅"
|
||||
echo " Step 7 — Own stack: started"
|
||||
echo " Step 8 — Remote cleanup: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "✅" || echo "skipped (unreachable)" )"
|
||||
echo " Step 9 — Mirror stack: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "started" || echo "skipped (unreachable)" )"
|
||||
echo " Step 10 — Keys revoked: $(_revoke_status)"
|
||||
echo " Step 4 — Disable sync: ✅"
|
||||
echo " Step 5 — Local cleanup: ✅"
|
||||
echo " Step 6 — Own stack: started"
|
||||
echo " Step 7 — Remote cleanup: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "✅" || echo "skipped (unreachable)" )"
|
||||
echo " Step 8 — Mirror stack: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "started" || echo "skipped (unreachable)" )"
|
||||
echo " Step 9 — Keys revoked: $(_revoke_status)"
|
||||
echo " Step 10 — State: INACTIVE ✅"
|
||||
echo ""
|
||||
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
|
||||
[[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]] && \
|
||||
echo " FolderView3: ${PARTNER_FOLDER_NAME:-} cleaned ✅"
|
||||
echo " FolderView3: ${PARTNER_FOLDER_NAME:-} (local) + mirror remote cleaned ✅"
|
||||
[[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \
|
||||
echo " Tailscale: $MIRROR removed ✅"
|
||||
echo ""
|
||||
|
||||
Reference in New Issue
Block a user