feat: slskd reconnect guard in downloaders_reset, mass v2 sync
- downloaders_reset: connection check block before slskd API sections; triggers PUT /api/v0/server reconnect if disconnected, polls 60s, gates Stuck Searches and Dead Transfer Records on SLSKD_CONNECTED - Sync all modified/new/deleted files from v2 refactor across Docker_Essentials, Media, Monitors, Partnership, Rsync, Tools, Transcodes, unRAID_Essentials, common.sh, master confs, and new Manual/README docs
This commit is contained in:
Gmer4Lfe
Executable
+678
@@ -0,0 +1,678 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= Partnership Offboard ===========================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Cleanly ends a partnership. Role is detected automatically — run on either server.
|
||||
# Owner path runs the full sequence including remote cleanup and final sync.
|
||||
# Mirror path handles the local side and signals the owner to complete its own cleanup.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# OWNER PATH (10 steps)
|
||||
# Step 1: Stop rsync — halt any running sync before state changes
|
||||
# Step 2: Final sync — mirror leaves with current Critical-Data state
|
||||
# Step 3: Reconfigure WebUIs — mirror's auth WebUIs → localhost
|
||||
# Step 4: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf
|
||||
# Step 5: Write state — INACTIVE locally + pushed to mirror, mirror blocklisted
|
||||
# Step 6: Local cleanup — remove fallback coverage containers + appdata
|
||||
# Step 7: Restart own stack — bring up owner's own parked containers
|
||||
# Step 8: Remote cleanup — remove auth/arr stack + fallback containers from mirror
|
||||
# Step 9: Restart mirror — bring up mirror's own parked containers
|
||||
# Step 10: Revocation — Emby admin, SSH keys, Tailscale device
|
||||
#
|
||||
# MIRROR PATH (8 steps)
|
||||
# Step 1: Stop rsync — halt any running sync
|
||||
# Step 2: Reconfigure WebUIs — local auth WebUIs → localhost
|
||||
# Step 3: Remote stack clean — remove owner-deployed containers locally (auth/arr stack)
|
||||
# Step 4: Fallback cleanup — remove fallback coverage containers
|
||||
# Step 5: Disable sync — CRITICAL_RSYNC_ENABLED=false in master.conf
|
||||
# Step 6: Revoke Emby admin — remove own admin account from local Emby instance
|
||||
# Step 7: Restart own stack — bring up own parked containers
|
||||
# Step 8: SSH revocation — revoke keys both directions, write state, signal owner
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master_host*.conf
|
||||
#
|
||||
# HOST*_PARTNERSHIP_AUTH_STACK
|
||||
# Auth container XMLs to push during onboard — used on offboard to identify what
|
||||
# to remove. Owner's PARTNERSHIP_AUTH_STACK determines which containers get removed
|
||||
# from the mirror on both owner-initiated and mirror-initiated offboard.
|
||||
#
|
||||
# HOST*_PARTNERSHIP_ARR_STACK
|
||||
# Arr container XMLs — same cleanup logic as auth stack.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Partnership/partnership_offboard.sh
|
||||
# Full offboard — role detected automatically
|
||||
#
|
||||
# Partnership/partnership_offboard.sh --dry-run
|
||||
# Preview all steps without executing
|
||||
#
|
||||
# Partnership/partnership_offboard.sh --log
|
||||
# Verbose per-step output
|
||||
#
|
||||
# Partnership/partnership_offboard.sh --reason=<string>
|
||||
# Tag the offboard reason in state file and blocklist (default: manual)
|
||||
# Called by partnership_manager.sh --offboard (reason passed through)
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
TEMPLATES_DIR="/boot/config/plugins/dockerMan/templates-user"
|
||||
SSH_TIMEOUT=15
|
||||
|
||||
source "$SCRIPTS_ROOT/load_config.sh"
|
||||
|
||||
# ── Parse flags ───────────────────────────────────────────────────────────────────────────────
|
||||
REASON="manual"
|
||||
FILTERED_ARGS=()
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--reason=*) REASON="${arg#--reason=}" ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ── Source partnership_manager.sh for shared helpers ──────────────────────────────────────────
|
||||
# PARTNERSHIP_LIB_MODE=1 skips mode dispatch — functions are defined, nothing is executed.
|
||||
PARTNERSHIP_LIB_MODE=1 source "$SCRIPT_DIR/partnership_manager.sh"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
[[ "$EUID" -ne 0 ]] && { error "Must be run as root"; exit 1; }
|
||||
|
||||
detect_hosts
|
||||
|
||||
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}"
|
||||
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
||||
OWNER="${!OWNER_ID}"
|
||||
MIRROR="${!MIRROR_ID}"
|
||||
MIRROR_SSH_KEY="$SSH_KEY"
|
||||
OWNER_SSH_KEY="$SSH_KEY"
|
||||
|
||||
AM_OWNER=false
|
||||
AM_MIRROR=false
|
||||
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
||||
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
||||
|
||||
LOCAL_STATE_FILE="/boot/config/partnership_${LOCAL_SERVER_NAME}.db"
|
||||
REMOTE_STATE_FILE="/boot/config/partnership_${REMOTE_SERVER_NAME}.db"
|
||||
OWNER_STATE_FILE="/boot/config/partnership_${OWNER}.db"
|
||||
MIRROR_STATE_FILE="/boot/config/partnership_${MIRROR}.db"
|
||||
OFFLINE_COUNTER="/boot/config/partnership_offline_days.db"
|
||||
|
||||
acquire_lock "strict"
|
||||
|
||||
# Check already offboarded
|
||||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||||
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||||
if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then
|
||||
warn "Partnership already INACTIVE — use partnership_manager.sh --status to verify both servers agree"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Partnership Offboard — $MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
|
||||
echo " This: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " Partner: $( [[ "$AM_OWNER" == true ]] && echo "$MIRROR_ID ($MIRROR)" || echo "$OWNER_ID ($OWNER)" )"
|
||||
echo " Reason: $REASON"
|
||||
echo ""
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no permanent changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER: remove owner-deployed containers from a remote host ───────────────────────────────
|
||||
#
|
||||
# Uses PARTNERSHIP_AUTH_STACK + PARTNERSHIP_ARR_STACK arrays (owner's conf) to derive
|
||||
# container names from local XML templates. SSHes to remote to stop, remove, and delete
|
||||
# appdata. Appdata paths are collected via docker inspect before removal so they aren't
|
||||
# lost once the container is gone. Safety gate: only /mnt/*/appdata* paths are deleted.
|
||||
# ==============================================================================================
|
||||
cleanup_deployed_stack_on_remote() {
|
||||
local remote_ip="$1" ssh_key="$2"
|
||||
local -a xml_names=()
|
||||
[[ ${#PARTNERSHIP_AUTH_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_AUTH_STACK[@]}")
|
||||
[[ ${#PARTNERSHIP_ARR_STACK[@]} -gt 0 ]] && xml_names+=("${PARTNERSHIP_ARR_STACK[@]}")
|
||||
|
||||
if [[ ${#xml_names[@]} -eq 0 ]]; then
|
||||
log "No auth/arr stack arrays configured — skipping deployed stack cleanup"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Removing owner-deployed containers (auth/arr stacks) from $MIRROR..."
|
||||
for xml_name in "${xml_names[@]}"; do
|
||||
[[ -z "$xml_name" ]] && continue
|
||||
local xml_file="${TEMPLATES_DIR}/${xml_name}"
|
||||
if [[ ! -f "$xml_file" ]]; then
|
||||
warn " $xml_name not found in local $TEMPLATES_DIR — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
local cname
|
||||
cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
|
||||
[[ -z "$cname" ]] && continue
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn " DRY RUN — would stop + rm $cname on $MIRROR"
|
||||
warn " DRY RUN — would delete appdata for $cname on $MIRROR"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Collect appdata paths via docker inspect before removal
|
||||
local appdata_paths
|
||||
appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||||
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$cname' 2>/dev/null \
|
||||
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
|
||||
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
||||
"docker stop '$cname' >/dev/null 2>&1
|
||||
docker rm '$cname' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
|
||||
grep -q removed && \
|
||||
log " $cname removed from $MIRROR ✅" || \
|
||||
log " $cname not found on $MIRROR — skipping"
|
||||
|
||||
while IFS= read -r path; do
|
||||
[[ -z "$path" ]] && continue
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
||||
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
|
||||
log " Appdata removed on $MIRROR: $path ✅" || \
|
||||
warn " Failed to remove appdata on $MIRROR: $path"
|
||||
done <<< "$appdata_paths"
|
||||
done
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER: remove owner-deployed containers locally (mirror-initiated offboard) ─────────────
|
||||
#
|
||||
# SSHes to owner to read PARTNERSHIP_AUTH_STACK + PARTNERSHIP_ARR_STACK, then uses the
|
||||
# local templates-user/ copies (SCPed there during onboard) to get container names and
|
||||
# appdata paths. Appdata collected before removal. Skips gracefully if owner unreachable.
|
||||
# ==============================================================================================
|
||||
cleanup_deployed_stack_locally() {
|
||||
local owner_ip="$1" ssh_key="$2"
|
||||
local -a xml_names=()
|
||||
|
||||
if [[ -n "$owner_ip" ]]; then
|
||||
local -a auth_arr arr_arr
|
||||
mapfile -t auth_arr < <(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$owner_ip" \
|
||||
"source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
|
||||
detect_hosts 2>/dev/null
|
||||
printf '%s\n' \"\${PARTNERSHIP_AUTH_STACK[@]:-}\"" 2>/dev/null | grep -v '^$')
|
||||
mapfile -t arr_arr < <(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$owner_ip" \
|
||||
"source '$SCRIPTS_ROOT/load_config.sh' 2>/dev/null
|
||||
detect_hosts 2>/dev/null
|
||||
printf '%s\n' \"\${PARTNERSHIP_ARR_STACK[@]:-}\"" 2>/dev/null | grep -v '^$')
|
||||
xml_names=("${auth_arr[@]}" "${arr_arr[@]}")
|
||||
fi
|
||||
|
||||
if [[ ${#xml_names[@]} -eq 0 ]]; then
|
||||
log "Could not read deployed stack from owner — skipping auth/arr cleanup"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Removing owner-deployed containers (auth/arr stacks) locally..."
|
||||
for xml_name in "${xml_names[@]}"; do
|
||||
[[ -z "$xml_name" ]] && continue
|
||||
local xml_file="${TEMPLATES_DIR}/${xml_name}"
|
||||
if [[ ! -f "$xml_file" ]]; then
|
||||
warn " $xml_name not found locally — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
local cname
|
||||
cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
|
||||
[[ -z "$cname" ]] && continue
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn " DRY RUN — would stop + rm $cname"
|
||||
warn " DRY RUN — would delete appdata for $cname"
|
||||
continue
|
||||
fi
|
||||
|
||||
local appdata_paths=""
|
||||
if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$cname" >/dev/null 2>&1; then
|
||||
appdata_paths=$(docker inspect \
|
||||
--format '{{range .HostConfig.Binds}}{{println .}}{{end}}' \
|
||||
"$cname" 2>/dev/null | awk -F: '{print $1}' | grep '^/mnt/.*/appdata')
|
||||
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$cname" >/dev/null 2>&1 || true
|
||||
timeout "${DOCKER_TIMEOUT:-30}" docker rm "$cname" >/dev/null 2>&1 && \
|
||||
log " $cname removed ✅" || warn " $cname rm failed"
|
||||
else
|
||||
log " $cname not found locally — skipping"
|
||||
fi
|
||||
|
||||
while IFS= read -r path; do
|
||||
[[ -z "$path" ]] && continue
|
||||
rm -rf "$path" && log " Appdata removed: $path ✅" || warn " Failed to remove: $path"
|
||||
done <<< "$appdata_paths"
|
||||
done
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER: revoke own admin account from local Emby instance ────────────────────────────────
|
||||
#
|
||||
# Mirror-initiated path only. Called before start_own_stack so Emby is still running.
|
||||
# Uses local EMBY_API_KEY and the mirror's own short name as the username to delete.
|
||||
# ==============================================================================================
|
||||
revoke_local_emby_admin() {
|
||||
local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}"
|
||||
local emby_url="http://127.0.0.1:${emby_port}"
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_EMBY Emby Admin Revocation ━━━"
|
||||
|
||||
if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then
|
||||
log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ -z "${EMBY_API_KEY:-}" ]]; then
|
||||
warn "EMBY_API_KEY not set — skipping local Emby admin revocation"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# The account to revoke is this server's own short name (the mirror user's account)
|
||||
local username="${PARTNERSHIP_EMBY_ADMIN_USER:-$(derive_short_name "$LOCAL_SERVER_NAME")}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would delete Emby admin '$username' at $emby_url"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Looking up Emby user '$username' at $emby_url..."
|
||||
|
||||
local users_json user_id
|
||||
users_json=$(curl -sf --max-time 15 \
|
||||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||||
"${emby_url}/Users" 2>/dev/null)
|
||||
|
||||
user_id=$(echo "$users_json" | \
|
||||
grep -o "\"Id\":\"[^\"]*\"[^}]*\"Name\":\"${username}\"" | \
|
||||
grep -o '"Id":"[^"]*"' | cut -d'"' -f4 | head -1)
|
||||
|
||||
if [[ -z "$user_id" ]]; then
|
||||
warn "Emby user '$username' not found at $emby_url — may already be removed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local del_code
|
||||
del_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \
|
||||
-X DELETE \
|
||||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||||
"${emby_url}/Users/${user_id}" 2>/dev/null)
|
||||
|
||||
if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then
|
||||
log "Emby admin '$username' removed ✅"
|
||||
else
|
||||
warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MIRROR PATH ───────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
if [[ "$AM_MIRROR" == true ]]; then
|
||||
warn "$MIRROR_ID ($MIRROR) is initiating offboard"
|
||||
warn "Owner ($OWNER) will see INACTIVE state on its next --check cycle and finalize"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
echo ""
|
||||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||||
sleep 10
|
||||
fi
|
||||
|
||||
OWNER_IP=$(resolve_tailscale_ip "$OWNER")
|
||||
OWNER_REACHABLE=false
|
||||
[[ -n "$OWNER_IP" ]] && OWNER_REACHABLE=true
|
||||
|
||||
STEP_STOP_RSYNC_OK=true
|
||||
STEP_WEBUI_OK=true
|
||||
STEP_STACK_CLEANUP_OK=true
|
||||
STEP_FALLBACK_CLEANUP_OK=true
|
||||
STEP_DISABLE_RSYNC_OK=true
|
||||
STEP_EMBY_OK=true
|
||||
SSH_REVOKE_REMOTE_OK=false
|
||||
SSH_REVOKE_LOCAL_OK=false
|
||||
|
||||
# ── Step 1: Stop rsync ────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Step 1/8 — Stop Rsync ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true
|
||||
log "Rsync stopped ✅"
|
||||
else
|
||||
warn "DRY RUN — would stop rsync"
|
||||
fi
|
||||
|
||||
# ── Step 2: Reconfigure local WebUIs → localhost ──────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 2/8 — Reconfigure Local WebUIs → localhost ━━━"
|
||||
|
||||
reconfigure_local_webuis "localhost" || STEP_WEBUI_OK=false
|
||||
|
||||
# ── Step 3: Remove owner-deployed containers (auth/arr stack) locally ─────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 3/8 — Remove Owner-Deployed Containers ━━━"
|
||||
|
||||
if [[ "$OWNER_REACHABLE" == true ]]; then
|
||||
cleanup_deployed_stack_locally "$OWNER_IP" "$OWNER_SSH_KEY" || STEP_STACK_CLEANUP_OK=false
|
||||
else
|
||||
warn "Owner unreachable — cannot read deployed stack list"
|
||||
warn "Auth/arr containers will remain — remove manually or re-run when owner is reachable"
|
||||
STEP_STACK_CLEANUP_OK=false
|
||||
fi
|
||||
|
||||
# ── Step 4: Remove fallback coverage containers ───────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 4/8 — Fallback Container Cleanup ━━━"
|
||||
|
||||
PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$OWNER")
|
||||
cleanup_partner_containers "$PARTNER_FOLDER_NAME" || STEP_FALLBACK_CLEANUP_OK=false
|
||||
|
||||
# ── Step 5: Disable critical sync ─────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 5/8 — Disable Critical Sync ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false" && \
|
||||
warn "CRITICAL_RSYNC_ENABLED=false ✅" || \
|
||||
{ warn "Failed to update CRITICAL_RSYNC_ENABLED"; STEP_DISABLE_RSYNC_OK=false; }
|
||||
else
|
||||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
||||
fi
|
||||
|
||||
# ── Step 6: Revoke Emby admin locally ─────────────────────────────────────────────────────
|
||||
revoke_local_emby_admin || STEP_EMBY_OK=false
|
||||
|
||||
# ── Step 7: Restart own stack ─────────────────────────────────────────────────────────────
|
||||
start_own_stack
|
||||
|
||||
# ── Step 8: SSH key revocation, write state, signal owner ────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Step 8/8 — SSH Revocation + State ━━━"
|
||||
|
||||
do_ssh_key_revocation "${OWNER_IP:-}"
|
||||
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||||
log "Local state: INACTIVE ✅"
|
||||
add_to_blocklist "$OWNER" "$REASON"
|
||||
else
|
||||
warn "DRY RUN — would write INACTIVE state and blocklist $OWNER"
|
||||
fi
|
||||
|
||||
if [[ "$OWNER_REACHABLE" == true ]]; then
|
||||
push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$OWNER_SSH_KEY"
|
||||
notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \
|
||||
"Partnership" "normal"
|
||||
else
|
||||
warn "$OWNER unreachable — state written locally, owner will see it when reachable"
|
||||
fi
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────────────────
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY (Mirror) ━━━━━"
|
||||
echo " Mirror: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " Owner: $OWNER_ID ($OWNER)"
|
||||
echo " Reason: $REASON"
|
||||
echo " Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
|
||||
_skip() { [[ "$1" == true ]] && echo "skipped" || echo "$(_ok "$2")"; }
|
||||
_revoke_status() {
|
||||
if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true && "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||||
echo "both directions ✅"
|
||||
elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||||
echo "local only ✅ — remote failed (revoke manually on $OWNER)"
|
||||
else
|
||||
echo "⚠️ failed — check warnings above"
|
||||
fi
|
||||
}
|
||||
|
||||
echo " Step 1 — Stop rsync: $(_ok "$STEP_STOP_RSYNC_OK")"
|
||||
echo " Step 2 — WebUIs: $(_ok "$STEP_WEBUI_OK")"
|
||||
echo " Step 3 — Stack cleanup: $(_ok "$STEP_STACK_CLEANUP_OK")"
|
||||
echo " Step 4 — Fallback cleanup: $(_ok "$STEP_FALLBACK_CLEANUP_OK")"
|
||||
echo " Step 5 — Disable sync: $(_ok "$STEP_DISABLE_RSYNC_OK")"
|
||||
echo " Step 6 — Emby revoke: $(_ok "$STEP_EMBY_OK")"
|
||||
echo " Step 7 — Own stack: started"
|
||||
echo " Step 8 — Keys revoked: $(_revoke_status)"
|
||||
echo ""
|
||||
echo " State: INACTIVE ✅"
|
||||
echo " Blocklist: $OWNER blocked ✅"
|
||||
echo " Owner: will finalise + final sync on next --check"
|
||||
echo ""
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
||||
warn "$ICON_DONE DONE — mirror separation complete ✅"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── OWNER PATH ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
warn "Offboarding $MIRROR_ID ($MIRROR) from partnership"
|
||||
warn "Final sync will run — mirror leaves with current state"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
echo ""
|
||||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||||
sleep 10
|
||||
echo "Proceeding..."
|
||||
fi
|
||||
|
||||
WEBUI_FAILURES=0
|
||||
SSH_REVOKE_REMOTE_OK=false
|
||||
SSH_REVOKE_LOCAL_OK=false
|
||||
|
||||
# ── Step 1: Stop rsync ────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Step 1/10 — Stop Rsync ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
bash "$SCRIPTS_ROOT/Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true
|
||||
log "Rsync stopped ✅"
|
||||
else
|
||||
warn "DRY RUN — would stop rsync"
|
||||
fi
|
||||
|
||||
# ── Step 2: Final sync ────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Step 2/10 — Final Sync ━━━"
|
||||
|
||||
do_final_sync
|
||||
|
||||
# ── Step 3: Reconfigure mirror WebUIs → localhost ─────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 3/10 — Reconfigure Mirror WebUIs → localhost ━━━"
|
||||
|
||||
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
||||
MIRROR_REACHABLE=false
|
||||
[[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true
|
||||
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
container="${entry%%|*}"
|
||||
port="${entry##*|}"
|
||||
reconfigure_webui "$container" "$port" "localhost" \
|
||||
"$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ ))
|
||||
done
|
||||
else
|
||||
warn "$MIRROR unreachable — WebUI reconfiguration skipped"
|
||||
warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on --check"
|
||||
(( WEBUI_FAILURES++ ))
|
||||
fi
|
||||
|
||||
# ── Step 4: Disable critical sync ─────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 4/10 — Disable Critical Sync ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false"
|
||||
warn "CRITICAL_RSYNC_ENABLED=false ✅"
|
||||
else
|
||||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
||||
fi
|
||||
|
||||
# ── Step 5: Write state, push to mirror, blocklist ────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Step 5/10 — Write State ━━━"
|
||||
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||||
log "Local state: INACTIVE ✅"
|
||||
add_to_blocklist "$MIRROR" "$REASON"
|
||||
[[ "$MIRROR_REACHABLE" == true ]] && \
|
||||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||||
else
|
||||
warn "DRY RUN — would write INACTIVE state, blocklist $MIRROR, push to remote"
|
||||
fi
|
||||
|
||||
# ── Step 6: Local container cleanup ───────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 6/10 — Local Container Cleanup ━━━"
|
||||
|
||||
PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR")
|
||||
cleanup_partner_containers "$PARTNER_FOLDER_NAME"
|
||||
|
||||
# ── Step 7: Restart own stack ─────────────────────────────────────────────────────────────────
|
||||
start_own_stack
|
||||
|
||||
# ── Step 8: Remote container cleanup ──────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Step 8/10 — Remote Container Cleanup ━━━"
|
||||
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
# Remove auth/arr stack containers deployed during onboard (by config array)
|
||||
cleanup_deployed_stack_on_remote "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||||
# Remove fallback coverage containers (by *-owner_short naming pattern)
|
||||
cleanup_owner_containers_on_mirror "$MIRROR_IP"
|
||||
else
|
||||
warn "$MIRROR unreachable — remote container cleanup skipped"
|
||||
warn "Run 'partnership_manager.sh --offboard' on $MIRROR to clean up manually"
|
||||
fi
|
||||
|
||||
# ── Step 9: Restart mirror's own stack ────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Step 9/10 — Restart Mirror Stack ━━━"
|
||||
|
||||
[[ "$MIRROR_REACHABLE" == true ]] && start_mirror_own_stack "$MIRROR_IP"
|
||||
|
||||
# ── Step 10: Revocation ───────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Step 10/10 — Revocation ━━━"
|
||||
|
||||
# Emby admin — before SSH key revocation while Emby still reachable
|
||||
[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP"
|
||||
|
||||
# SSH key revocation — mutual, both directions; must run while Tailscale still active
|
||||
do_ssh_key_revocation "${MIRROR_IP:-}"
|
||||
|
||||
# Tailscale removal — after SSH revocation, guard with grace window
|
||||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET Tailscale Separation ━━━"
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 ))
|
||||
warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace — mirror can collect backups..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
trap 'warn "Offboard interrupted during grace sleep"; exit 0' SIGTERM SIGINT
|
||||
sleep "$grace_seconds"
|
||||
trap - SIGTERM SIGINT
|
||||
fi
|
||||
fi
|
||||
remove_tailscale_device "$MIRROR"
|
||||
fi
|
||||
|
||||
# Backup handover notification
|
||||
if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_DISK Backup Handover ━━━"
|
||||
log "Backups available for $MIRROR:"
|
||||
for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do
|
||||
[[ -z "$path" ]] && continue
|
||||
echo " $path"
|
||||
done
|
||||
notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-6}hr. Tailscale access expires then." \
|
||||
"Partnership" "warning"
|
||||
fi
|
||||
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
|
||||
END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY (Owner) ━━━━━"
|
||||
echo " Owner: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo " Mirror: $MIRROR_ID ($MIRROR)"
|
||||
echo " Reason: $REASON"
|
||||
echo " Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
_ok() { [[ "$1" == true ]] && echo "✅" || echo "❌"; }
|
||||
_revoke_status() {
|
||||
if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true && "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||||
echo "both directions ✅"
|
||||
elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||||
echo "local only ✅ — remote failed (revoke manually on $MIRROR)"
|
||||
else
|
||||
echo "⚠️ failed — check warnings above"
|
||||
fi
|
||||
}
|
||||
|
||||
echo " Step 1 — Stop rsync: ✅"
|
||||
echo " Step 2 — Final sync: ✅"
|
||||
echo " Step 3 — WebUI failures: $WEBUI_FAILURES"
|
||||
echo " Step 4 — Disable sync: ✅"
|
||||
echo " Step 5 — State: INACTIVE ✅"
|
||||
echo " Step 6 — Local cleanup: ✅"
|
||||
echo " Step 7 — Own stack: started"
|
||||
echo " Step 8 — Remote cleanup: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "✅" || echo "skipped (unreachable)" )"
|
||||
echo " Step 9 — Mirror stack: $( [[ "$MIRROR_REACHABLE" == true ]] && echo "started" || echo "skipped (unreachable)" )"
|
||||
echo " Step 10 — Keys revoked: $(_revoke_status)"
|
||||
echo ""
|
||||
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
|
||||
[[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]] && \
|
||||
echo " FolderView3: ${PARTNER_FOLDER_NAME:-} cleaned ✅"
|
||||
[[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \
|
||||
echo " Tailscale: $MIRROR removed ✅"
|
||||
echo ""
|
||||
echo " $MIRROR leaves with:"
|
||||
echo " ✓ Current auth config (final sync)"
|
||||
echo " ✓ Auth WebUIs → localhost"
|
||||
echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups"
|
||||
echo ""
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
||||
warn "$ICON_DONE DONE — clean separation complete ✅"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
Reference in New Issue
Block a user