feat: slskd reconnect guard in downloaders_reset, mass v2 sync
- downloaders_reset: connection check block before slskd API sections; triggers PUT /api/v0/server reconnect if disconnected, polls 60s, gates Stuck Searches and Dead Transfer Records on SLSKD_CONNECTED - Sync all modified/new/deleted files from v2 refactor across Docker_Essentials, Media, Monitors, Partnership, Rsync, Tools, Transcodes, unRAID_Essentials, common.sh, master confs, and new Manual/README docs
This commit is contained in:
Gmer4Lfe
+269
-337
@@ -1,93 +1,187 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Partnership Manager ========================================
|
||||
# ============================= Partnership Manager ============================================
|
||||
# ==============================================================================================
|
||||
# Manages the relationship lifecycle between two unRAID servers.
|
||||
# HOST1 is always the owner (source of truth). HOST2 is always the mirror.
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Manages the full lifecycle of a two-server partnership — onboard, daily health
|
||||
# monitoring, offboard, and ownership transfer. Called by partnership_onboard.sh
|
||||
# during initial setup, and by critical_sync_maintenance.sh every 15 minutes for
|
||||
# the --check mode. All other modes are run manually.
|
||||
#
|
||||
# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a successful --transfer.
|
||||
# AM_OWNER / AM_MIRROR flags (set by detect_hosts) control all routing —
|
||||
# no hostname string comparisons anywhere in this script.
|
||||
#
|
||||
# ── MODES ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# --onboard ← owner only — establish mirror relationship
|
||||
# Reconfigures HOST2 auth WebUIs → HOST1 Tailscale IP
|
||||
# HOST2 clicks NPM → gets HOST1's NPM automatically
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# --offboard ← either server — clean separation
|
||||
# Mirror-initiated: writes INACTIVE state, reconfigures own WebUIs → localhost
|
||||
# Owner finalises on next --check: final sync, Tailscale removal
|
||||
# Owner-initiated: final sync, reconfigures mirror WebUIs, Tailscale removal
|
||||
# Both leave with current state, clean exit ✅
|
||||
# --onboard (owner only)
|
||||
# Reconfigures mirror auth WebUIs → owner's Tailscale IP
|
||||
# Mirror operator clicks NPM → gets owner's NPM via Tailscale automatically
|
||||
# Writes ACTIVE state on both servers
|
||||
#
|
||||
# --transfer ← owner only — flip ownership between servers
|
||||
# Requires confirmation string + consecutive health strikes
|
||||
# Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in master.conf
|
||||
# --offboard (either server)
|
||||
# Mirror-initiated: reconfigures own WebUIs → localhost, writes INACTIVE state
|
||||
# Owner finalises on next --check: final sync, Tailscale removal
|
||||
# Owner-initiated: final sync, reconfigures mirror WebUIs → localhost, removes
|
||||
# mirror containers + appdata, SSH key revocation, Tailscale removal
|
||||
# Both leave with clean state ✅
|
||||
#
|
||||
# --check ← called by critical_sync_maintenance.sh every 15min
|
||||
# Reads both state files via SSH
|
||||
# Detects offboard requests → finalises from owner side
|
||||
# Updates last_seen_remote timestamp
|
||||
# Increments offline counter → auto-offboards after threshold
|
||||
# Silent when healthy ← never noisy on clean runs
|
||||
# --transfer (owner only)
|
||||
# Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in master.conf
|
||||
# Requires confirmation string + consecutive health check passes
|
||||
#
|
||||
# --status ← either server — show current state, both sides
|
||||
# --check (called every 15min by critical_sync_maintenance.sh)
|
||||
# --remote-seen: rsync succeeded → reset offline counter, read remote state
|
||||
# --remote-unseen: rsync failed → increment offline counter → auto-offboard at threshold
|
||||
# Silent when healthy — never noisy on clean runs
|
||||
#
|
||||
# ── STATE FILES ───────────────────────────────────────────────────────────────────────────────
|
||||
# On /boot/config — survives reboots, available before array starts:
|
||||
# /boot/config/partnership_HOST1.db ← HOST1 writes only
|
||||
# /boot/config/partnership_HOST2.db ← HOST2 writes only
|
||||
# Propagated via SSH — no rsync needed
|
||||
# --status (either server)
|
||||
# Show state files from both servers, blocklist, SSH key status
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Root check — all operations require root
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
# Role enforcement — mirror cannot run owner-only modes
|
||||
# AM_OWNER / AM_MIRROR — all routing via these flags, not hostname strings
|
||||
# version parity — onboard checks both servers match unRAID version
|
||||
# remote docker daemon — onboard verifies remote daemon responsive
|
||||
# SSH_TIMEOUT — all SSH calls timeout-protected
|
||||
# flock on state writes — prevents concurrent state file corruption
|
||||
# SIGTERM trap — grace period sleep interruptible
|
||||
# Silent by default — only warns/errors produce output (--check is always silent healthy)
|
||||
# ==============================================================================================
|
||||
# DESIGN PRINCIPLES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_PARTNERSHIP_AUTH_WEBUIS — containers reconfigured on onboard/offboard
|
||||
# HOST*_PARTNERSHIP_MIRROR_BACKUPS — paths available after offboard
|
||||
# All aliased by detect_hosts() — script uses PARTNERSHIP_AUTH_WEBUIS etc.
|
||||
# Deferred offboard
|
||||
# Either server can offboard without the other being reachable. The initiating
|
||||
# server writes its state immediately and becomes independent. The other server
|
||||
# reads the INACTIVE state on its next --check and finalises automatically.
|
||||
# No message passing, no coordination required.
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# PARTNERSHIP_ENABLED — global enable gate
|
||||
# PARTNERSHIP_OWNER_HOST — "HOST1" or "HOST2" — flips on --transfer
|
||||
# PARTNERSHIP_REMOVE_TAILSCALE — remove mirror from tailnet on offboard
|
||||
# PARTNERSHIP_GRACE_HOURS — hours before Tailscale removal after offboard
|
||||
# PARTNERSHIP_OFFLINE_THRESHOLD — days unreachable before auto-offboard
|
||||
# PARTNERSHIP_TRANSFER_CONFIRM — exact string required for --transfer
|
||||
# PARTNERSHIP_TRANSFER_STRIKES — consecutive health checks required
|
||||
# PARTNERSHIP_TRANSFER_MAX_ATTEMPTS — max attempts before giving up
|
||||
# PARTNERSHIP_ONBOARD_VERIFY — verify WebUI connectivity after onboard
|
||||
# PARTNERSHIP_ONBOARD_NOTIFY — notify both servers on onboard completion
|
||||
# PARTNERSHIP_SYNC_INTERVAL — informational — actual schedule in cron
|
||||
# TAILSCALE_API_KEY / TAILSCALE_TAILNET — required when PARTNERSHIP_REMOVE_TAILSCALE=true
|
||||
# Appdata cleanup on offboard
|
||||
# Partner containers are stopped and removed. Their appdata bind-mount paths
|
||||
# (collected via docker inspect before removal) are also deleted. Safety gate:
|
||||
# only paths matching /mnt/*/appdata* are deleted — media and config shares
|
||||
# outside the appdata tree are never touched.
|
||||
#
|
||||
# ── BLOCKLIST ─────────────────────────────────────────────────────────────────────────────────
|
||||
# After offboard, the former partner's hostname is written to:
|
||||
# /boot/config/partnership_blocklist.db (format: hostname|timestamp|reason)
|
||||
# Blocklist is application-layer; Tailscale removal is network-layer
|
||||
# Both happen on offboard. The blocklist prevents re-onboard until explicitly
|
||||
# cleared with --unblock. Tailscale removal ends encrypted access at the network
|
||||
# level. Grace period controls both simultaneously — one var, consistent behaviour.
|
||||
#
|
||||
# --onboard is hard-blocked if the remote is on the blocklist — exits with error.
|
||||
# --check silently skips remote state reads for blocklisted hosts (no noise every 15min).
|
||||
# --unblock <hostname> removes an entry to permit re-onboarding.
|
||||
# --status shows the full blocklist.
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# The blocklist persists until explicitly cleared — surviving reboots, array restarts,
|
||||
# and Tailscale reconnections. Tailscale removal is a separate step at the network layer;
|
||||
# the blocklist is the application-layer guard.
|
||||
# Root check
|
||||
# All operations require root.
|
||||
#
|
||||
# Role enforcement
|
||||
# Mirror cannot run --onboard or --transfer. Blocked with a clear error message.
|
||||
#
|
||||
# Version parity check
|
||||
# --onboard verifies both servers are on compatible unRAID versions.
|
||||
#
|
||||
# SSH_TIMEOUT on all remote calls
|
||||
# Every ssh/scp call is timeout-protected. No operation hangs on an unreachable peer.
|
||||
#
|
||||
# flock on state writes
|
||||
# Prevents concurrent state file corruption from overlapping --check cycles.
|
||||
#
|
||||
# SIGTERM trap on grace period sleep
|
||||
# Offboard grace period is interruptible — Ctrl-C aborts cleanly.
|
||||
#
|
||||
# Silent by default
|
||||
# --check produces no output when both servers are healthy. Only state changes
|
||||
# and threshold crossings produce output.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# STATE FILES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# /boot/config/partnership_HOST1.db — HOST1 writes, HOST2 reads via SSH
|
||||
# /boot/config/partnership_HOST2.db — HOST2 writes, HOST1 reads via SSH
|
||||
# /boot/config/partnership_blocklist.db — hostname|timestamp|reason, persists until cleared
|
||||
#
|
||||
# On /boot/config — survives reboots, available before array starts, minimal flash wear.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# PARTNERSHIP_ENABLED
|
||||
# Global gate — set true once both servers are configured (default: false)
|
||||
#
|
||||
# PARTNERSHIP_OWNER_HOST
|
||||
# "HOST1" or "HOST2" — flips on --transfer (default: "HOST1")
|
||||
#
|
||||
# PARTNERSHIP_REMOVE_TAILSCALE
|
||||
# Remove mirror from tailnet on offboard (default: true)
|
||||
#
|
||||
# PARTNERSHIP_GRACE_HOURS
|
||||
# Hours before Tailscale removal after offboard — backup access also expires then (default: 6)
|
||||
#
|
||||
# PARTNERSHIP_OFFLINE_THRESHOLD
|
||||
# Days of missed sync cycles before auto-offboard triggers (default: 30)
|
||||
#
|
||||
# PARTNERSHIP_TRANSFER_CONFIRM
|
||||
# Exact string required for --transfer (default: "i-understand-this-transfers-ownership")
|
||||
#
|
||||
# PARTNERSHIP_TRANSFER_STRIKES
|
||||
# Consecutive health checks required before transfer proceeds (default: 3)
|
||||
#
|
||||
# PARTNERSHIP_TRANSFER_MAX_ATTEMPTS
|
||||
# Max health check attempts before giving up (default: 20)
|
||||
#
|
||||
# PARTNERSHIP_ONBOARD_VERIFY
|
||||
# Curl-verify each WebUI after reconfigure to confirm Tailscale routing works (default: true)
|
||||
#
|
||||
# PARTNERSHIP_ONBOARD_NOTIFY
|
||||
# Notify both servers on successful onboard (default: true)
|
||||
#
|
||||
# PARTNERSHIP_SYNC_INTERVAL
|
||||
# Informational — actual schedule is in cron (default: 15)
|
||||
#
|
||||
# TAILSCALE_API_KEY / TAILSCALE_TAILNET
|
||||
# Required when PARTNERSHIP_REMOVE_TAILSCALE=true
|
||||
#
|
||||
# master_host*.conf
|
||||
#
|
||||
# HOST*_PARTNERSHIP_AUTH_WEBUIS
|
||||
# Containers reconfigured on onboard/offboard. Format: "ContainerName|WebUIPort"
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_AUTH_WEBUIS
|
||||
#
|
||||
# HOST*_PARTNERSHIP_MIRROR_BACKUPS
|
||||
# Paths accessible to the partner during grace window after offboard.
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_MIRROR_BACKUPS
|
||||
#
|
||||
# HOST*_PARTNERSHIP_OWN_CONTAINERS
|
||||
# Containers parked here during partnership, restarted on offboard.
|
||||
# Aliased by detect_hosts() → PARTNERSHIP_OWN_CONTAINERS
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# partnership_manager.sh --onboard
|
||||
# Establish mirror relationship — owner only
|
||||
#
|
||||
# partnership_manager.sh --offboard
|
||||
# Clean separation — either server. 10-second countdown before executing.
|
||||
#
|
||||
# partnership_manager.sh --offboard --dry-run
|
||||
# Show the complete offboard sequence without executing
|
||||
#
|
||||
# partnership_manager.sh --transfer --confirm=i-understand-this-transfers-ownership
|
||||
# Flip ownership — owner only. Requires exact confirmation string.
|
||||
#
|
||||
# partnership_manager.sh --check --remote-seen|--remote-unseen
|
||||
# Called by critical_sync_maintenance.sh every 15min — do not run manually
|
||||
#
|
||||
# partnership_manager.sh --status
|
||||
# Show state files, blocklist, SSH key status from both servers
|
||||
#
|
||||
# partnership_manager.sh --unblock <hostname>
|
||||
# Remove hostname from blocklist to permit re-onboarding
|
||||
#
|
||||
# Any mode supports --dry-run and --log
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# partnership_manage.sh --onboard
|
||||
# partnership_manage.sh --offboard
|
||||
# partnership_manage.sh --transfer --confirm=i-understand-this-transfers-ownership
|
||||
# partnership_manage.sh --check --remote-seen|--remote-unseen
|
||||
# partnership_manage.sh --status
|
||||
# partnership_manage.sh --unblock <hostname>
|
||||
# Any mode supports --dry-run and --log
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -152,10 +246,12 @@ MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
||||
|
||||
OWNER="${!OWNER_ID}" # hostname string
|
||||
MIRROR="${!MIRROR_ID}"
|
||||
OWNER_SSH_KEY_VAR="${OWNER_ID}_SSH_KEY"
|
||||
MIRROR_SSH_KEY_VAR="${MIRROR_ID}_SSH_KEY"
|
||||
OWNER_SSH_KEY="${!OWNER_SSH_KEY_VAR}"
|
||||
MIRROR_SSH_KEY="${!MIRROR_SSH_KEY_VAR}"
|
||||
# SSH_KEY (set by detect_hosts) is this server's own private key.
|
||||
# The remote accepts it because this server's PUBLIC key was installed there via ssh_setup.sh.
|
||||
# With sparse checkout, each server only has its own master_host{N}.conf — the other server's
|
||||
# key path is never available here. Use SSH_KEY for all outbound SSH regardless of mode.
|
||||
MIRROR_SSH_KEY="$SSH_KEY"
|
||||
OWNER_SSH_KEY="$SSH_KEY"
|
||||
|
||||
AM_OWNER=false
|
||||
AM_MIRROR=false
|
||||
@@ -169,34 +265,36 @@ OWNER_STATE_FILE="/boot/config/partnership_${OWNER}.db"
|
||||
MIRROR_STATE_FILE="/boot/config/partnership_${MIRROR}.db"
|
||||
OFFLINE_COUNTER="/boot/config/partnership_offline_days.db"
|
||||
|
||||
if [[ -z "$MODE" ]]; then
|
||||
error "No mode specified"
|
||||
echo "Usage:"
|
||||
echo " partnership_manage.sh --onboard"
|
||||
echo " partnership_manage.sh --offboard"
|
||||
echo " partnership_manage.sh --transfer --confirm=..."
|
||||
echo " partnership_manage.sh --check --remote-seen|--remote-unseen"
|
||||
echo " partnership_manage.sh --status"
|
||||
echo " partnership_manage.sh --unblock <hostname>"
|
||||
exit 1
|
||||
if [[ "${PARTNERSHIP_LIB_MODE:-}" != "1" ]]; then
|
||||
if [[ -z "$MODE" ]]; then
|
||||
error "No mode specified"
|
||||
echo "Usage:"
|
||||
echo " partnership_manager.sh --onboard"
|
||||
echo " partnership_manager.sh --offboard"
|
||||
echo " partnership_manager.sh --transfer --confirm=..."
|
||||
echo " partnership_manager.sh --check --remote-seen|--remote-unseen"
|
||||
echo " partnership_manager.sh --status"
|
||||
echo " partnership_manager.sh --unblock <hostname>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Role-based access control
|
||||
if [[ "$AM_MIRROR" == true ]]; then
|
||||
case "$MODE" in
|
||||
onboard|transfer)
|
||||
error "Only the owner ($OWNER / $OWNER_ID) can run --$MODE"
|
||||
error "Run from $OWNER or use --offboard to separate cleanly"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Lock for all modes except --check (frequent) and --offboard (offboard script holds its own)
|
||||
[[ "$MODE" != "check" && "$MODE" != "offboard" ]] && acquire_lock "strict"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
fi
|
||||
|
||||
# Role-based access control
|
||||
if [[ "$AM_MIRROR" == true ]]; then
|
||||
case "$MODE" in
|
||||
onboard|transfer)
|
||||
error "Only the owner ($OWNER / $OWNER_ID) can run --$MODE"
|
||||
error "Run from $OWNER or use --offboard to separate cleanly"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Lock for all modes except --check (check is called frequently, lock would pile up)
|
||||
[[ "$MODE" != "check" ]] && acquire_lock "strict"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
@@ -678,19 +776,39 @@ start_own_stack() {
|
||||
done
|
||||
}
|
||||
|
||||
# Remove partnership containers on this server.
|
||||
# Remove partnership containers on this server + their appdata bind-mount paths.
|
||||
# Uses FolderView3 folder if enabled (precise list), else falls back to FALLBACK_*_COVERS_* config.
|
||||
# Appdata paths collected via docker inspect BEFORE removal — inspect fails on removed containers.
|
||||
# Safety gate: only paths matching /mnt/*/appdata* are deleted.
|
||||
cleanup_partner_containers() {
|
||||
local folder_name="$1"
|
||||
declare -a containers=()
|
||||
gather_partner_fallback_containers containers
|
||||
|
||||
if [[ ${#containers[@]} -eq 0 ]]; then
|
||||
log "No partner containers found to remove"
|
||||
if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then
|
||||
folderview3_remove_partner_folder "$folder_name"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Collect appdata paths BEFORE any removal — inspect fails once container is gone
|
||||
local all_appdata_paths=""
|
||||
for container in "${containers[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then
|
||||
local cpaths
|
||||
cpaths=$(docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' \
|
||||
"$container" 2>/dev/null | awk -F: '{print $1}' | grep '^/mnt/.*/appdata')
|
||||
[[ -n "$cpaths" ]] && all_appdata_paths+=$'\n'"$cpaths"
|
||||
fi
|
||||
done
|
||||
|
||||
# Remove containers
|
||||
if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then
|
||||
folderview3_remove_partner_folder "$folder_name"
|
||||
else
|
||||
declare -a containers=()
|
||||
gather_partner_fallback_containers containers
|
||||
if [[ ${#containers[@]} -eq 0 ]]; then
|
||||
log "No partner containers found to remove"
|
||||
return 0
|
||||
fi
|
||||
for container in "${containers[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
@@ -706,9 +824,22 @@ cleanup_partner_containers() {
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Delete appdata after containers are gone
|
||||
while IFS= read -r path; do
|
||||
[[ -z "$path" ]] && continue
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn " DRY RUN — would rm -rf $path"
|
||||
continue
|
||||
fi
|
||||
rm -rf "$path" && log " Appdata removed: $path ✅" || warn " Failed to remove: $path"
|
||||
done <<< "$all_appdata_paths"
|
||||
}
|
||||
|
||||
# SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers).
|
||||
# SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers)
|
||||
# and their appdata bind-mount paths.
|
||||
# Appdata paths collected via SSH docker inspect before removal, then deleted via SSH.
|
||||
# Safety gate: only paths matching /mnt/*/appdata* are deleted on the remote.
|
||||
cleanup_owner_containers_on_mirror() {
|
||||
local mirror_ip="$1"
|
||||
local owner_short
|
||||
@@ -716,7 +847,7 @@ cleanup_owner_containers_on_mirror() {
|
||||
|
||||
log "Removing owner-deployed containers from $MIRROR..."
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove *-${owner_short} containers from $MIRROR"
|
||||
warn "DRY RUN — would remove *-${owner_short} containers + appdata from $MIRROR"
|
||||
return 0
|
||||
fi
|
||||
|
||||
@@ -732,6 +863,14 @@ cleanup_owner_containers_on_mirror() {
|
||||
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
|
||||
# Collect appdata paths before removal
|
||||
local appdata_paths
|
||||
appdata_paths=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
||||
"docker inspect --format '{{range .HostConfig.Binds}}{{println .}}{{end}}' '$container' 2>/dev/null \
|
||||
| awk -F: '{print \$1}' | grep '^/mnt/.*/appdata'" 2>/dev/null)
|
||||
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
||||
"docker stop '$container' >/dev/null 2>&1
|
||||
@@ -739,6 +878,16 @@ cleanup_owner_containers_on_mirror() {
|
||||
grep -q removed && \
|
||||
log "$container removed from $MIRROR ✅" || \
|
||||
warn "Failed to remove $container from $MIRROR"
|
||||
|
||||
# Delete appdata on remote after container removal
|
||||
while IFS= read -r path; do
|
||||
[[ -z "$path" ]] && continue
|
||||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
||||
"rm -rf '$path' && echo removed" 2>/dev/null | grep -q removed && \
|
||||
log " Appdata removed on $MIRROR: $path ✅" || \
|
||||
warn " Failed to remove appdata on $MIRROR: $path"
|
||||
done <<< "$appdata_paths"
|
||||
done <<< "$container_list"
|
||||
}
|
||||
|
||||
@@ -983,6 +1132,11 @@ update_master_conf() {
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Library-mode guard — source only, skip all mode dispatch ─────────────────────────────────
|
||||
# partnership_offboard.sh sources this file with PARTNERSHIP_LIB_MODE=1 to get helper
|
||||
# functions without triggering any mode execution.
|
||||
[[ "${PARTNERSHIP_LIB_MODE:-}" == "1" ]] && { return 0 2>/dev/null || exit 0; }
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Unblock ━━━
|
||||
# ==============================================================================================
|
||||
@@ -1166,8 +1320,8 @@ if [[ "$MODE" == "check" ]]; then
|
||||
REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE")
|
||||
if [[ -z "$REMOTE_CONTENT" ]]; then
|
||||
# IP resolved but SSH returned nothing — could be auth failure, not just missing file
|
||||
if [[ -f "$SCRIPT_DIR/../Initial_run/ssh_setup.sh" ]]; then
|
||||
bash "$SCRIPT_DIR/../Initial_run/ssh_setup.sh" --validate 2>/dev/null || true
|
||||
if [[ -f "$SCRIPT_DIR/ssh_setup.sh" ]]; then
|
||||
bash "$SCRIPT_DIR/ssh_setup.sh" --validate 2>/dev/null || true
|
||||
fi
|
||||
log "Partnership check — remote state file not found"
|
||||
exit 0
|
||||
@@ -1383,229 +1537,7 @@ fi
|
||||
# ━━━ Offboard ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$MODE" == "offboard" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_FALLBACK Offboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
# Check already offboarded
|
||||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||||
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||||
if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then
|
||||
warn "Partnership already INACTIVE — use --status to verify both servers agree"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Mirror-initiated offboard ─────────────────────────────────────────────────────────────
|
||||
if [[ "$AM_MIRROR" == true ]]; then
|
||||
warn "$MIRROR_ID ($MIRROR) is initiating offboard"
|
||||
warn "Local auth WebUIs will be reconfigured → localhost"
|
||||
warn "$OWNER_ID ($OWNER) will finalise on its next --check cycle"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
echo ""
|
||||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||||
sleep 10
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Reconfigure Local WebUIs → localhost ━━━"
|
||||
reconfigure_local_webuis "localhost"
|
||||
|
||||
# Remove owner's deployed containers + restart own stack
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Cleanup Partner Containers ━━━"
|
||||
PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$OWNER")
|
||||
cleanup_partner_containers "$PARTNER_FOLDER_NAME"
|
||||
start_own_stack
|
||||
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||||
log "Local state: INACTIVE ✅"
|
||||
|
||||
[[ "$DRY_RUN" == false ]] && add_to_blocklist "$OWNER" "$REASON"
|
||||
|
||||
OWNER_IP=$(resolve_tailscale_ip "$OWNER")
|
||||
if [[ -n "$OWNER_IP" ]]; then
|
||||
push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$MIRROR_SSH_KEY"
|
||||
notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \
|
||||
"Partnership" "normal"
|
||||
else
|
||||
warn "$OWNER unreachable — state written locally, owner will see it when reachable"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━"
|
||||
echo " Your WebUIs: reconfigured → localhost ✅"
|
||||
echo " State: INACTIVE ✅"
|
||||
echo " Blocklist: $OWNER blocked — re-onboard to permit access again ✅"
|
||||
echo " Owner: will finalise + final sync on next --check ✅"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Owner-initiated offboard ──────────────────────────────────────────────────────────────
|
||||
warn "Offboarding $MIRROR_ID ($MIRROR) from partnership"
|
||||
warn "Final sync will run — mirror leaves with current state"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
echo ""
|
||||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||||
sleep 10
|
||||
echo "Proceeding..."
|
||||
fi
|
||||
|
||||
resolve_remote_ip
|
||||
|
||||
# Stop any running rsync first
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Stop Running Rsync ━━━"
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true
|
||||
|
||||
# Final sync
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Final Sync ━━━"
|
||||
do_final_sync
|
||||
|
||||
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
||||
MIRROR_REACHABLE=false
|
||||
[[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true
|
||||
|
||||
# Reconfigure mirror WebUIs → localhost
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → localhost ━━━"
|
||||
|
||||
WEBUI_FAILURES=0
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
container="${entry%%|*}"
|
||||
port="${entry##*|}"
|
||||
reconfigure_webui "$container" "$port" "localhost" \
|
||||
"$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ ))
|
||||
done
|
||||
else
|
||||
warn "$MIRROR unreachable — WebUI reconfiguration skipped"
|
||||
warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on --check"
|
||||
(( WEBUI_FAILURES++ ))
|
||||
fi
|
||||
|
||||
# Disable critical rsync
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Disable Critical Sync ━━━"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false"
|
||||
warn "CRITICAL_RSYNC_ENABLED=false ✅"
|
||||
else
|
||||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
||||
fi
|
||||
|
||||
# Write state files
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Write State ━━━"
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||||
log "Local state: INACTIVE ✅"
|
||||
|
||||
[[ "$DRY_RUN" == false ]] && add_to_blocklist "$MIRROR" "$REASON"
|
||||
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||||
fi
|
||||
|
||||
# Local: remove fallback-coverage containers for mirror, restart own stack
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Local Container Cleanup ━━━"
|
||||
PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR")
|
||||
cleanup_partner_containers "$PARTNER_FOLDER_NAME"
|
||||
start_own_stack
|
||||
|
||||
# Remote: remove owner's deployed containers from mirror, restart mirror's own stack
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Remote Container Cleanup ━━━"
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
cleanup_owner_containers_on_mirror "$MIRROR_IP"
|
||||
start_mirror_own_stack "$MIRROR_IP"
|
||||
else
|
||||
warn "$MIRROR unreachable — remote container cleanup skipped"
|
||||
warn "Run 'partnership_manager.sh --offboard' on $MIRROR to clean up manually"
|
||||
fi
|
||||
|
||||
# Emby admin revocation — before SSH key revocation while Emby still reachable
|
||||
[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP"
|
||||
|
||||
# SSH key revocation — mutual, both directions
|
||||
# Must run before Tailscale removal (SSH needs network) and after state is pushed
|
||||
SSH_REVOKE_REMOTE_OK=false
|
||||
SSH_REVOKE_LOCAL_OK=false
|
||||
do_ssh_key_revocation "${MIRROR_IP:-}"
|
||||
|
||||
# Tailscale removal
|
||||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET Tailscale Separation ━━━"
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 ))
|
||||
warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace — mirror can collect backups..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
trap 'warn "Offboard interrupted during grace sleep"; exit 0' SIGTERM SIGINT
|
||||
sleep "$grace_seconds"
|
||||
trap - SIGTERM SIGINT
|
||||
fi
|
||||
fi
|
||||
remove_tailscale_device "$MIRROR"
|
||||
fi
|
||||
|
||||
# Backup notification
|
||||
if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_DISK Backup Handover ━━━"
|
||||
log "Backups available for $MIRROR:"
|
||||
for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do
|
||||
[[ -z "$path" ]] && continue
|
||||
echo " $path"
|
||||
done
|
||||
notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-6}hr. Tailscale access expires then." \
|
||||
"Partnership" "warning"
|
||||
fi
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━"
|
||||
echo " Owner: $OWNER_ID ($OWNER)"
|
||||
echo " Mirror: $MIRROR_ID ($MIRROR)"
|
||||
echo " Final sync: complete ✅"
|
||||
echo " WebUI failures: $WEBUI_FAILURES"
|
||||
echo " Critical rsync: disabled ✅"
|
||||
echo " State: INACTIVE ✅"
|
||||
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
|
||||
_revoke_status() {
|
||||
if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true ]] && [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||||
echo "both directions ✅"
|
||||
elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||||
echo "local only ✅ — remote failed (revoke manually on $MIRROR)"
|
||||
else
|
||||
echo "⚠️ failed — check warnings above"
|
||||
fi
|
||||
}
|
||||
echo " Keys revoked: $(_revoke_status)"
|
||||
[[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]] && \
|
||||
echo " FolderView3: ${PARTNER_FOLDER_NAME:-} cleaned ✅"
|
||||
[[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \
|
||||
echo " Tailscale: $MIRROR removed ✅"
|
||||
echo ""
|
||||
echo " $MIRROR leaves with:"
|
||||
echo " ✓ Current auth config (final sync)"
|
||||
echo " ✓ Auth WebUIs → localhost"
|
||||
echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups"
|
||||
echo " ✓ Full ecosystem — just stop the sync"
|
||||
echo ""
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
||||
warn "$ICON_DONE DONE — clean separation complete ✅"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
exec bash "$SCRIPT_DIR/partnership_offboard.sh" "${FILTERED_ARGS[@]}" --reason="$REASON"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
|
||||
Reference in New Issue
Block a user