Gmer4LfeandClaude Sonnet 4.6
0ae31b5fa6
common.sh:
- Add resolve_tailscale_ip() helper — tries `tailscale ip -4` first, falls back to
parsing `tailscale status` output; handles hosts where MagicDNS short-name resolution
is not active
- Add PARTNERSHIP_OWN_CONTAINERS alias in detect_hosts()
- Add aliasing for 4 Emby provisioning vars (PARTNERSHIP_PROVISION_EMBY_ADMIN,
PARTNERSHIP_EMBY_ADMIN_USER, PARTNERSHIP_EMBY_ADMIN_PASS, PARTNERSHIP_EMBY_PORT)
Partnership/partnership_manager.sh:
- Replace 9 bare `tailscale ip -4` calls with resolve_tailscale_ip()
- Add read_remote_conf_var() and read_remote_conf_array() — SSH to mirror, source its
own load_config.sh + detect_hosts(), return aliased variable; solves sparse-checkout
problem where HOST1 cannot read master_host2.conf directly
- Add derive_short_name() — strips unraid- prefix, capitalises first char
- Add cleanup_partner_containers() — removes partner containers via FolderView3 folder
if enabled, else falls back to FALLBACK_*_COVERS_*_TIER* arrays
- Add cleanup_owner_containers_on_mirror() — SSH to mirror, stops and removes containers
matching *-${OWNER_SHORT} naming convention
- Add start_own_stack() and start_mirror_own_stack() — restart own containers locally
or on mirror via SSH using PARTNERSHIP_OWN_CONTAINERS
- Add provision_emby_admin() — reads mirror credentials via read_remote_conf_var, checks
for username collision, creates user + sets password + grants admin policy via Emby API
- Add revoke_emby_admin() — looks up mirror username on local Emby, deletes via REST API
- Wire offboard paths (both mirror-initiated and owner-initiated) to call container
cleanup and stack restart; update --check finalisation paths accordingly
- Fix write_state_file in --onboard not gated on DRY_RUN (was writing ACTIVE state on
dry runs)
master_host1.conf:
- Add HOST1_PARTNERSHIP_OWN_CONTAINERS array
- Add partnership Emby provisioning config (toggle + port + per-host credentials)
master_host2.conf:
- Add HOST2_PARTNERSHIP_OWN_CONTAINERS array
- Add HOST2_PARTNERSHIP_EMBY_ADMIN_USER and HOST2_PARTNERSHIP_EMBY_ADMIN_PASS
Tailscale fix applied to:
- Initial_run/ssh_setup.sh (2 callsites)
- unRAID_Essentials/rsync_stop.sh (1 callsite)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1756 lines
72 KiB
Bash
1756 lines
72 KiB
Bash
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ================================= Partnership Manager ========================================
|
||
# ==============================================================================================
|
||
# Manages the relationship lifecycle between two unRAID servers.
|
||
# HOST1 is always the owner (source of truth). HOST2 is always the mirror.
|
||
# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a successful --transfer.
|
||
#
|
||
# ── MODES ─────────────────────────────────────────────────────────────────────────────────────
|
||
# --onboard ← owner only — establish mirror relationship
|
||
# Reconfigures HOST2 auth WebUIs → HOST1 Tailscale IP
|
||
# HOST2 clicks NPM → gets HOST1's NPM automatically
|
||
#
|
||
# --offboard ← either server — clean separation
|
||
# Mirror-initiated: writes INACTIVE state, reconfigures own WebUIs → localhost
|
||
# Owner finalises on next --check: final sync, Tailscale removal
|
||
# Owner-initiated: final sync, reconfigures mirror WebUIs, Tailscale removal
|
||
# Both leave with current state, clean exit ✅
|
||
#
|
||
# --transfer ← owner only — flip ownership between servers
|
||
# Requires confirmation string + consecutive health strikes
|
||
# Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in master.conf
|
||
#
|
||
# --check ← called by critical_sync_maintenance.sh every 15min
|
||
# Reads both state files via SSH
|
||
# Detects offboard requests → finalises from owner side
|
||
# Updates last_seen_remote timestamp
|
||
# Increments offline counter → auto-offboards after threshold
|
||
# Silent when healthy ← never noisy on clean runs
|
||
#
|
||
# --status ← either server — show current state, both sides
|
||
#
|
||
# ── STATE FILES ───────────────────────────────────────────────────────────────────────────────
|
||
# On /boot/config — survives reboots, available before array starts:
|
||
# /boot/config/partnership_HOST1.db ← HOST1 writes only
|
||
# /boot/config/partnership_HOST2.db ← HOST2 writes only
|
||
# Propagated via SSH — no rsync needed
|
||
#
|
||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||
# Root check — all operations require root
|
||
# validate_unraid_cmd — notify validated before use
|
||
# Role enforcement — mirror cannot run owner-only modes
|
||
# AM_OWNER / AM_MIRROR — all routing via these flags, not hostname strings
|
||
# version parity — onboard checks both servers match unRAID version
|
||
# remote docker daemon — onboard verifies remote daemon responsive
|
||
# SSH_TIMEOUT — all SSH calls timeout-protected
|
||
# flock on state writes — prevents concurrent state file corruption
|
||
# SIGTERM trap — grace period sleep interruptible
|
||
# Silent by default — only warns/errors produce output (--check is always silent healthy)
|
||
#
|
||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||
# HOST*_PARTNERSHIP_AUTH_WEBUIS — containers reconfigured on onboard/offboard
|
||
# HOST*_PARTNERSHIP_MIRROR_BACKUPS — paths available after offboard
|
||
# All aliased by detect_hosts() — script uses PARTNERSHIP_AUTH_WEBUIS etc.
|
||
#
|
||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||
# PARTNERSHIP_ENABLED — global enable gate
|
||
# PARTNERSHIP_OWNER_HOST — "HOST1" or "HOST2" — flips on --transfer
|
||
# PARTNERSHIP_REMOVE_TAILSCALE — remove mirror from tailnet on offboard
|
||
# PARTNERSHIP_GRACE_HOURS — hours before Tailscale removal after offboard
|
||
# PARTNERSHIP_OFFLINE_THRESHOLD — days unreachable before auto-offboard
|
||
# PARTNERSHIP_TRANSFER_CONFIRM — exact string required for --transfer
|
||
# PARTNERSHIP_TRANSFER_STRIKES — consecutive health checks required
|
||
# PARTNERSHIP_TRANSFER_MAX_ATTEMPTS — max attempts before giving up
|
||
# PARTNERSHIP_ONBOARD_VERIFY — verify WebUI connectivity after onboard
|
||
# PARTNERSHIP_ONBOARD_NOTIFY — notify both servers on onboard completion
|
||
# PARTNERSHIP_SYNC_INTERVAL — informational — actual schedule in cron
|
||
# TAILSCALE_API_KEY / TAILSCALE_TAILNET — required when PARTNERSHIP_REMOVE_TAILSCALE=true
|
||
#
|
||
# ── BLOCKLIST ─────────────────────────────────────────────────────────────────────────────────
|
||
# After offboard, the former partner's hostname is written to:
|
||
# /boot/config/partnership_blocklist.db (format: hostname|timestamp|reason)
|
||
#
|
||
# --onboard is hard-blocked if the remote is on the blocklist — exits with error.
|
||
# --check silently skips remote state reads for blocklisted hosts (no noise every 15min).
|
||
# --unblock <hostname> removes an entry to permit re-onboarding.
|
||
# --status shows the full blocklist.
|
||
#
|
||
# The blocklist persists until explicitly cleared — surviving reboots, array restarts,
|
||
# and Tailscale reconnections. Tailscale removal is a separate step at the network layer;
|
||
# the blocklist is the application-layer guard.
|
||
#
|
||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||
# partnership_manage.sh --onboard
|
||
# partnership_manage.sh --offboard
|
||
# partnership_manage.sh --transfer --confirm=i-understand-this-transfers-ownership
|
||
# partnership_manage.sh --check --remote-seen|--remote-unseen
|
||
# partnership_manage.sh --status
|
||
# partnership_manage.sh --unblock <hostname>
|
||
# Any mode supports --dry-run and --log
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../load_config.sh"
|
||
|
||
SSH_TIMEOUT=15
|
||
BLOCKLIST_FILE="/boot/config/partnership_blocklist.db"
|
||
|
||
# ── Parse mode flags before parse_args ────────────────────────────────────────────────────────
|
||
MODE=""
|
||
TRANSFER_CONFIRM_INPUT=""
|
||
REMOTE_SEEN=false
|
||
REMOTE_UNSEEN=false
|
||
REASON="manual"
|
||
UNBLOCK_HOST=""
|
||
FILTERED_ARGS=()
|
||
|
||
for arg in "$@"; do
|
||
case "$arg" in
|
||
--onboard) MODE="onboard" ;;
|
||
--offboard) MODE="offboard" ;;
|
||
--transfer) MODE="transfer" ;;
|
||
--check) MODE="check" ;;
|
||
--status) MODE="status" ;;
|
||
--unblock) MODE="unblock" ;;
|
||
--confirm=*) TRANSFER_CONFIRM_INPUT="${arg#--confirm=}" ;;
|
||
--remote-seen) REMOTE_SEEN=true ;;
|
||
--remote-unseen) REMOTE_UNSEEN=true ;;
|
||
--reason=*) REASON="${arg#--reason=}" ;;
|
||
*)
|
||
if [[ "$MODE" == "unblock" ]] && [[ -z "$UNBLOCK_HOST" ]] && [[ "$arg" != --* ]]; then
|
||
UNBLOCK_HOST="$arg"
|
||
else
|
||
FILTERED_ARGS+=("$arg")
|
||
fi
|
||
;;
|
||
esac
|
||
done
|
||
|
||
parse_args "${FILTERED_ARGS[@]}"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
validate_unraid_cmd \
|
||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||
"" "" \
|
||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||
|
||
# detect_hosts() sets MY_ID and aliases PARTNERSHIP_AUTH_WEBUIS, PARTNERSHIP_MIRROR_BACKUPS
|
||
detect_hosts
|
||
|
||
# ── Derive owner and mirror from PARTNERSHIP_OWNER_HOST ───────────────────────────────────────
|
||
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" # e.g. "HOST1"
|
||
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
||
|
||
OWNER="${!OWNER_ID}" # hostname string
|
||
MIRROR="${!MIRROR_ID}"
|
||
OWNER_SSH_KEY_VAR="${OWNER_ID}_SSH_KEY"
|
||
MIRROR_SSH_KEY_VAR="${MIRROR_ID}_SSH_KEY"
|
||
OWNER_SSH_KEY="${!OWNER_SSH_KEY_VAR}"
|
||
MIRROR_SSH_KEY="${!MIRROR_SSH_KEY_VAR}"
|
||
|
||
AM_OWNER=false
|
||
AM_MIRROR=false
|
||
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
||
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
||
|
||
# State files
|
||
LOCAL_STATE_FILE="/boot/config/partnership_${LOCAL_SERVER_NAME}.db"
|
||
REMOTE_STATE_FILE="/boot/config/partnership_${REMOTE_SERVER_NAME}.db"
|
||
OWNER_STATE_FILE="/boot/config/partnership_${OWNER}.db"
|
||
MIRROR_STATE_FILE="/boot/config/partnership_${MIRROR}.db"
|
||
OFFLINE_COUNTER="/boot/config/partnership_offline_days.db"
|
||
|
||
if [[ -z "$MODE" ]]; then
|
||
error "No mode specified"
|
||
echo "Usage:"
|
||
echo " partnership_manage.sh --onboard"
|
||
echo " partnership_manage.sh --offboard"
|
||
echo " partnership_manage.sh --transfer --confirm=..."
|
||
echo " partnership_manage.sh --check --remote-seen|--remote-unseen"
|
||
echo " partnership_manage.sh --status"
|
||
echo " partnership_manage.sh --unblock <hostname>"
|
||
exit 1
|
||
fi
|
||
|
||
# Role-based access control
|
||
if [[ "$AM_MIRROR" == true ]]; then
|
||
case "$MODE" in
|
||
onboard|transfer)
|
||
error "Only the owner ($OWNER / $OWNER_ID) can run --$MODE"
|
||
error "Run from $OWNER or use --offboard to separate cleanly"
|
||
exit 1
|
||
;;
|
||
esac
|
||
fi
|
||
|
||
# Lock for all modes except --check (check is called frequently, lock would pile up)
|
||
[[ "$MODE" != "check" ]] && acquire_lock "strict"
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||
|
||
# ==============================================================================================
|
||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
read_state_file() {
|
||
local file="$1" key="$2"
|
||
grep "^${key}=" "$file" 2>/dev/null | cut -d= -f2
|
||
}
|
||
|
||
write_state_file() {
|
||
local file="$1"
|
||
shift
|
||
# flock prevents concurrent writes to the same state file
|
||
(
|
||
flock -x 200
|
||
cat > "$file" << EOF
|
||
state=${1:-UNKNOWN}
|
||
owner=${OWNER}
|
||
mirror=${MIRROR}
|
||
onboarded=${2:-}
|
||
offboarded=${3:-}
|
||
triggered_by=${4:-}
|
||
reason=${5:-}
|
||
last_seen_remote=${6:-}
|
||
updated=$(date '+%Y-%m-%d %H:%M:%S')
|
||
EOF
|
||
) 200>"${file}.lock"
|
||
}
|
||
|
||
push_state_to_remote() {
|
||
local local_file="$1" remote_ip="$2" ssh_key="$3"
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would push state file to remote"
|
||
return 0
|
||
fi
|
||
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||
"$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \
|
||
log "State file pushed to remote ✅" || \
|
||
warn "Could not push state file to remote — will propagate on next sync"
|
||
}
|
||
|
||
read_remote_state() {
|
||
local remote_ip="$1" ssh_key="$2" remote_file="$3"
|
||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
||
"cat '$remote_file' 2>/dev/null" 2>/dev/null
|
||
}
|
||
|
||
# Reconfigure a container's WebUI on the remote server
|
||
reconfigure_webui() {
|
||
local container="$1" port="$2" target_ip="$3"
|
||
local ssh_key="$4" remote_ip="$5" label="${6:-remote}"
|
||
|
||
log "Reconfiguring $container WebUI → ${target_ip}:${port} on $label..."
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would reconfigure $container WebUI to http://${target_ip}:${port}/"
|
||
return 0
|
||
fi
|
||
|
||
local template
|
||
template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
||
"grep -rl '<WebUI>' /boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \
|
||
xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null)
|
||
|
||
if [[ -z "$template" ]]; then
|
||
warn "$container template not found on $label — WebUI needs manual reconfiguration"
|
||
return 1
|
||
fi
|
||
|
||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
||
"sed -i 's|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g' '$template'" \
|
||
2>/dev/null && \
|
||
log "$container → http://${target_ip}:${port}/ ✅" || {
|
||
error "Failed to reconfigure $container WebUI on $label"
|
||
return 1
|
||
}
|
||
}
|
||
|
||
# Reconfigure local auth WebUIs to target IP
|
||
reconfigure_local_webuis() {
|
||
local target_ip="$1"
|
||
log "Reconfiguring local auth WebUIs → ${target_ip}..."
|
||
|
||
local failures=0
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
local container="${entry%%|*}"
|
||
local port="${entry##*|}"
|
||
|
||
local template
|
||
template=$(grep -rl '<WebUI>' \
|
||
/boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \
|
||
xargs grep -l "\"$container\"" 2>/dev/null | head -1)
|
||
|
||
if [[ -z "$template" ]]; then
|
||
warn "$container template not found locally"
|
||
(( failures++ ))
|
||
continue
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would reconfigure $container → http://${target_ip}:${port}/"
|
||
continue
|
||
fi
|
||
|
||
sed -i "s|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g" \
|
||
"$template" 2>/dev/null && \
|
||
log "$container → http://${target_ip}:${port}/ ✅" || \
|
||
{ error "Failed to reconfigure $container"; (( failures++ )); }
|
||
done
|
||
return $failures
|
||
}
|
||
|
||
get_tailscale_device_id() {
|
||
local hostname="$1"
|
||
curl -sf --max-time 10 \
|
||
-H "Authorization: Bearer $TAILSCALE_API_KEY" \
|
||
"https://api.tailscale.com/api/v2/tailnet/${TAILSCALE_TAILNET}/devices" \
|
||
2>/dev/null | \
|
||
grep -o "\"id\":\"[^\"]*\"[^}]*\"hostname\":\"${hostname}\"" | \
|
||
grep -o '"id":"[^"]*"' | \
|
||
grep -o '[^"]*"$' | tr -d '"'
|
||
}
|
||
|
||
remove_tailscale_device() {
|
||
local hostname="$1"
|
||
|
||
if [[ -z "${TAILSCALE_API_KEY:-}" ]] || [[ -z "${TAILSCALE_TAILNET:-}" ]]; then
|
||
warn "TAILSCALE_API_KEY or TAILSCALE_TAILNET not configured — skipping Tailscale removal"
|
||
return 1
|
||
fi
|
||
|
||
log "Looking up Tailscale device ID for $hostname..."
|
||
local device_id
|
||
device_id=$(get_tailscale_device_id "$hostname")
|
||
|
||
if [[ -z "$device_id" ]]; then
|
||
warn "$hostname not found in Tailscale — may already be removed"
|
||
return 0
|
||
fi
|
||
|
||
log "Removing $hostname (device $device_id) from Tailscale..."
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove $hostname from Tailscale tailnet"
|
||
return 0
|
||
fi
|
||
|
||
curl -sf --max-time 10 -X DELETE \
|
||
-H "Authorization: Bearer $TAILSCALE_API_KEY" \
|
||
"https://api.tailscale.com/api/v2/devices/${device_id}" 2>/dev/null && \
|
||
warn "$hostname removed from Tailscale ✅" || \
|
||
error "Failed to remove $hostname from Tailscale — remove manually"
|
||
}
|
||
|
||
# ── Blocklist helpers ─────────────────────────────────────────────────────────────────────────
|
||
|
||
is_blocklisted() {
|
||
local hostname="$1"
|
||
[[ -f "$BLOCKLIST_FILE" ]] && grep -q "^${hostname}|" "$BLOCKLIST_FILE" 2>/dev/null
|
||
}
|
||
|
||
add_to_blocklist() {
|
||
local hostname="$1" reason="${2:-offboard}"
|
||
local now
|
||
now=$(date '+%Y-%m-%d %H:%M:%S')
|
||
if ! is_blocklisted "$hostname"; then
|
||
echo "${hostname}|${now}|${reason}" >> "$BLOCKLIST_FILE"
|
||
log "Blocklisted: $hostname (reason: $reason)"
|
||
else
|
||
log "$hostname already on blocklist"
|
||
fi
|
||
}
|
||
|
||
remove_from_blocklist() {
|
||
local hostname="$1"
|
||
if [[ ! -f "$BLOCKLIST_FILE" ]]; then
|
||
log "Blocklist empty — nothing to remove"
|
||
return 0
|
||
fi
|
||
sed -i "/^${hostname}|/d" "$BLOCKLIST_FILE" 2>/dev/null
|
||
log "Removed from blocklist: $hostname"
|
||
}
|
||
|
||
# Revoke SSH access on both sides — call after all other SSH operations complete.
|
||
# Matches by key comment (format: keyname@hostname — set by ssh_setup.sh at keygen time).
|
||
# SSH_REVOKE_REMOTE_OK / SSH_REVOKE_LOCAL_OK set in caller scope for summary display.
|
||
do_ssh_key_revocation() {
|
||
local remote_ip="$1"
|
||
local pub_key_file="${SSH_KEY}.pub"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD SSH Key Revocation ━━━"
|
||
|
||
# Step 1: remove our pubkey from remote's authorized_keys while SSH still works
|
||
SSH_REVOKE_REMOTE_OK=false
|
||
if [[ -f "$pub_key_file" ]]; then
|
||
local our_comment
|
||
our_comment=$(awk '{print $3}' "$pub_key_file" 2>/dev/null)
|
||
if [[ -n "$our_comment" ]]; then
|
||
log "Revoking our pubkey ($our_comment) from $REMOTE_SERVER_NAME..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove our pubkey from $REMOTE_SERVER_NAME authorized_keys"
|
||
SSH_REVOKE_REMOTE_OK=true
|
||
elif [[ -n "$remote_ip" ]] && timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$remote_ip" \
|
||
"grep -v '${our_comment}' /root/.ssh/authorized_keys \
|
||
> /root/.ssh/authorized_keys.tmp 2>/dev/null \
|
||
&& mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys \
|
||
&& echo removed" 2>/dev/null | grep -q removed; then
|
||
log "Our pubkey revoked from $REMOTE_SERVER_NAME ✅"
|
||
SSH_REVOKE_REMOTE_OK=true
|
||
else
|
||
warn "Remote revocation failed — revoke manually on $REMOTE_SERVER_NAME:"
|
||
warn " grep -v '@${LOCAL_SERVER_NAME}' /root/.ssh/authorized_keys > /root/.ssh/authorized_keys"
|
||
fi
|
||
else
|
||
warn "Could not read pubkey comment from $pub_key_file — skipping remote revocation"
|
||
fi
|
||
else
|
||
warn "Pubkey not found at $pub_key_file — skipping remote revocation"
|
||
fi
|
||
|
||
# Step 2: remove remote's pubkey from our local authorized_keys
|
||
# Remote key comment ends with @REMOTE_SERVER_NAME — unique, no special chars
|
||
SSH_REVOKE_LOCAL_OK=false
|
||
log "Revoking $REMOTE_SERVER_NAME pubkey from local authorized_keys..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove $REMOTE_SERVER_NAME pubkey from local authorized_keys"
|
||
SSH_REVOKE_LOCAL_OK=true
|
||
elif [[ -f /root/.ssh/authorized_keys ]]; then
|
||
if grep -q "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys 2>/dev/null; then
|
||
if grep -v "@${REMOTE_SERVER_NAME}" /root/.ssh/authorized_keys \
|
||
> /root/.ssh/authorized_keys.tmp 2>/dev/null && \
|
||
mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys; then
|
||
log "$REMOTE_SERVER_NAME pubkey revoked locally ✅"
|
||
SSH_REVOKE_LOCAL_OK=true
|
||
else
|
||
warn "Failed to update local authorized_keys — remove @${REMOTE_SERVER_NAME} entry manually"
|
||
fi
|
||
else
|
||
log "$REMOTE_SERVER_NAME pubkey not found in local authorized_keys — already removed"
|
||
SSH_REVOKE_LOCAL_OK=true
|
||
fi
|
||
else
|
||
log "/root/.ssh/authorized_keys not found — nothing to remove locally"
|
||
SSH_REVOKE_LOCAL_OK=true
|
||
fi
|
||
}
|
||
|
||
# ── FolderView3 integration ───────────────────────────────────────────────────────────────────
|
||
FOLDERVIEW3_DIR="/usr/local/emhttp/plugins/folder.view3"
|
||
FOLDERVIEW3_JSON="/boot/config/plugins/folder.view3/docker.json"
|
||
|
||
# Derive short partner name: strip unraid- prefix (case-insensitive) if present
|
||
derive_partner_folder_name() {
|
||
local hostname="$1"
|
||
local short="${hostname,,}"
|
||
[[ "$short" == unraid-* ]] && short="${short:7}"
|
||
# Capitalise first char for readability: jayred365 → Jayred365-Failover
|
||
echo "${short^}-Fallback"
|
||
}
|
||
|
||
folderview3_ensure_plugin() {
|
||
if [[ -d "$FOLDERVIEW3_DIR" ]]; then
|
||
log "FolderView3 plugin present ✅"
|
||
return 0
|
||
fi
|
||
if [[ -z "${PARTNERSHIP_FOLDERVIEW3_URL:-}" ]]; then
|
||
warn "FolderView3 plugin not installed and PARTNERSHIP_FOLDERVIEW3_URL is empty"
|
||
warn "Install manually from Community Applications or set PARTNERSHIP_FOLDERVIEW3_URL in master.conf"
|
||
return 1
|
||
fi
|
||
warn "FolderView3 not found — installing from CA..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would run: plugin install $PARTNERSHIP_FOLDERVIEW3_URL"
|
||
return 0
|
||
fi
|
||
plugin install "$PARTNERSHIP_FOLDERVIEW3_URL" 2>/dev/null && \
|
||
log "FolderView3 installed ✅" || {
|
||
warn "FolderView3 install failed — folder will not be created"
|
||
return 1
|
||
}
|
||
}
|
||
|
||
folderview3_create_partner_folder() {
|
||
local partner_name="$1"
|
||
shift
|
||
local containers=("$@")
|
||
|
||
log "FolderView3: creating folder '$partner_name' with ${#containers[@]} container(s)..."
|
||
|
||
folderview3_ensure_plugin || return 1
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would create FolderView3 folder: $partner_name"
|
||
for c in "${containers[@]}"; do
|
||
[[ -n "$c" ]] && warn " DRY RUN — container: $c"
|
||
done
|
||
return 0
|
||
fi
|
||
|
||
# Initialise JSON if missing
|
||
if [[ ! -f "$FOLDERVIEW3_JSON" ]]; then
|
||
mkdir -p "$(dirname "$FOLDERVIEW3_JSON")"
|
||
echo '{}' > "$FOLDERVIEW3_JSON"
|
||
fi
|
||
|
||
# Check folder doesn't already exist
|
||
local existing
|
||
existing=$(jq -r --arg name "$partner_name" \
|
||
'to_entries[] | select(.value.name == $name) | .key' \
|
||
"$FOLDERVIEW3_JSON" 2>/dev/null)
|
||
if [[ -n "$existing" ]]; then
|
||
log "FolderView3: folder '$partner_name' already exists (id: $existing) — skipping"
|
||
return 0
|
||
fi
|
||
|
||
# Build containers JSON array
|
||
local containers_json
|
||
containers_json=$(printf '%s\n' "${containers[@]}" | \
|
||
grep -v '^$' | jq -R . | jq -s .)
|
||
|
||
# Generate a stable random ID from timestamp+name
|
||
local folder_id
|
||
folder_id=$(echo "${partner_name}$(date +%s%N)" | md5sum | head -c 12)
|
||
|
||
jq --arg id "$folder_id" --arg name "$partner_name" \
|
||
--argjson containers "$containers_json" \
|
||
'.[$id] = {"name": $name, "containers": $containers, "containerImages": {}}' \
|
||
"$FOLDERVIEW3_JSON" > "${FOLDERVIEW3_JSON}.tmp" && \
|
||
mv "${FOLDERVIEW3_JSON}.tmp" "$FOLDERVIEW3_JSON" && \
|
||
log "FolderView3: folder '$partner_name' created with ${#containers[@]} container(s) ✅" || {
|
||
warn "FolderView3: failed to write JSON — check $FOLDERVIEW3_JSON"
|
||
return 1
|
||
}
|
||
}
|
||
|
||
folderview3_remove_partner_folder() {
|
||
local partner_name="$1"
|
||
|
||
log "FolderView3: removing folder '$partner_name' and stopping its containers..."
|
||
|
||
if [[ ! -f "$FOLDERVIEW3_JSON" ]]; then
|
||
log "FolderView3: JSON not found — nothing to remove"
|
||
return 0
|
||
fi
|
||
|
||
if ! command -v jq >/dev/null 2>&1; then
|
||
warn "jq not found — cannot manage FolderView3 JSON"
|
||
return 1
|
||
fi
|
||
|
||
# Get containers from this folder
|
||
local containers_json
|
||
containers_json=$(jq -r --arg name "$partner_name" \
|
||
'to_entries[] | select(.value.name == $name) | .value.containers[]' \
|
||
"$FOLDERVIEW3_JSON" 2>/dev/null)
|
||
|
||
if [[ -z "$containers_json" ]]; then
|
||
log "FolderView3: folder '$partner_name' not found — nothing to remove"
|
||
return 0
|
||
fi
|
||
|
||
# Stop and remove each container in the folder
|
||
local stopped=0 removed=0 failed=0
|
||
while IFS= read -r container; do
|
||
[[ -z "$container" ]] && continue
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would stop + rm: $container"
|
||
continue
|
||
fi
|
||
# Stop if running
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1; then
|
||
log "$container stopped ✅"
|
||
(( stopped++ ))
|
||
else
|
||
warn "$container stop failed"
|
||
(( failed++ ))
|
||
fi
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1; then
|
||
log "$container removed ✅"
|
||
(( removed++ ))
|
||
else
|
||
warn "$container rm failed (may not exist)"
|
||
fi
|
||
else
|
||
log "$container not found locally — skipping"
|
||
fi
|
||
done <<< "$containers_json"
|
||
|
||
# Remove folder entry from JSON
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
jq --arg name "$partner_name" \
|
||
'with_entries(select(.value.name != $name))' \
|
||
"$FOLDERVIEW3_JSON" > "${FOLDERVIEW3_JSON}.tmp" && \
|
||
mv "${FOLDERVIEW3_JSON}.tmp" "$FOLDERVIEW3_JSON" && \
|
||
log "FolderView3: folder '$partner_name' removed ✅" || \
|
||
warn "FolderView3: failed to remove folder from JSON"
|
||
else
|
||
warn "DRY RUN — would remove folder '$partner_name' from $FOLDERVIEW3_JSON"
|
||
fi
|
||
|
||
[[ "$DRY_RUN" == false ]] && \
|
||
log "FolderView3 cleanup: $stopped stopped, $removed removed, $failed failed"
|
||
return 0
|
||
}
|
||
|
||
# Gather all partner failover containers for this server (all tiers)
|
||
gather_partner_fallback_containers() {
|
||
local out_var="$1"
|
||
eval "${out_var}=()"
|
||
local tier var
|
||
for tier in TIER1 TIER2 TIER3 TIER4; do
|
||
var="FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_${tier}"
|
||
# Check if var is set and is an array
|
||
if declare -p "$var" >/dev/null 2>&1; then
|
||
local -a _tmp_arr
|
||
eval "_tmp_arr=(\"\${${var}[@]}\")"
|
||
for c in "${_tmp_arr[@]}"; do
|
||
[[ -n "$c" ]] && eval "${out_var}+=(\"\$c\")"
|
||
done
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Read a scalar var from the mirror's own config via SSH.
|
||
# Sources load_config.sh + detect_hosts() on the remote so HOST* aliasing works.
|
||
read_remote_conf_var() {
|
||
local mirror_ip="$1" var_name="$2"
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
|
||
detect_hosts 2>/dev/null
|
||
printf '%s' \"\${${var_name}:-}\"" 2>/dev/null
|
||
}
|
||
|
||
# Read an array var from the mirror's own config via SSH — one element per line.
|
||
read_remote_conf_array() {
|
||
local mirror_ip="$1" var_name="$2"
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" -o BatchMode=yes root@"$mirror_ip" \
|
||
"source '$SCRIPT_DIR/../load_config.sh' 2>/dev/null
|
||
detect_hosts 2>/dev/null
|
||
printf '%s\n' \"\${${var_name}[@]:-}\"" 2>/dev/null
|
||
}
|
||
|
||
# Returns just the short name portion: "unRAID-Gmer4Lfe" → "Gmer4Lfe"
|
||
derive_short_name() {
|
||
local hostname="$1"
|
||
local short="${hostname,,}"
|
||
[[ "$short" == unraid-* ]] && short="${short:7}"
|
||
echo "${short^}"
|
||
}
|
||
|
||
# Start this server's own parked containers after partnership ends.
|
||
start_own_stack() {
|
||
echo ""
|
||
echo "━━━ $ICON_START Restart Own Stack ━━━"
|
||
if [[ ${#PARTNERSHIP_OWN_CONTAINERS[@]} -eq 0 ]]; then
|
||
log "No PARTNERSHIP_OWN_CONTAINERS configured — skipping"
|
||
return 0
|
||
fi
|
||
for container in "${PARTNERSHIP_OWN_CONTAINERS[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would start: $container"
|
||
continue
|
||
fi
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker start "$container" >/dev/null 2>&1; then
|
||
log "$container started ✅"
|
||
else
|
||
warn "$container failed to start — check manually"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# Remove partnership containers on this server.
|
||
# Uses FolderView3 folder if enabled (precise list), else falls back to FALLBACK_*_COVERS_* config.
|
||
cleanup_partner_containers() {
|
||
local folder_name="$1"
|
||
if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then
|
||
folderview3_remove_partner_folder "$folder_name"
|
||
else
|
||
declare -a containers=()
|
||
gather_partner_fallback_containers containers
|
||
if [[ ${#containers[@]} -eq 0 ]]; then
|
||
log "No partner containers found to remove"
|
||
return 0
|
||
fi
|
||
for container in "${containers[@]}"; do
|
||
[[ -z "$container" ]] && continue
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would stop + rm: $container"
|
||
continue
|
||
fi
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then
|
||
timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1 || true
|
||
timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1 && \
|
||
log "$container removed ✅" || warn "$container rm failed"
|
||
else
|
||
log "$container not found — skipping"
|
||
fi
|
||
done
|
||
fi
|
||
}
|
||
|
||
# SSH to mirror — remove all containers named *-${OWNER_SHORT} (owner's deployed containers).
|
||
cleanup_owner_containers_on_mirror() {
|
||
local mirror_ip="$1"
|
||
local owner_short
|
||
owner_short=$(derive_short_name "$OWNER")
|
||
|
||
log "Removing owner-deployed containers from $MIRROR..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove *-${owner_short} containers from $MIRROR"
|
||
return 0
|
||
fi
|
||
|
||
local container_list
|
||
container_list=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
||
"docker ps -a --format '{{.Names}}' 2>/dev/null | grep -i -- '-${owner_short}$'" 2>/dev/null)
|
||
|
||
if [[ -z "$container_list" ]]; then
|
||
log "No *-${owner_short} containers found on $MIRROR"
|
||
return 0
|
||
fi
|
||
|
||
while IFS= read -r container; do
|
||
[[ -z "$container" ]] && continue
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
||
"docker stop '$container' >/dev/null 2>&1
|
||
docker rm '$container' >/dev/null 2>&1 && echo removed" 2>/dev/null | \
|
||
grep -q removed && \
|
||
log "$container removed from $MIRROR ✅" || \
|
||
warn "Failed to remove $container from $MIRROR"
|
||
done <<< "$container_list"
|
||
}
|
||
|
||
# SSH to mirror — start mirror's own parked containers.
|
||
# Reads PARTNERSHIP_OWN_CONTAINERS from the mirror's own conf via SSH.
|
||
start_mirror_own_stack() {
|
||
local mirror_ip="$1"
|
||
|
||
log "Reading own stack list from $MIRROR conf..."
|
||
local -a mirror_own=()
|
||
mapfile -t mirror_own < <(read_remote_conf_array "$mirror_ip" "PARTNERSHIP_OWN_CONTAINERS")
|
||
# Remove empty entries
|
||
local -a filtered=()
|
||
for c in "${mirror_own[@]}"; do [[ -n "$c" ]] && filtered+=("$c"); done
|
||
mirror_own=("${filtered[@]}")
|
||
|
||
if [[ ${#mirror_own[@]} -eq 0 ]]; then
|
||
log "No PARTNERSHIP_OWN_CONTAINERS configured on $MIRROR — skipping remote stack restart"
|
||
return 0
|
||
fi
|
||
|
||
log "Restarting own stack on $MIRROR: ${mirror_own[*]}"
|
||
for container in "${mirror_own[@]}"; do
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would start $container on $MIRROR"
|
||
continue
|
||
fi
|
||
timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
||
"docker start '$container' >/dev/null 2>&1 && echo started" 2>/dev/null | \
|
||
grep -q started && \
|
||
log "$container started on $MIRROR ✅" || \
|
||
warn "$container failed to start on $MIRROR — check manually"
|
||
done
|
||
}
|
||
|
||
# Create the mirror's Emby admin account on the owner's deployed Emby.
|
||
# Credentials come from the MIRROR's conf (HOST*_PARTNERSHIP_EMBY_ADMIN_USER/PASS).
|
||
# Checks for username collision before creating — exits with guidance if taken.
|
||
provision_emby_admin() {
|
||
local mirror_ip="$1"
|
||
local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}"
|
||
local emby_url="http://${mirror_ip}:${emby_port}"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_EMBY Emby Admin Provisioning ━━━"
|
||
|
||
if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then
|
||
log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping"
|
||
return 0
|
||
fi
|
||
|
||
if [[ -z "$EMBY_API_KEY" ]]; then
|
||
warn "EMBY_API_KEY not set — skipping Emby admin provisioning"
|
||
return 1
|
||
fi
|
||
|
||
# Read mirror's desired credentials from their own conf via SSH
|
||
log "Reading Emby credentials from $MIRROR conf..."
|
||
local username password
|
||
username=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_USER")
|
||
password=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_PASS")
|
||
[[ -z "$username" ]] && username="$(derive_short_name "$MIRROR")"
|
||
|
||
if [[ -z "$password" ]]; then
|
||
warn "PARTNERSHIP_EMBY_ADMIN_PASS is empty in $MIRROR conf"
|
||
warn "$MIRROR must set HOST${MIRROR_ID: -1}_PARTNERSHIP_EMBY_ADMIN_PASS before onboard"
|
||
return 1
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would create Emby admin '$username' at $emby_url"
|
||
return 0
|
||
fi
|
||
|
||
# Collision check — username already exists?
|
||
local existing_users
|
||
existing_users=$(curl -sf --max-time 15 \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
"${emby_url}/Users" 2>/dev/null)
|
||
|
||
if echo "$existing_users" | grep -q "\"Name\":\"${username}\""; then
|
||
error "Emby username '${username}' is already taken on the shared instance"
|
||
error "Options:"
|
||
error " 1. Sign in with that account — it may already be yours"
|
||
error " 2. Set a different name in ${user_var} and re-run --onboard"
|
||
return 1
|
||
fi
|
||
|
||
log "Creating Emby admin '$username' at $emby_url..."
|
||
|
||
local create_response http_code body
|
||
create_response=$(curl -sf --max-time 15 -w "\n%{http_code}" \
|
||
-X POST \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
-H "Content-Type: application/json" \
|
||
-d "{\"Name\": \"$username\"}" \
|
||
"${emby_url}/Users/New" 2>/dev/null)
|
||
http_code=$(echo "$create_response" | tail -1)
|
||
body=$(echo "$create_response" | head -n -1)
|
||
|
||
if [[ "$http_code" != "200" ]] && [[ "$http_code" != "204" ]]; then
|
||
warn "Failed to create Emby user '$username' (HTTP $http_code)"
|
||
return 1
|
||
fi
|
||
|
||
local user_id
|
||
user_id=$(echo "$body" | grep -o '"Id":"[^"]*"' | head -1 | cut -d'"' -f4)
|
||
if [[ -z "$user_id" ]]; then
|
||
warn "Emby user created but could not parse user ID — set password manually"
|
||
return 1
|
||
fi
|
||
|
||
local pw_code
|
||
pw_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \
|
||
-X POST \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
-H "Content-Type: application/json" \
|
||
-d "{\"NewPw\": \"$password\"}" \
|
||
"${emby_url}/Users/${user_id}/Password" 2>/dev/null)
|
||
|
||
if [[ "$pw_code" == "200" ]] || [[ "$pw_code" == "204" ]]; then
|
||
log "Emby admin '$username' created (id: $user_id) ✅"
|
||
else
|
||
warn "User created but password set failed (HTTP $pw_code) — set password manually"
|
||
fi
|
||
|
||
curl -sf --max-time 15 -o /dev/null \
|
||
-X POST \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
-H "Content-Type: application/json" \
|
||
-d '{"IsAdministrator": true, "IsDisabled": false}' \
|
||
"${emby_url}/Users/${user_id}/Policy" 2>/dev/null && \
|
||
log "$username granted admin policy ✅" || \
|
||
warn "Could not set admin policy — grant manually in Emby dashboard"
|
||
}
|
||
|
||
# Delete the mirror's Emby admin account from the owner's deployed Emby.
|
||
revoke_emby_admin() {
|
||
local mirror_ip="$1"
|
||
local emby_port="${PARTNERSHIP_EMBY_PORT:-8096}"
|
||
local emby_url="http://${mirror_ip}:${emby_port}"
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_EMBY Emby Admin Revocation ━━━"
|
||
|
||
local username
|
||
username=$(read_remote_conf_var "$mirror_ip" "PARTNERSHIP_EMBY_ADMIN_USER")
|
||
[[ -z "$username" ]] && username="$(derive_short_name "$MIRROR")"
|
||
|
||
if [[ "${PARTNERSHIP_PROVISION_EMBY_ADMIN:-false}" != true ]]; then
|
||
log "PARTNERSHIP_PROVISION_EMBY_ADMIN=false — skipping"
|
||
return 0
|
||
fi
|
||
|
||
if [[ -z "$EMBY_API_KEY" ]]; then
|
||
warn "EMBY_API_KEY not set — skipping Emby admin revocation"
|
||
return 1
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would delete Emby admin '$username' at $emby_url"
|
||
return 0
|
||
fi
|
||
|
||
log "Looking up Emby user '$username' at $emby_url..."
|
||
|
||
local users_json user_id
|
||
users_json=$(curl -sf --max-time 15 \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
"${emby_url}/Users" 2>/dev/null)
|
||
|
||
user_id=$(echo "$users_json" | \
|
||
grep -o "\"Id\":\"[^\"]*\"[^}]*\"Name\":\"${username}\"" | \
|
||
grep -o '"Id":"[^"]*"' | cut -d'"' -f4 | head -1)
|
||
|
||
if [[ -z "$user_id" ]]; then
|
||
warn "Emby user '$username' not found at $emby_url — may already be removed"
|
||
return 0
|
||
fi
|
||
|
||
local del_code
|
||
del_code=$(curl -sf --max-time 15 -w "%{http_code}" -o /dev/null \
|
||
-X DELETE \
|
||
-H "X-Emby-Authorization: MediaBrowser Token=\"$EMBY_API_KEY\"" \
|
||
"${emby_url}/Users/${user_id}" 2>/dev/null)
|
||
|
||
if [[ "$del_code" == "200" ]] || [[ "$del_code" == "204" ]] || [[ "$del_code" == "404" ]]; then
|
||
log "Emby admin '$username' removed ✅"
|
||
else
|
||
warn "Failed to delete Emby user '$username' (HTTP $del_code) — remove manually"
|
||
fi
|
||
}
|
||
|
||
check_both_healthy() {
|
||
mountpoint -q /mnt/user 2>/dev/null || { error "Local array not healthy"; return 1; }
|
||
|
||
local mirror_ip
|
||
mirror_ip=$(resolve_tailscale_ip "$MIRROR")
|
||
[[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }
|
||
|
||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
||
"mountpoint -q /mnt/user && timeout 10 docker ps" >/dev/null 2>&1 || {
|
||
error "Mirror $MIRROR not healthy"
|
||
return 1
|
||
}
|
||
return 0
|
||
}
|
||
|
||
do_final_sync() {
|
||
log "Running final Critical-Data and Emby sync..."
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
# Use CRITICAL_SYNC_SHARES from HOST* conf if available
|
||
# Falls back to known paths — these are the critical ones
|
||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
|
||
"/mnt/user/appdata-Fallback/Critical-Data" \
|
||
--profile=critical-fallback --log
|
||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
|
||
"/mnt/user/Media_Server/Emby" \
|
||
--profile=emby-fallback --log
|
||
else
|
||
warn "DRY RUN — would run final Critical-Data and Emby sync"
|
||
fi
|
||
warn "Final sync complete — mirror has current state ✅"
|
||
}
|
||
|
||
# Safe master.conf modification with error handling
|
||
update_master_conf() {
|
||
local key="$1" value="$2"
|
||
local conf="$SCRIPT_DIR/../master.conf"
|
||
if [[ ! -f "$conf" ]]; then
|
||
error "master.conf not found at $conf"
|
||
return 1
|
||
fi
|
||
if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then
|
||
log "master.conf updated: ${key}=${value}"
|
||
return 0
|
||
else
|
||
error "Failed to update master.conf: ${key}=${value}"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Unblock ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "unblock" ]]; then
|
||
if [[ -z "$UNBLOCK_HOST" ]]; then
|
||
error "Usage: partnership_manager.sh --unblock <hostname>"
|
||
error "Example: partnership_manager.sh --unblock unRAID-Jayred365"
|
||
if [[ -f "$BLOCKLIST_FILE" ]] && [[ -s "$BLOCKLIST_FILE" ]]; then
|
||
echo ""
|
||
echo "Currently blocklisted:"
|
||
while IFS='|' read -r host ts reason; do
|
||
echo " $host (blocked $ts — $reason)"
|
||
done < "$BLOCKLIST_FILE"
|
||
else
|
||
echo "Blocklist is empty."
|
||
fi
|
||
exit 1
|
||
fi
|
||
|
||
if ! is_blocklisted "$UNBLOCK_HOST"; then
|
||
warn "$UNBLOCK_HOST is not on the blocklist"
|
||
exit 0
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove $UNBLOCK_HOST from blocklist"
|
||
exit 0
|
||
fi
|
||
|
||
remove_from_blocklist "$UNBLOCK_HOST"
|
||
warn "$UNBLOCK_HOST unblocked — re-onboarding is now permitted ✅"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "status" ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY PARTNERSHIP STATUS ━━━━━"
|
||
OWNER_IP=$(resolve_tailscale_ip "$OWNER" || echo "unreachable")
|
||
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR" || echo "unreachable")
|
||
echo " $ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo " Owner: $OWNER_ID ($OWNER — $OWNER_IP)"
|
||
echo " Mirror: $MIRROR_ID ($MIRROR — $MIRROR_IP)"
|
||
echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
|
||
echo " Enabled: ${PARTNERSHIP_ENABLED:-false}"
|
||
echo ""
|
||
|
||
# Local state
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||
LOCAL_ONBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "onboarded")
|
||
LOCAL_OFFBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "offboarded")
|
||
LOCAL_LAST_SEEN=$(read_state_file "$LOCAL_STATE_FILE" "last_seen_remote")
|
||
echo " Local state: $LOCAL_STATE"
|
||
[[ -n "$LOCAL_ONBOARDED" ]] && echo " Onboarded: $LOCAL_ONBOARDED"
|
||
[[ -n "$LOCAL_OFFBOARDED" ]] && echo " Offboarded: $LOCAL_OFFBOARDED"
|
||
[[ -n "$LOCAL_LAST_SEEN" ]] && echo " Remote seen: $LOCAL_LAST_SEEN"
|
||
else
|
||
echo " Local state: no state file found"
|
||
fi
|
||
|
||
echo ""
|
||
|
||
# Remote state
|
||
REMOTE_IP=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME")
|
||
if [[ -n "$REMOTE_IP" ]]; then
|
||
REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE")
|
||
if [[ -n "$REMOTE_CONTENT" ]]; then
|
||
REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2)
|
||
REMOTE_ONBOARDED=$(echo "$REMOTE_CONTENT" | grep "^onboarded=" | cut -d= -f2)
|
||
echo " Remote state: $REMOTE_STATE"
|
||
[[ -n "$REMOTE_ONBOARDED" ]] && echo " Onboarded: $REMOTE_ONBOARDED"
|
||
if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]]; then
|
||
echo ""
|
||
echo " ✅ Both servers agree: $LOCAL_STATE"
|
||
else
|
||
echo ""
|
||
echo " ⚠️ State mismatch — local: $LOCAL_STATE remote: $REMOTE_STATE"
|
||
fi
|
||
else
|
||
echo " Remote state: not found on $REMOTE_SERVER_NAME"
|
||
fi
|
||
else
|
||
echo " Remote state: $REMOTE_SERVER_NAME unreachable"
|
||
fi
|
||
|
||
echo ""
|
||
echo " Auth WebUIs:"
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
echo " ${entry%%|*} → port ${entry##*|}"
|
||
done
|
||
|
||
if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then
|
||
echo ""
|
||
FOLDER_STATUS_NAME=$(derive_partner_folder_name "$REMOTE_SERVER_NAME")
|
||
if [[ -f "$FOLDERVIEW3_JSON" ]]; then
|
||
FOLDER_EXISTS=$(jq -r --arg name "$FOLDER_STATUS_NAME" \
|
||
'to_entries[] | select(.value.name == $name) | .key' \
|
||
"$FOLDERVIEW3_JSON" 2>/dev/null)
|
||
if [[ -n "$FOLDER_EXISTS" ]]; then
|
||
FOLDER_CONTAINERS=$(jq -r --arg name "$FOLDER_STATUS_NAME" \
|
||
'[to_entries[] | select(.value.name == $name) | .value.containers[]] | join(", ")' \
|
||
"$FOLDERVIEW3_JSON" 2>/dev/null)
|
||
echo " FolderView3: $FOLDER_STATUS_NAME ✅"
|
||
echo " Containers: $FOLDER_CONTAINERS"
|
||
else
|
||
echo " FolderView3: $FOLDER_STATUS_NAME — not found in JSON"
|
||
fi
|
||
else
|
||
echo " FolderView3: config not found ($FOLDERVIEW3_JSON)"
|
||
fi
|
||
fi
|
||
|
||
if [[ -f "$OFFLINE_COUNTER" ]]; then
|
||
OFFLINE_DAYS=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0)
|
||
[[ "$OFFLINE_DAYS" -gt 0 ]] && \
|
||
echo "" && \
|
||
echo " ⚠️ Remote offline counter: ${OFFLINE_DAYS}/${PARTNERSHIP_OFFLINE_THRESHOLD}"
|
||
fi
|
||
|
||
echo ""
|
||
if [[ -f "$BLOCKLIST_FILE" ]] && [[ -s "$BLOCKLIST_FILE" ]]; then
|
||
echo " Blocklist:"
|
||
while IFS='|' read -r host ts reason; do
|
||
echo " ⛔ $host (blocked $ts — $reason)"
|
||
done < "$BLOCKLIST_FILE"
|
||
else
|
||
echo " Blocklist: empty"
|
||
fi
|
||
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Check ━━━
|
||
# Called every 15min by critical_sync_maintenance.sh — must be silent when healthy
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "check" ]]; then
|
||
|
||
# Update last_seen_remote and offline counter based on rsync outcome
|
||
if [[ "$REMOTE_SEEN" == true ]]; then
|
||
echo "0" > "$OFFLINE_COUNTER"
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
sed -i "s|^last_seen_remote=.*|last_seen_remote=$(date '+%Y-%m-%d %H:%M:%S')|" \
|
||
"$LOCAL_STATE_FILE" 2>/dev/null
|
||
fi
|
||
elif [[ "$REMOTE_UNSEEN" == true ]]; then
|
||
OFFLINE_COUNT=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0)
|
||
OFFLINE_COUNT=$(( OFFLINE_COUNT + 1 ))
|
||
echo "$OFFLINE_COUNT" > "$OFFLINE_COUNTER"
|
||
|
||
# Auto-offboard threshold: threshold_days × 96 intervals/day (every 15min)
|
||
THRESHOLD_INTERVALS=$(( ${PARTNERSHIP_OFFLINE_THRESHOLD:-30} * 96 ))
|
||
if [[ "$OFFLINE_COUNT" -ge "$THRESHOLD_INTERVALS" ]]; then
|
||
warn "Remote offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days — triggering auto-offboard"
|
||
notify "Partnership auto-offboard on $(hostname) — $REMOTE_SERVER_NAME offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days" \
|
||
"Partnership" "warning"
|
||
bash "$0" --offboard --reason=auto-offboard-timeout
|
||
exit 0
|
||
fi
|
||
log "Partnership check — remote unseen ($OFFLINE_COUNT/$THRESHOLD_INTERVALS)"
|
||
fi
|
||
|
||
# Skip blocklisted partners — they can't auto-reconnect; only --onboard re-establishes
|
||
if is_blocklisted "$REMOTE_SERVER_NAME"; then
|
||
log "Partnership check — $REMOTE_SERVER_NAME is blocklisted, skipping"
|
||
exit 0
|
||
fi
|
||
|
||
# Read remote state file
|
||
REMOTE_IP=$(resolve_tailscale_ip "$REMOTE_SERVER_NAME")
|
||
if [[ -z "$REMOTE_IP" ]]; then
|
||
log "Partnership check — remote unreachable, skipping state check"
|
||
exit 0
|
||
fi
|
||
|
||
REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE")
|
||
if [[ -z "$REMOTE_CONTENT" ]]; then
|
||
# IP resolved but SSH returned nothing — could be auth failure, not just missing file
|
||
if [[ -f "$SCRIPT_DIR/../Initial_run/ssh_setup.sh" ]]; then
|
||
bash "$SCRIPT_DIR/../Initial_run/ssh_setup.sh" --validate 2>/dev/null || true
|
||
fi
|
||
log "Partnership check — remote state file not found"
|
||
exit 0
|
||
fi
|
||
|
||
REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2)
|
||
LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state" 2>/dev/null || echo "UNKNOWN")
|
||
|
||
# Both agree and active — healthy, silent
|
||
if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then
|
||
log "Partnership check — ACTIVE, both servers agree ✅"
|
||
exit 0
|
||
fi
|
||
|
||
# Remote requested offboard
|
||
if [[ "$REMOTE_STATE" == "INACTIVE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then
|
||
warn "Partnership check — $REMOTE_SERVER_NAME requested offboard"
|
||
|
||
if [[ "$AM_OWNER" == true ]]; then
|
||
warn "Owner finalising offboard request from mirror..."
|
||
do_final_sync
|
||
|
||
# Remove owner's fallback containers, restart own stack
|
||
cleanup_partner_containers "$(derive_partner_folder_name "$MIRROR")"
|
||
start_own_stack
|
||
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
write_state_file "$LOCAL_STATE_FILE" \
|
||
"INACTIVE" "" "$NOW" "$REMOTE_SERVER_NAME" "mirror-requested"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false"
|
||
fi
|
||
|
||
# SSH key revocation — mutual, both directions
|
||
# REMOTE_IP already resolved above — this is the last SSH operation
|
||
SSH_REVOKE_REMOTE_OK=false
|
||
SSH_REVOKE_LOCAL_OK=false
|
||
do_ssh_key_revocation "$REMOTE_IP"
|
||
|
||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||
local grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 ))
|
||
warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace period..."
|
||
# Grace sleep — interruptible via SIGTERM trap
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
trap 'warn "Partnership check interrupted during grace sleep"; exit 0' SIGTERM SIGINT
|
||
sleep "$grace_seconds"
|
||
trap - SIGTERM SIGINT
|
||
fi
|
||
remove_tailscale_device "$MIRROR"
|
||
fi
|
||
notify "Partnership offboard finalised — $MIRROR requested separation" \
|
||
"Partnership" "normal"
|
||
else
|
||
# Mirror sees owner is INACTIVE — clean up own side
|
||
warn "Owner has offboarded — cleaning up mirror side..."
|
||
reconfigure_local_webuis "localhost"
|
||
cleanup_partner_containers "$(derive_partner_folder_name "$OWNER")"
|
||
start_own_stack
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
write_state_file "$LOCAL_STATE_FILE" \
|
||
"INACTIVE" "" "$NOW" "$OWNER" "owner-offboarded"
|
||
notify "Partnership ended — $OWNER offboarded. Auth WebUIs → localhost." \
|
||
"Partnership" "normal"
|
||
fi
|
||
exit 0
|
||
fi
|
||
|
||
# Both inactive — nothing to do
|
||
if [[ "$LOCAL_STATE" == "INACTIVE" ]] && [[ "$REMOTE_STATE" == "INACTIVE" ]]; then
|
||
log "Partnership check — INACTIVE on both servers"
|
||
exit 0
|
||
fi
|
||
|
||
log "Partnership check — local: $LOCAL_STATE remote: $REMOTE_STATE"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Onboard ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "onboard" ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Onboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
|
||
# Check already onboarded
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||
if [[ "$CURRENT_STATE" == "ACTIVE" ]]; then
|
||
warn "Partnership already ACTIVE — use --status for detail or --offboard to separate"
|
||
exit 0
|
||
fi
|
||
fi
|
||
|
||
# Warn if re-onboarding a previously blocked partner — onboard is deliberate so it proceeds
|
||
if is_blocklisted "$MIRROR"; then
|
||
warn "$MIRROR is on the blocklist from a previous offboard"
|
||
warn "Proceeding — blocklist will be cleared on successful onboard"
|
||
fi
|
||
|
||
log "Owner: $OWNER_ID ($OWNER)"
|
||
log "Mirror: $MIRROR_ID ($MIRROR)"
|
||
|
||
# Pre-flight
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||
|
||
resolve_remote_ip
|
||
check_connectivity
|
||
|
||
# Version parity — both servers must agree on unRAID version
|
||
check_unraid_version_parity || exit 1
|
||
|
||
# Remote array and Docker daemon
|
||
check_remote_array || exit 1
|
||
check_remote_docker_daemon || exit 1
|
||
|
||
OWNER_IP=$(resolve_tailscale_ip "$OWNER")
|
||
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
||
|
||
[[ -z "$OWNER_IP" ]] && { error "Cannot resolve $OWNER Tailscale IP"; exit 1; }
|
||
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; exit 1; }
|
||
|
||
log "Owner IP: $OWNER_IP"
|
||
log "Mirror IP: $MIRROR_IP"
|
||
|
||
# Reconfigure mirror WebUIs → owner IP
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → $OWNER_IP ━━━"
|
||
|
||
WEBUI_FAILURES=0
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
reconfigure_webui "$container" "$port" "$OWNER_IP" \
|
||
"$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ ))
|
||
done
|
||
|
||
# Verify WebUI connectivity
|
||
if [[ "${PARTNERSHIP_ONBOARD_VERIFY:-true}" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_VERIFY WebUI Connectivity ━━━"
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
if curl -sf --max-time 10 "http://${OWNER_IP}:${port}/" >/dev/null 2>&1; then
|
||
log "$container reachable at http://${OWNER_IP}:${port}/ ✅"
|
||
else
|
||
warn "$container not reachable at http://${OWNER_IP}:${port}/ — may not be running"
|
||
fi
|
||
done
|
||
fi
|
||
|
||
# Write state files
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Write State ━━━"
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "onboard"
|
||
log "Local state: ACTIVE ✅"
|
||
remove_from_blocklist "$MIRROR"
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||
echo "0" > "$OFFLINE_COUNTER"
|
||
else
|
||
warn "DRY RUN — would write ACTIVE state and push to remote"
|
||
fi
|
||
|
||
# FolderView3 — create partner folder with this server's failover containers for remote
|
||
if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS FolderView3 Integration ━━━"
|
||
PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR")
|
||
declare -a PARTNER_CONTAINERS=()
|
||
gather_partner_fallback_containers PARTNER_CONTAINERS
|
||
if [[ ${#PARTNER_CONTAINERS[@]} -gt 0 ]]; then
|
||
folderview3_create_partner_folder "$PARTNER_FOLDER_NAME" "${PARTNER_CONTAINERS[@]}"
|
||
else
|
||
log "No FALLBACK_${MY_ID}_COVERS_${REMOTE_ID}_TIER* containers configured — skipping folder creation"
|
||
fi
|
||
fi
|
||
|
||
# Emby admin provisioning — runs after container deployment (deploy step not yet built)
|
||
provision_emby_admin "$MIRROR_IP"
|
||
|
||
# Summary
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━"
|
||
echo " Owner: $OWNER_ID ($OWNER_IP)"
|
||
echo " Mirror: $MIRROR_ID ($MIRROR_IP)"
|
||
echo " WebUI failures: $WEBUI_FAILURES"
|
||
echo " Sync interval: ${PARTNERSHIP_SYNC_INTERVAL}min"
|
||
echo ""
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — no changes made"
|
||
elif [[ "$WEBUI_FAILURES" -eq 0 ]]; then
|
||
warn "$ICON_DONE DONE — $MIRROR is now mirroring $OWNER via Tailscale ✅"
|
||
[[ "${PARTNERSHIP_ONBOARD_NOTIFY:-true}" == true ]] && \
|
||
notify "Partnership onboard complete — $MIRROR is now mirroring $OWNER via Tailscale" \
|
||
"Partnership" "normal"
|
||
else
|
||
warn "DONE with $WEBUI_FAILURES WebUI warning(s) — check manually"
|
||
notify "Partnership onboard complete with $WEBUI_FAILURES WebUI warning(s) on $(hostname)" \
|
||
"Partnership" "warning"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Offboard ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "offboard" ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Offboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
|
||
# Check already offboarded
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||
if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then
|
||
warn "Partnership already INACTIVE — use --status to verify both servers agree"
|
||
exit 0
|
||
fi
|
||
fi
|
||
|
||
# ── Mirror-initiated offboard ─────────────────────────────────────────────────────────────
|
||
if [[ "$AM_MIRROR" == true ]]; then
|
||
warn "$MIRROR_ID ($MIRROR) is initiating offboard"
|
||
warn "Local auth WebUIs will be reconfigured → localhost"
|
||
warn "$OWNER_ID ($OWNER) will finalise on its next --check cycle"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
echo ""
|
||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||
sleep 10
|
||
fi
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Reconfigure Local WebUIs → localhost ━━━"
|
||
reconfigure_local_webuis "localhost"
|
||
|
||
# Remove owner's deployed containers + restart own stack
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Cleanup Partner Containers ━━━"
|
||
PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$OWNER")
|
||
cleanup_partner_containers "$PARTNER_FOLDER_NAME"
|
||
start_own_stack
|
||
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
write_state_file "$LOCAL_STATE_FILE" \
|
||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||
log "Local state: INACTIVE ✅"
|
||
|
||
[[ "$DRY_RUN" == false ]] && add_to_blocklist "$OWNER" "$REASON"
|
||
|
||
OWNER_IP=$(resolve_tailscale_ip "$OWNER")
|
||
if [[ -n "$OWNER_IP" ]]; then
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$MIRROR_SSH_KEY"
|
||
notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \
|
||
"Partnership" "normal"
|
||
else
|
||
warn "$OWNER unreachable — state written locally, owner will see it when reachable"
|
||
fi
|
||
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━"
|
||
echo " Your WebUIs: reconfigured → localhost ✅"
|
||
echo " State: INACTIVE ✅"
|
||
echo " Blocklist: $OWNER blocked — re-onboard to permit access again ✅"
|
||
echo " Owner: will finalise + final sync on next --check ✅"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ── Owner-initiated offboard ──────────────────────────────────────────────────────────────
|
||
warn "Offboarding $MIRROR_ID ($MIRROR) from partnership"
|
||
warn "Final sync will run — mirror leaves with current state"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
echo ""
|
||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||
sleep 10
|
||
echo "Proceeding..."
|
||
fi
|
||
|
||
resolve_remote_ip
|
||
|
||
# Stop any running rsync first
|
||
echo ""
|
||
echo "━━━ $ICON_STOP Stop Running Rsync ━━━"
|
||
bash "$SCRIPT_DIR/../Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true
|
||
|
||
# Final sync
|
||
echo ""
|
||
echo "━━━ $ICON_SYNC Final Sync ━━━"
|
||
do_final_sync
|
||
|
||
MIRROR_IP=$(resolve_tailscale_ip "$MIRROR")
|
||
MIRROR_REACHABLE=false
|
||
[[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true
|
||
|
||
# Reconfigure mirror WebUIs → localhost
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → localhost ━━━"
|
||
|
||
WEBUI_FAILURES=0
|
||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
reconfigure_webui "$container" "$port" "localhost" \
|
||
"$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ ))
|
||
done
|
||
else
|
||
warn "$MIRROR unreachable — WebUI reconfiguration skipped"
|
||
warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on --check"
|
||
(( WEBUI_FAILURES++ ))
|
||
fi
|
||
|
||
# Disable critical rsync
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Disable Critical Sync ━━━"
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false"
|
||
warn "CRITICAL_RSYNC_ENABLED=false ✅"
|
||
else
|
||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
||
fi
|
||
|
||
# Write state files
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Write State ━━━"
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
|
||
write_state_file "$LOCAL_STATE_FILE" \
|
||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||
log "Local state: INACTIVE ✅"
|
||
|
||
[[ "$DRY_RUN" == false ]] && add_to_blocklist "$MIRROR" "$REASON"
|
||
|
||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||
fi
|
||
|
||
# Local: remove fallback-coverage containers for mirror, restart own stack
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Local Container Cleanup ━━━"
|
||
PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR")
|
||
cleanup_partner_containers "$PARTNER_FOLDER_NAME"
|
||
start_own_stack
|
||
|
||
# Remote: remove owner's deployed containers from mirror, restart mirror's own stack
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Remote Container Cleanup ━━━"
|
||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||
cleanup_owner_containers_on_mirror "$MIRROR_IP"
|
||
start_mirror_own_stack "$MIRROR_IP"
|
||
else
|
||
warn "$MIRROR unreachable — remote container cleanup skipped"
|
||
warn "Run 'partnership_manager.sh --offboard' on $MIRROR to clean up manually"
|
||
fi
|
||
|
||
# Emby admin revocation — before SSH key revocation while Emby still reachable
|
||
[[ "$MIRROR_REACHABLE" == true ]] && revoke_emby_admin "$MIRROR_IP"
|
||
|
||
# SSH key revocation — mutual, both directions
|
||
# Must run before Tailscale removal (SSH needs network) and after state is pushed
|
||
SSH_REVOKE_REMOTE_OK=false
|
||
SSH_REVOKE_LOCAL_OK=false
|
||
do_ssh_key_revocation "${MIRROR_IP:-}"
|
||
|
||
# Tailscale removal
|
||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_NET Tailscale Separation ━━━"
|
||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||
grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 ))
|
||
warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace — mirror can collect backups..."
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
trap 'warn "Offboard interrupted during grace sleep"; exit 0' SIGTERM SIGINT
|
||
sleep "$grace_seconds"
|
||
trap - SIGTERM SIGINT
|
||
fi
|
||
fi
|
||
remove_tailscale_device "$MIRROR"
|
||
fi
|
||
|
||
# Backup notification
|
||
if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_DISK Backup Handover ━━━"
|
||
log "Backups available for $MIRROR:"
|
||
for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do
|
||
[[ -z "$path" ]] && continue
|
||
echo " $path"
|
||
done
|
||
notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-6}hr. Tailscale access expires then." \
|
||
"Partnership" "warning"
|
||
fi
|
||
|
||
# Summary
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━"
|
||
echo " Owner: $OWNER_ID ($OWNER)"
|
||
echo " Mirror: $MIRROR_ID ($MIRROR)"
|
||
echo " Final sync: complete ✅"
|
||
echo " WebUI failures: $WEBUI_FAILURES"
|
||
echo " Critical rsync: disabled ✅"
|
||
echo " State: INACTIVE ✅"
|
||
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
|
||
_revoke_status() {
|
||
if [[ "${SSH_REVOKE_REMOTE_OK:-false}" == true ]] && [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||
echo "both directions ✅"
|
||
elif [[ "${SSH_REVOKE_LOCAL_OK:-false}" == true ]]; then
|
||
echo "local only ✅ — remote failed (revoke manually on $MIRROR)"
|
||
else
|
||
echo "⚠️ failed — check warnings above"
|
||
fi
|
||
}
|
||
echo " Keys revoked: $(_revoke_status)"
|
||
[[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]] && \
|
||
echo " FolderView3: ${PARTNER_FOLDER_NAME:-} cleaned ✅"
|
||
[[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \
|
||
echo " Tailscale: $MIRROR removed ✅"
|
||
echo ""
|
||
echo " $MIRROR leaves with:"
|
||
echo " ✓ Current auth config (final sync)"
|
||
echo " ✓ Auth WebUIs → localhost"
|
||
echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups"
|
||
echo " ✓ Full ecosystem — just stop the sync"
|
||
echo ""
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
||
warn "$ICON_DONE DONE — clean separation complete ✅"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Transfer ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "transfer" ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_FALLBACK Transfer Ownership — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
echo ""
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo "⚠️ WARNING — OWNERSHIP TRANSFER"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " Current owner: $OWNER_ID ($OWNER)"
|
||
echo " Current mirror: $MIRROR_ID ($MIRROR)"
|
||
echo ""
|
||
echo " After transfer:"
|
||
echo " New owner: $MIRROR_ID ($MIRROR)"
|
||
echo " New mirror: $OWNER_ID ($OWNER)"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
# Confirmation check
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
if [[ -z "$TRANSFER_CONFIRM_INPUT" ]]; then
|
||
echo ""
|
||
echo "To proceed pass exactly:"
|
||
echo " --confirm=${PARTNERSHIP_TRANSFER_CONFIRM}"
|
||
echo ""
|
||
error "Transfer cancelled — confirmation required"
|
||
exit 1
|
||
fi
|
||
if [[ "$TRANSFER_CONFIRM_INPUT" != "$PARTNERSHIP_TRANSFER_CONFIRM" ]]; then
|
||
error "Confirmation string does not match — transfer cancelled"
|
||
exit 1
|
||
fi
|
||
log "Confirmation accepted"
|
||
else
|
||
warn "DRY RUN — confirmation check skipped"
|
||
fi
|
||
|
||
# Health strike system
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Health Verification ━━━"
|
||
log "Both servers must pass ${PARTNERSHIP_TRANSFER_STRIKES} consecutive health checks"
|
||
|
||
STRIKES=0
|
||
ATTEMPTS=0
|
||
MAX_ATTEMPTS="${PARTNERSHIP_TRANSFER_MAX_ATTEMPTS:-20}"
|
||
|
||
while [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]]; do
|
||
(( ATTEMPTS++ ))
|
||
if [[ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]]; then
|
||
error "Health checks failed after $MAX_ATTEMPTS attempts — servers not stable"
|
||
error "Transfer cancelled — try again when both servers are healthy"
|
||
exit 1
|
||
fi
|
||
|
||
if check_both_healthy; then
|
||
(( STRIKES++ ))
|
||
log "Health check passed ($STRIKES/${PARTNERSHIP_TRANSFER_STRIKES})"
|
||
[[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]] && sleep 10
|
||
else
|
||
warn "Health check failed — resetting (attempt $ATTEMPTS/$MAX_ATTEMPTS)"
|
||
STRIKES=0
|
||
sleep 30
|
||
fi
|
||
done
|
||
warn "Both servers healthy — proceeding with transfer"
|
||
|
||
NEW_OWNER_ID="$MIRROR_ID"
|
||
NEW_MIRROR_ID="$OWNER_ID"
|
||
NEW_OWNER="$MIRROR"
|
||
NEW_MIRROR="$OWNER"
|
||
NEW_OWNER_SSH_KEY_VAR="${NEW_OWNER_ID}_SSH_KEY"
|
||
NEW_MIRROR_SSH_KEY_VAR="${NEW_MIRROR_ID}_SSH_KEY"
|
||
NEW_OWNER_SSH_KEY="${!NEW_OWNER_SSH_KEY_VAR}"
|
||
NEW_MIRROR_SSH_KEY="${!NEW_MIRROR_SSH_KEY_VAR}"
|
||
|
||
NEW_OWNER_IP=$(resolve_tailscale_ip "$NEW_OWNER")
|
||
NEW_MIRROR_IP=$(resolve_tailscale_ip "$NEW_MIRROR")
|
||
|
||
[[ -z "$NEW_OWNER_IP" ]] && { error "Cannot resolve new owner Tailscale IP"; exit 1; }
|
||
[[ -z "$NEW_MIRROR_IP" ]] && { error "Cannot resolve new mirror Tailscale IP"; exit 1; }
|
||
|
||
# Final sync in current direction
|
||
echo ""
|
||
echo "━━━ $ICON_SYNC Pre-transfer Sync ━━━"
|
||
do_final_sync
|
||
|
||
# Reconfigure new mirror WebUIs → new owner
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Reconfigure WebUIs ━━━"
|
||
log "New mirror ($NEW_MIRROR) WebUIs → new owner ($NEW_OWNER_IP)"
|
||
|
||
WEBUI_FAILURES=0
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
reconfigure_webui "$container" "$port" "$NEW_OWNER_IP" \
|
||
"$NEW_MIRROR_SSH_KEY" "$NEW_MIRROR_IP" "$NEW_MIRROR" || (( WEBUI_FAILURES++ ))
|
||
done
|
||
|
||
# New owner WebUIs → localhost (now manages directly)
|
||
log "New owner ($NEW_OWNER) WebUIs → localhost"
|
||
reconfigure_local_webuis "localhost"
|
||
|
||
# Flip PARTNERSHIP_OWNER_HOST in master.conf on both servers
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Update Ownership ━━━"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
update_master_conf "PARTNERSHIP_OWNER_HOST" "\"$NEW_OWNER_ID\""
|
||
|
||
# Update remote master.conf
|
||
timeout "$SSH_TIMEOUT" ssh -i "$NEW_MIRROR_SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$NEW_MIRROR_IP" \
|
||
"sed -i 's|^[[:space:]]*PARTNERSHIP_OWNER_HOST=.*| PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\"|' \
|
||
'$SCRIPT_DIR/../master.conf'" 2>/dev/null && \
|
||
log "Remote master.conf updated ✅" || \
|
||
error "Failed to update remote master.conf — update manually"
|
||
else
|
||
warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers"
|
||
fi
|
||
|
||
# Write state files
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "transfer"
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$NEW_MIRROR_IP" "$NEW_MIRROR_SSH_KEY"
|
||
|
||
# Summary
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY TRANSFER SUMMARY ━━━━━"
|
||
echo " New owner: $NEW_OWNER_ID ($NEW_OWNER_IP)"
|
||
echo " New mirror: $NEW_MIRROR_ID ($NEW_MIRROR_IP)"
|
||
echo " WebUI failures: $WEBUI_FAILURES"
|
||
echo " Sync direction: $NEW_OWNER → $NEW_MIRROR"
|
||
echo " Owner host: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID"
|
||
echo ""
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — no changes made"
|
||
else
|
||
warn "$ICON_DONE DONE — ownership transferred to $NEW_OWNER_ID ✅"
|
||
notify "Partnership ownership transferred — new owner: $NEW_OWNER ($NEW_OWNER_ID)" \
|
||
"Partnership" "normal"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi |