Gmer4LfeandClaude Sonnet 4.6
69464ba385
After offboard, the former partner's hostname is written to
/boot/config/partnership_blocklist.db. This blocks passive reconnection:
- --check skips the remote entirely (no auto-reconnect noise)
- rsync.sh refuses to sync with a blocklisted host
- --onboard warns about the previous offboard but proceeds,
and clears the block on success (onboard = deliberate intent)
New --unblock <hostname> mode removes an entry to permit re-onboarding
without running a full --onboard first. --status shows the full blocklist.
Blocklist file survives reboots (/boot/config) and Tailscale reconnections —
application-layer guard that complements Tailscale device removal.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1339 lines
55 KiB
Bash
1339 lines
55 KiB
Bash
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ================================= Partnership Manager ========================================
|
||
# ==============================================================================================
|
||
# Manages the relationship lifecycle between two unRAID servers.
|
||
# HOST1 is always the owner (source of truth). HOST2 is always the mirror.
|
||
# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a successful --transfer.
|
||
#
|
||
# ── MODES ─────────────────────────────────────────────────────────────────────────────────────
|
||
# --onboard ← owner only — establish mirror relationship
|
||
# Reconfigures HOST2 auth WebUIs → HOST1 Tailscale IP
|
||
# HOST2 clicks NPM → gets HOST1's NPM automatically
|
||
#
|
||
# --offboard ← either server — clean separation
|
||
# Mirror-initiated: writes INACTIVE state, reconfigures own WebUIs → localhost
|
||
# Owner finalises on next --check: final sync, Tailscale removal
|
||
# Owner-initiated: final sync, reconfigures mirror WebUIs, Tailscale removal
|
||
# Both leave with current state, clean exit ✅
|
||
#
|
||
# --transfer ← owner only — flip ownership between servers
|
||
# Requires confirmation string + consecutive health strikes
|
||
# Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in master.conf
|
||
#
|
||
# --check ← called by critical_sync_maintenance.sh every 15min
|
||
# Reads both state files via SSH
|
||
# Detects offboard requests → finalises from owner side
|
||
# Updates last_seen_remote timestamp
|
||
# Increments offline counter → auto-offboards after threshold
|
||
# Silent when healthy ← never noisy on clean runs
|
||
#
|
||
# --status ← either server — show current state, both sides
|
||
#
|
||
# ── STATE FILES ───────────────────────────────────────────────────────────────────────────────
|
||
# On /boot/config — survives reboots, available before array starts:
|
||
# /boot/config/partnership_HOST1.db ← HOST1 writes only
|
||
# /boot/config/partnership_HOST2.db ← HOST2 writes only
|
||
# Propagated via SSH — no rsync needed
|
||
#
|
||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||
# Root check — all operations require root
|
||
# validate_unraid_cmd — notify validated before use
|
||
# Role enforcement — mirror cannot run owner-only modes
|
||
# AM_OWNER / AM_MIRROR — all routing via these flags, not hostname strings
|
||
# version parity — onboard checks both servers match unRAID version
|
||
# remote docker daemon — onboard verifies remote daemon responsive
|
||
# SSH_TIMEOUT — all SSH calls timeout-protected
|
||
# flock on state writes — prevents concurrent state file corruption
|
||
# SIGTERM trap — grace period sleep interruptible
|
||
# Silent by default — only warns/errors produce output (--check is always silent healthy)
|
||
#
|
||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||
# HOST*_PARTNERSHIP_AUTH_WEBUIS — containers reconfigured on onboard/offboard
|
||
# HOST*_PARTNERSHIP_MIRROR_BACKUPS — paths available after offboard
|
||
# All aliased by detect_hosts() — script uses PARTNERSHIP_AUTH_WEBUIS etc.
|
||
#
|
||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||
# PARTNERSHIP_ENABLED — global enable gate
|
||
# PARTNERSHIP_OWNER_HOST — "HOST1" or "HOST2" — flips on --transfer
|
||
# PARTNERSHIP_REMOVE_TAILSCALE — remove mirror from tailnet on offboard
|
||
# PARTNERSHIP_GRACE_HOURS — hours before Tailscale removal after offboard
|
||
# PARTNERSHIP_OFFLINE_THRESHOLD — days unreachable before auto-offboard
|
||
# PARTNERSHIP_TRANSFER_CONFIRM — exact string required for --transfer
|
||
# PARTNERSHIP_TRANSFER_STRIKES — consecutive health checks required
|
||
# PARTNERSHIP_TRANSFER_MAX_ATTEMPTS — max attempts before giving up
|
||
# PARTNERSHIP_ONBOARD_VERIFY — verify WebUI connectivity after onboard
|
||
# PARTNERSHIP_ONBOARD_NOTIFY — notify both servers on onboard completion
|
||
# PARTNERSHIP_SYNC_INTERVAL — informational — actual schedule in cron
|
||
# TAILSCALE_API_KEY / TAILSCALE_TAILNET — required when PARTNERSHIP_REMOVE_TAILSCALE=true
|
||
#
|
||
# ── BLOCKLIST ─────────────────────────────────────────────────────────────────────────────────
|
||
# After offboard, the former partner's hostname is written to:
|
||
# /boot/config/partnership_blocklist.db (format: hostname|timestamp|reason)
|
||
#
|
||
# --onboard is hard-blocked if the remote is on the blocklist — exits with error.
|
||
# --check silently skips remote state reads for blocklisted hosts (no noise every 15min).
|
||
# --unblock <hostname> removes an entry to permit re-onboarding.
|
||
# --status shows the full blocklist.
|
||
#
|
||
# The blocklist persists until explicitly cleared — surviving reboots, array restarts,
|
||
# and Tailscale reconnections. Tailscale removal is a separate step at the network layer;
|
||
# the blocklist is the application-layer guard.
|
||
#
|
||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||
# partnership_manage.sh --onboard
|
||
# partnership_manage.sh --offboard
|
||
# partnership_manage.sh --transfer --confirm=i-understand-this-transfers-ownership
|
||
# partnership_manage.sh --check --remote-seen|--remote-unseen
|
||
# partnership_manage.sh --status
|
||
# partnership_manage.sh --unblock <hostname>
|
||
# Any mode supports --dry-run and --log
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
||
source "$SCRIPT_DIR/../load_config.sh"
|
||
|
||
SSH_TIMEOUT=15
|
||
BLOCKLIST_FILE="/boot/config/partnership_blocklist.db"
|
||
|
||
# ── Parse mode flags before parse_args ────────────────────────────────────────────────────────
|
||
MODE=""
|
||
TRANSFER_CONFIRM_INPUT=""
|
||
REMOTE_SEEN=false
|
||
REMOTE_UNSEEN=false
|
||
REASON="manual"
|
||
UNBLOCK_HOST=""
|
||
FILTERED_ARGS=()
|
||
|
||
for arg in "$@"; do
|
||
case "$arg" in
|
||
--onboard) MODE="onboard" ;;
|
||
--offboard) MODE="offboard" ;;
|
||
--transfer) MODE="transfer" ;;
|
||
--check) MODE="check" ;;
|
||
--status) MODE="status" ;;
|
||
--unblock) MODE="unblock" ;;
|
||
--confirm=*) TRANSFER_CONFIRM_INPUT="${arg#--confirm=}" ;;
|
||
--remote-seen) REMOTE_SEEN=true ;;
|
||
--remote-unseen) REMOTE_UNSEEN=true ;;
|
||
--reason=*) REASON="${arg#--reason=}" ;;
|
||
*)
|
||
if [[ "$MODE" == "unblock" ]] && [[ -z "$UNBLOCK_HOST" ]] && [[ "$arg" != --* ]]; then
|
||
UNBLOCK_HOST="$arg"
|
||
else
|
||
FILTERED_ARGS+=("$arg")
|
||
fi
|
||
;;
|
||
esac
|
||
done
|
||
|
||
parse_args "${FILTERED_ARGS[@]}"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
validate_unraid_cmd \
|
||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||
"" "" \
|
||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||
|
||
# detect_hosts() sets MY_ID and aliases PARTNERSHIP_AUTH_WEBUIS, PARTNERSHIP_MIRROR_BACKUPS
|
||
detect_hosts
|
||
|
||
# ── Derive owner and mirror from PARTNERSHIP_OWNER_HOST ───────────────────────────────────────
|
||
OWNER_ID="${PARTNERSHIP_OWNER_HOST:-HOST1}" # e.g. "HOST1"
|
||
MIRROR_ID=$( [[ "$OWNER_ID" == "HOST1" ]] && echo "HOST2" || echo "HOST1" )
|
||
|
||
OWNER="${!OWNER_ID}" # hostname string
|
||
MIRROR="${!MIRROR_ID}"
|
||
OWNER_SSH_KEY_VAR="${OWNER_ID}_SSH_KEY"
|
||
MIRROR_SSH_KEY_VAR="${MIRROR_ID}_SSH_KEY"
|
||
OWNER_SSH_KEY="${!OWNER_SSH_KEY_VAR}"
|
||
MIRROR_SSH_KEY="${!MIRROR_SSH_KEY_VAR}"
|
||
|
||
AM_OWNER=false
|
||
AM_MIRROR=false
|
||
[[ "$MY_ID" == "$OWNER_ID" ]] && AM_OWNER=true
|
||
[[ "$MY_ID" == "$MIRROR_ID" ]] && AM_MIRROR=true
|
||
|
||
# State files
|
||
LOCAL_STATE_FILE="/boot/config/partnership_${LOCAL_SERVER_NAME}.db"
|
||
REMOTE_STATE_FILE="/boot/config/partnership_${REMOTE_SERVER_NAME}.db"
|
||
OWNER_STATE_FILE="/boot/config/partnership_${OWNER}.db"
|
||
MIRROR_STATE_FILE="/boot/config/partnership_${MIRROR}.db"
|
||
OFFLINE_COUNTER="/boot/config/partnership_offline_days.db"
|
||
|
||
if [[ -z "$MODE" ]]; then
|
||
error "No mode specified"
|
||
echo "Usage:"
|
||
echo " partnership_manage.sh --onboard"
|
||
echo " partnership_manage.sh --offboard"
|
||
echo " partnership_manage.sh --transfer --confirm=..."
|
||
echo " partnership_manage.sh --check --remote-seen|--remote-unseen"
|
||
echo " partnership_manage.sh --status"
|
||
echo " partnership_manage.sh --unblock <hostname>"
|
||
exit 1
|
||
fi
|
||
|
||
# Role-based access control
|
||
if [[ "$AM_MIRROR" == true ]]; then
|
||
case "$MODE" in
|
||
onboard|transfer)
|
||
error "Only the owner ($OWNER / $OWNER_ID) can run --$MODE"
|
||
error "Run from $OWNER or use --offboard to separate cleanly"
|
||
exit 1
|
||
;;
|
||
esac
|
||
fi
|
||
|
||
# Lock for all modes except --check (check is called frequently, lock would pile up)
|
||
[[ "$MODE" != "check" ]] && acquire_lock "strict"
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||
|
||
# ==============================================================================================
|
||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||
# ==============================================================================================
|
||
|
||
read_state_file() {
|
||
local file="$1" key="$2"
|
||
grep "^${key}=" "$file" 2>/dev/null | cut -d= -f2
|
||
}
|
||
|
||
write_state_file() {
|
||
local file="$1"
|
||
shift
|
||
# flock prevents concurrent writes to the same state file
|
||
(
|
||
flock -x 200
|
||
cat > "$file" << EOF
|
||
state=${1:-UNKNOWN}
|
||
owner=${OWNER}
|
||
mirror=${MIRROR}
|
||
onboarded=${2:-}
|
||
offboarded=${3:-}
|
||
triggered_by=${4:-}
|
||
reason=${5:-}
|
||
last_seen_remote=${6:-}
|
||
updated=$(date '+%Y-%m-%d %H:%M:%S')
|
||
EOF
|
||
) 200>"${file}.lock"
|
||
}
|
||
|
||
push_state_to_remote() {
|
||
local local_file="$1" remote_ip="$2" ssh_key="$3"
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would push state file to remote"
|
||
return 0
|
||
fi
|
||
timeout "$SSH_TIMEOUT" scp -i "$ssh_key" -o ConnectTimeout="$SSH_TIMEOUT" \
|
||
"$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \
|
||
log "State file pushed to remote ✅" || \
|
||
warn "Could not push state file to remote — will propagate on next sync"
|
||
}
|
||
|
||
read_remote_state() {
|
||
local remote_ip="$1" ssh_key="$2" remote_file="$3"
|
||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
||
"cat '$remote_file' 2>/dev/null" 2>/dev/null
|
||
}
|
||
|
||
# Reconfigure a container's WebUI on the remote server
|
||
reconfigure_webui() {
|
||
local container="$1" port="$2" target_ip="$3"
|
||
local ssh_key="$4" remote_ip="$5" label="${6:-remote}"
|
||
|
||
log "Reconfiguring $container WebUI → ${target_ip}:${port} on $label..."
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would reconfigure $container WebUI to http://${target_ip}:${port}/"
|
||
return 0
|
||
fi
|
||
|
||
local template
|
||
template=$(timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
||
"grep -rl '<WebUI>' /boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \
|
||
xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null)
|
||
|
||
if [[ -z "$template" ]]; then
|
||
warn "$container template not found on $label — WebUI needs manual reconfiguration"
|
||
return 1
|
||
fi
|
||
|
||
timeout "$SSH_TIMEOUT" ssh -i "$ssh_key" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$remote_ip" \
|
||
"sed -i 's|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g' '$template'" \
|
||
2>/dev/null && \
|
||
log "$container → http://${target_ip}:${port}/ ✅" || {
|
||
error "Failed to reconfigure $container WebUI on $label"
|
||
return 1
|
||
}
|
||
}
|
||
|
||
# Reconfigure local auth WebUIs to target IP
|
||
reconfigure_local_webuis() {
|
||
local target_ip="$1"
|
||
log "Reconfiguring local auth WebUIs → ${target_ip}..."
|
||
|
||
local failures=0
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
local container="${entry%%|*}"
|
||
local port="${entry##*|}"
|
||
|
||
local template
|
||
template=$(grep -rl '<WebUI>' \
|
||
/boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \
|
||
xargs grep -l "\"$container\"" 2>/dev/null | head -1)
|
||
|
||
if [[ -z "$template" ]]; then
|
||
warn "$container template not found locally"
|
||
(( failures++ ))
|
||
continue
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would reconfigure $container → http://${target_ip}:${port}/"
|
||
continue
|
||
fi
|
||
|
||
sed -i "s|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g" \
|
||
"$template" 2>/dev/null && \
|
||
log "$container → http://${target_ip}:${port}/ ✅" || \
|
||
{ error "Failed to reconfigure $container"; (( failures++ )); }
|
||
done
|
||
return $failures
|
||
}
|
||
|
||
get_tailscale_device_id() {
|
||
local hostname="$1"
|
||
curl -sf --max-time 10 \
|
||
-H "Authorization: Bearer $TAILSCALE_API_KEY" \
|
||
"https://api.tailscale.com/api/v2/tailnet/${TAILSCALE_TAILNET}/devices" \
|
||
2>/dev/null | \
|
||
grep -o "\"id\":\"[^\"]*\"[^}]*\"hostname\":\"${hostname}\"" | \
|
||
grep -o '"id":"[^"]*"' | \
|
||
grep -o '[^"]*"$' | tr -d '"'
|
||
}
|
||
|
||
remove_tailscale_device() {
|
||
local hostname="$1"
|
||
|
||
if [[ -z "${TAILSCALE_API_KEY:-}" ]] || [[ -z "${TAILSCALE_TAILNET:-}" ]]; then
|
||
warn "TAILSCALE_API_KEY or TAILSCALE_TAILNET not configured — skipping Tailscale removal"
|
||
return 1
|
||
fi
|
||
|
||
log "Looking up Tailscale device ID for $hostname..."
|
||
local device_id
|
||
device_id=$(get_tailscale_device_id "$hostname")
|
||
|
||
if [[ -z "$device_id" ]]; then
|
||
warn "$hostname not found in Tailscale — may already be removed"
|
||
return 0
|
||
fi
|
||
|
||
log "Removing $hostname (device $device_id) from Tailscale..."
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove $hostname from Tailscale tailnet"
|
||
return 0
|
||
fi
|
||
|
||
curl -sf --max-time 10 -X DELETE \
|
||
-H "Authorization: Bearer $TAILSCALE_API_KEY" \
|
||
"https://api.tailscale.com/api/v2/devices/${device_id}" 2>/dev/null && \
|
||
warn "$hostname removed from Tailscale ✅" || \
|
||
error "Failed to remove $hostname from Tailscale — remove manually"
|
||
}
|
||
|
||
# ── Blocklist helpers ─────────────────────────────────────────────────────────────────────────
|
||
|
||
is_blocklisted() {
|
||
local hostname="$1"
|
||
[[ -f "$BLOCKLIST_FILE" ]] && grep -q "^${hostname}|" "$BLOCKLIST_FILE" 2>/dev/null
|
||
}
|
||
|
||
add_to_blocklist() {
|
||
local hostname="$1" reason="${2:-offboard}"
|
||
local now
|
||
now=$(date '+%Y-%m-%d %H:%M:%S')
|
||
if ! is_blocklisted "$hostname"; then
|
||
echo "${hostname}|${now}|${reason}" >> "$BLOCKLIST_FILE"
|
||
log "Blocklisted: $hostname (reason: $reason)"
|
||
else
|
||
log "$hostname already on blocklist"
|
||
fi
|
||
}
|
||
|
||
remove_from_blocklist() {
|
||
local hostname="$1"
|
||
if [[ ! -f "$BLOCKLIST_FILE" ]]; then
|
||
log "Blocklist empty — nothing to remove"
|
||
return 0
|
||
fi
|
||
sed -i "/^${hostname}|/d" "$BLOCKLIST_FILE" 2>/dev/null
|
||
log "Removed from blocklist: $hostname"
|
||
}
|
||
|
||
# ── FolderView3 integration ───────────────────────────────────────────────────────────────────
|
||
FOLDERVIEW3_DIR="/usr/local/emhttp/plugins/folder.view3"
|
||
FOLDERVIEW3_JSON="/boot/config/plugins/folder.view3/docker.json"
|
||
|
||
# Derive short partner name: strip unraid- prefix (case-insensitive) if present
|
||
derive_partner_folder_name() {
|
||
local hostname="$1"
|
||
local short="${hostname,,}"
|
||
[[ "$short" == unraid-* ]] && short="${short:7}"
|
||
# Capitalise first char for readability: jayred365 → Jayred365-Failover
|
||
echo "${short^}-Failover"
|
||
}
|
||
|
||
folderview3_ensure_plugin() {
|
||
if [[ -d "$FOLDERVIEW3_DIR" ]]; then
|
||
log "FolderView3 plugin present ✅"
|
||
return 0
|
||
fi
|
||
if [[ -z "${PARTNERSHIP_FOLDERVIEW3_URL:-}" ]]; then
|
||
warn "FolderView3 plugin not installed and PARTNERSHIP_FOLDERVIEW3_URL is empty"
|
||
warn "Install manually from Community Applications or set PARTNERSHIP_FOLDERVIEW3_URL in master.conf"
|
||
return 1
|
||
fi
|
||
warn "FolderView3 not found — installing from CA..."
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would run: plugin install $PARTNERSHIP_FOLDERVIEW3_URL"
|
||
return 0
|
||
fi
|
||
plugin install "$PARTNERSHIP_FOLDERVIEW3_URL" 2>/dev/null && \
|
||
log "FolderView3 installed ✅" || {
|
||
warn "FolderView3 install failed — folder will not be created"
|
||
return 1
|
||
}
|
||
}
|
||
|
||
folderview3_create_partner_folder() {
|
||
local partner_name="$1"
|
||
shift
|
||
local containers=("$@")
|
||
|
||
log "FolderView3: creating folder '$partner_name' with ${#containers[@]} container(s)..."
|
||
|
||
folderview3_ensure_plugin || return 1
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would create FolderView3 folder: $partner_name"
|
||
for c in "${containers[@]}"; do
|
||
[[ -n "$c" ]] && warn " DRY RUN — container: $c"
|
||
done
|
||
return 0
|
||
fi
|
||
|
||
# Initialise JSON if missing
|
||
if [[ ! -f "$FOLDERVIEW3_JSON" ]]; then
|
||
mkdir -p "$(dirname "$FOLDERVIEW3_JSON")"
|
||
echo '{}' > "$FOLDERVIEW3_JSON"
|
||
fi
|
||
|
||
# Check folder doesn't already exist
|
||
local existing
|
||
existing=$(jq -r --arg name "$partner_name" \
|
||
'to_entries[] | select(.value.name == $name) | .key' \
|
||
"$FOLDERVIEW3_JSON" 2>/dev/null)
|
||
if [[ -n "$existing" ]]; then
|
||
log "FolderView3: folder '$partner_name' already exists (id: $existing) — skipping"
|
||
return 0
|
||
fi
|
||
|
||
# Build containers JSON array
|
||
local containers_json
|
||
containers_json=$(printf '%s\n' "${containers[@]}" | \
|
||
grep -v '^$' | jq -R . | jq -s .)
|
||
|
||
# Generate a stable random ID from timestamp+name
|
||
local folder_id
|
||
folder_id=$(echo "${partner_name}$(date +%s%N)" | md5sum | head -c 12)
|
||
|
||
jq --arg id "$folder_id" --arg name "$partner_name" \
|
||
--argjson containers "$containers_json" \
|
||
'.[$id] = {"name": $name, "containers": $containers, "containerImages": {}}' \
|
||
"$FOLDERVIEW3_JSON" > "${FOLDERVIEW3_JSON}.tmp" && \
|
||
mv "${FOLDERVIEW3_JSON}.tmp" "$FOLDERVIEW3_JSON" && \
|
||
log "FolderView3: folder '$partner_name' created with ${#containers[@]} container(s) ✅" || {
|
||
warn "FolderView3: failed to write JSON — check $FOLDERVIEW3_JSON"
|
||
return 1
|
||
}
|
||
}
|
||
|
||
folderview3_remove_partner_folder() {
|
||
local partner_name="$1"
|
||
|
||
log "FolderView3: removing folder '$partner_name' and stopping its containers..."
|
||
|
||
if [[ ! -f "$FOLDERVIEW3_JSON" ]]; then
|
||
log "FolderView3: JSON not found — nothing to remove"
|
||
return 0
|
||
fi
|
||
|
||
if ! command -v jq >/dev/null 2>&1; then
|
||
warn "jq not found — cannot manage FolderView3 JSON"
|
||
return 1
|
||
fi
|
||
|
||
# Get containers from this folder
|
||
local containers_json
|
||
containers_json=$(jq -r --arg name "$partner_name" \
|
||
'to_entries[] | select(.value.name == $name) | .value.containers[]' \
|
||
"$FOLDERVIEW3_JSON" 2>/dev/null)
|
||
|
||
if [[ -z "$containers_json" ]]; then
|
||
log "FolderView3: folder '$partner_name' not found — nothing to remove"
|
||
return 0
|
||
fi
|
||
|
||
# Stop and remove each container in the folder
|
||
local stopped=0 removed=0 failed=0
|
||
while IFS= read -r container; do
|
||
[[ -z "$container" ]] && continue
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would stop + rm: $container"
|
||
continue
|
||
fi
|
||
# Stop if running
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker inspect "$container" >/dev/null 2>&1; then
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker stop "$container" >/dev/null 2>&1; then
|
||
log "$container stopped ✅"
|
||
(( stopped++ ))
|
||
else
|
||
warn "$container stop failed"
|
||
(( failed++ ))
|
||
fi
|
||
if timeout "${DOCKER_TIMEOUT:-30}" docker rm "$container" >/dev/null 2>&1; then
|
||
log "$container removed ✅"
|
||
(( removed++ ))
|
||
else
|
||
warn "$container rm failed (may not exist)"
|
||
fi
|
||
else
|
||
log "$container not found locally — skipping"
|
||
fi
|
||
done <<< "$containers_json"
|
||
|
||
# Remove folder entry from JSON
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
jq --arg name "$partner_name" \
|
||
'with_entries(select(.value.name != $name))' \
|
||
"$FOLDERVIEW3_JSON" > "${FOLDERVIEW3_JSON}.tmp" && \
|
||
mv "${FOLDERVIEW3_JSON}.tmp" "$FOLDERVIEW3_JSON" && \
|
||
log "FolderView3: folder '$partner_name' removed ✅" || \
|
||
warn "FolderView3: failed to remove folder from JSON"
|
||
else
|
||
warn "DRY RUN — would remove folder '$partner_name' from $FOLDERVIEW3_JSON"
|
||
fi
|
||
|
||
[[ "$DRY_RUN" == false ]] && \
|
||
log "FolderView3 cleanup: $stopped stopped, $removed removed, $failed failed"
|
||
return 0
|
||
}
|
||
|
||
# Gather all partner failover containers for this server (all tiers)
|
||
gather_partner_failover_containers() {
|
||
local out_var="$1"
|
||
eval "${out_var}=()"
|
||
local tier var
|
||
for tier in TIER1 TIER2 TIER3 TIER4; do
|
||
var="FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_${tier}"
|
||
# Check if var is set and is an array
|
||
if declare -p "$var" >/dev/null 2>&1; then
|
||
local -a _tmp_arr
|
||
eval "_tmp_arr=(\"\${${var}[@]}\")"
|
||
for c in "${_tmp_arr[@]}"; do
|
||
[[ -n "$c" ]] && eval "${out_var}+=(\"\$c\")"
|
||
done
|
||
fi
|
||
done
|
||
}
|
||
|
||
check_both_healthy() {
|
||
mountpoint -q /mnt/user 2>/dev/null || { error "Local array not healthy"; return 1; }
|
||
|
||
local mirror_ip
|
||
mirror_ip=$(tailscale ip -4 "${MIRROR,,}" 2>/dev/null)
|
||
[[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }
|
||
|
||
timeout "$SSH_TIMEOUT" ssh -i "$MIRROR_SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$mirror_ip" \
|
||
"mountpoint -q /mnt/user && timeout 10 docker ps" >/dev/null 2>&1 || {
|
||
error "Mirror $MIRROR not healthy"
|
||
return 1
|
||
}
|
||
return 0
|
||
}
|
||
|
||
do_final_sync() {
|
||
log "Running final Critical-Data and Emby sync..."
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
# Use CRITICAL_SYNC_SHARES from HOST* conf if available
|
||
# Falls back to known paths — these are the critical ones
|
||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
|
||
"/mnt/user/appdata-Failover/Critical-Data" \
|
||
--profile=critical-failover --log
|
||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
|
||
"/mnt/user/Media_Server/Emby" \
|
||
--profile=emby-failover --log
|
||
else
|
||
warn "DRY RUN — would run final Critical-Data and Emby sync"
|
||
fi
|
||
warn "Final sync complete — mirror has current state ✅"
|
||
}
|
||
|
||
# Safe master.conf modification with error handling
|
||
update_master_conf() {
|
||
local key="$1" value="$2"
|
||
local conf="$SCRIPT_DIR/../master.conf"
|
||
if [[ ! -f "$conf" ]]; then
|
||
error "master.conf not found at $conf"
|
||
return 1
|
||
fi
|
||
if sed -i "s|^[[:space:]]*${key}=.*| ${key}=${value}|" "$conf" 2>/dev/null; then
|
||
log "master.conf updated: ${key}=${value}"
|
||
return 0
|
||
else
|
||
error "Failed to update master.conf: ${key}=${value}"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Unblock ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "unblock" ]]; then
|
||
if [[ -z "$UNBLOCK_HOST" ]]; then
|
||
error "Usage: partnership_manager.sh --unblock <hostname>"
|
||
error "Example: partnership_manager.sh --unblock unRAID-Jayred365"
|
||
if [[ -f "$BLOCKLIST_FILE" ]] && [[ -s "$BLOCKLIST_FILE" ]]; then
|
||
echo ""
|
||
echo "Currently blocklisted:"
|
||
while IFS='|' read -r host ts reason; do
|
||
echo " $host (blocked $ts — $reason)"
|
||
done < "$BLOCKLIST_FILE"
|
||
else
|
||
echo "Blocklist is empty."
|
||
fi
|
||
exit 1
|
||
fi
|
||
|
||
if ! is_blocklisted "$UNBLOCK_HOST"; then
|
||
warn "$UNBLOCK_HOST is not on the blocklist"
|
||
exit 0
|
||
fi
|
||
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — would remove $UNBLOCK_HOST from blocklist"
|
||
exit 0
|
||
fi
|
||
|
||
remove_from_blocklist "$UNBLOCK_HOST"
|
||
warn "$UNBLOCK_HOST unblocked — re-onboarding is now permitted ✅"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "status" ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY PARTNERSHIP STATUS ━━━━━"
|
||
OWNER_IP=$(tailscale ip -4 "${OWNER,,}" 2>/dev/null || echo "unreachable")
|
||
MIRROR_IP=$(tailscale ip -4 "${MIRROR,,}" 2>/dev/null || echo "unreachable")
|
||
echo " $ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo " Owner: $OWNER_ID ($OWNER — $OWNER_IP)"
|
||
echo " Mirror: $MIRROR_ID ($MIRROR — $MIRROR_IP)"
|
||
echo " Role: $( [[ "$AM_OWNER" == true ]] && echo "OWNER" || echo "MIRROR" )"
|
||
echo " Enabled: ${PARTNERSHIP_ENABLED:-false}"
|
||
echo ""
|
||
|
||
# Local state
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||
LOCAL_ONBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "onboarded")
|
||
LOCAL_OFFBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "offboarded")
|
||
LOCAL_LAST_SEEN=$(read_state_file "$LOCAL_STATE_FILE" "last_seen_remote")
|
||
echo " Local state: $LOCAL_STATE"
|
||
[[ -n "$LOCAL_ONBOARDED" ]] && echo " Onboarded: $LOCAL_ONBOARDED"
|
||
[[ -n "$LOCAL_OFFBOARDED" ]] && echo " Offboarded: $LOCAL_OFFBOARDED"
|
||
[[ -n "$LOCAL_LAST_SEEN" ]] && echo " Remote seen: $LOCAL_LAST_SEEN"
|
||
else
|
||
echo " Local state: no state file found"
|
||
fi
|
||
|
||
echo ""
|
||
|
||
# Remote state
|
||
REMOTE_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null)
|
||
if [[ -n "$REMOTE_IP" ]]; then
|
||
REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE")
|
||
if [[ -n "$REMOTE_CONTENT" ]]; then
|
||
REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2)
|
||
REMOTE_ONBOARDED=$(echo "$REMOTE_CONTENT" | grep "^onboarded=" | cut -d= -f2)
|
||
echo " Remote state: $REMOTE_STATE"
|
||
[[ -n "$REMOTE_ONBOARDED" ]] && echo " Onboarded: $REMOTE_ONBOARDED"
|
||
if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]]; then
|
||
echo ""
|
||
echo " ✅ Both servers agree: $LOCAL_STATE"
|
||
else
|
||
echo ""
|
||
echo " ⚠️ State mismatch — local: $LOCAL_STATE remote: $REMOTE_STATE"
|
||
fi
|
||
else
|
||
echo " Remote state: not found on $REMOTE_SERVER_NAME"
|
||
fi
|
||
else
|
||
echo " Remote state: $REMOTE_SERVER_NAME unreachable"
|
||
fi
|
||
|
||
echo ""
|
||
echo " Auth WebUIs:"
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
echo " ${entry%%|*} → port ${entry##*|}"
|
||
done
|
||
|
||
if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then
|
||
echo ""
|
||
FOLDER_STATUS_NAME=$(derive_partner_folder_name "$REMOTE_SERVER_NAME")
|
||
if [[ -f "$FOLDERVIEW3_JSON" ]]; then
|
||
FOLDER_EXISTS=$(jq -r --arg name "$FOLDER_STATUS_NAME" \
|
||
'to_entries[] | select(.value.name == $name) | .key' \
|
||
"$FOLDERVIEW3_JSON" 2>/dev/null)
|
||
if [[ -n "$FOLDER_EXISTS" ]]; then
|
||
FOLDER_CONTAINERS=$(jq -r --arg name "$FOLDER_STATUS_NAME" \
|
||
'[to_entries[] | select(.value.name == $name) | .value.containers[]] | join(", ")' \
|
||
"$FOLDERVIEW3_JSON" 2>/dev/null)
|
||
echo " FolderView3: $FOLDER_STATUS_NAME ✅"
|
||
echo " Containers: $FOLDER_CONTAINERS"
|
||
else
|
||
echo " FolderView3: $FOLDER_STATUS_NAME — not found in JSON"
|
||
fi
|
||
else
|
||
echo " FolderView3: config not found ($FOLDERVIEW3_JSON)"
|
||
fi
|
||
fi
|
||
|
||
if [[ -f "$OFFLINE_COUNTER" ]]; then
|
||
OFFLINE_DAYS=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0)
|
||
[[ "$OFFLINE_DAYS" -gt 0 ]] && \
|
||
echo "" && \
|
||
echo " ⚠️ Remote offline counter: ${OFFLINE_DAYS}/${PARTNERSHIP_OFFLINE_THRESHOLD}"
|
||
fi
|
||
|
||
echo ""
|
||
if [[ -f "$BLOCKLIST_FILE" ]] && [[ -s "$BLOCKLIST_FILE" ]]; then
|
||
echo " Blocklist:"
|
||
while IFS='|' read -r host ts reason; do
|
||
echo " ⛔ $host (blocked $ts — $reason)"
|
||
done < "$BLOCKLIST_FILE"
|
||
else
|
||
echo " Blocklist: empty"
|
||
fi
|
||
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Check ━━━
|
||
# Called every 15min by critical_sync_maintenance.sh — must be silent when healthy
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "check" ]]; then
|
||
|
||
# Update last_seen_remote and offline counter based on rsync outcome
|
||
if [[ "$REMOTE_SEEN" == true ]]; then
|
||
echo "0" > "$OFFLINE_COUNTER"
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
sed -i "s|^last_seen_remote=.*|last_seen_remote=$(date '+%Y-%m-%d %H:%M:%S')|" \
|
||
"$LOCAL_STATE_FILE" 2>/dev/null
|
||
fi
|
||
elif [[ "$REMOTE_UNSEEN" == true ]]; then
|
||
OFFLINE_COUNT=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0)
|
||
OFFLINE_COUNT=$(( OFFLINE_COUNT + 1 ))
|
||
echo "$OFFLINE_COUNT" > "$OFFLINE_COUNTER"
|
||
|
||
# Auto-offboard threshold: threshold_days × 96 intervals/day (every 15min)
|
||
THRESHOLD_INTERVALS=$(( ${PARTNERSHIP_OFFLINE_THRESHOLD:-30} * 96 ))
|
||
if [[ "$OFFLINE_COUNT" -ge "$THRESHOLD_INTERVALS" ]]; then
|
||
warn "Remote offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days — triggering auto-offboard"
|
||
notify "Partnership auto-offboard on $(hostname) — $REMOTE_SERVER_NAME offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days" \
|
||
"Partnership" "warning"
|
||
bash "$0" --offboard --reason=auto-offboard-timeout
|
||
exit 0
|
||
fi
|
||
log "Partnership check — remote unseen ($OFFLINE_COUNT/$THRESHOLD_INTERVALS)"
|
||
fi
|
||
|
||
# Skip blocklisted partners — they can't auto-reconnect; only --onboard re-establishes
|
||
if is_blocklisted "$REMOTE_SERVER_NAME"; then
|
||
log "Partnership check — $REMOTE_SERVER_NAME is blocklisted, skipping"
|
||
exit 0
|
||
fi
|
||
|
||
# Read remote state file
|
||
REMOTE_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null)
|
||
if [[ -z "$REMOTE_IP" ]]; then
|
||
log "Partnership check — remote unreachable, skipping state check"
|
||
exit 0
|
||
fi
|
||
|
||
REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE")
|
||
if [[ -z "$REMOTE_CONTENT" ]]; then
|
||
# IP resolved but SSH returned nothing — could be auth failure, not just missing file
|
||
if [[ -f "$SCRIPT_DIR/../Initial_run/ssh_setup.sh" ]]; then
|
||
bash "$SCRIPT_DIR/../Initial_run/ssh_setup.sh" --validate 2>/dev/null || true
|
||
fi
|
||
log "Partnership check — remote state file not found"
|
||
exit 0
|
||
fi
|
||
|
||
REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2)
|
||
LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state" 2>/dev/null || echo "UNKNOWN")
|
||
|
||
# Both agree and active — healthy, silent
|
||
if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then
|
||
log "Partnership check — ACTIVE, both servers agree ✅"
|
||
exit 0
|
||
fi
|
||
|
||
# Remote requested offboard
|
||
if [[ "$REMOTE_STATE" == "INACTIVE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then
|
||
warn "Partnership check — $REMOTE_SERVER_NAME requested offboard"
|
||
|
||
if [[ "$AM_OWNER" == true ]]; then
|
||
warn "Owner finalising offboard request from mirror..."
|
||
do_final_sync
|
||
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
write_state_file "$LOCAL_STATE_FILE" \
|
||
"INACTIVE" "" "$NOW" "$REMOTE_SERVER_NAME" "mirror-requested"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false"
|
||
fi
|
||
|
||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||
local grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 ))
|
||
warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace period..."
|
||
# Grace sleep — interruptible via SIGTERM trap
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
trap 'warn "Partnership check interrupted during grace sleep"; exit 0' SIGTERM SIGINT
|
||
sleep "$grace_seconds"
|
||
trap - SIGTERM SIGINT
|
||
fi
|
||
remove_tailscale_device "$MIRROR"
|
||
fi
|
||
notify "Partnership offboard finalised — $MIRROR requested separation" \
|
||
"Partnership" "normal"
|
||
else
|
||
# Mirror sees owner is INACTIVE — clean up own side
|
||
warn "Owner has offboarded — cleaning up mirror side..."
|
||
reconfigure_local_webuis "localhost"
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
write_state_file "$LOCAL_STATE_FILE" \
|
||
"INACTIVE" "" "$NOW" "$OWNER" "owner-offboarded"
|
||
notify "Partnership ended — $OWNER offboarded. Auth WebUIs → localhost." \
|
||
"Partnership" "normal"
|
||
fi
|
||
exit 0
|
||
fi
|
||
|
||
# Both inactive — nothing to do
|
||
if [[ "$LOCAL_STATE" == "INACTIVE" ]] && [[ "$REMOTE_STATE" == "INACTIVE" ]]; then
|
||
log "Partnership check — INACTIVE on both servers"
|
||
exit 0
|
||
fi
|
||
|
||
log "Partnership check — local: $LOCAL_STATE remote: $REMOTE_STATE"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Onboard ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "onboard" ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_FAILOVER Onboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
|
||
# Check already onboarded
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||
if [[ "$CURRENT_STATE" == "ACTIVE" ]]; then
|
||
warn "Partnership already ACTIVE — use --status for detail or --offboard to separate"
|
||
exit 0
|
||
fi
|
||
fi
|
||
|
||
# Warn if re-onboarding a previously blocked partner — onboard is deliberate so it proceeds
|
||
if is_blocklisted "$MIRROR"; then
|
||
warn "$MIRROR is on the blocklist from a previous offboard"
|
||
warn "Proceeding — blocklist will be cleared on successful onboard"
|
||
fi
|
||
|
||
log "Owner: $OWNER_ID ($OWNER)"
|
||
log "Mirror: $MIRROR_ID ($MIRROR)"
|
||
|
||
# Pre-flight
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||
|
||
resolve_remote_ip
|
||
check_connectivity
|
||
|
||
# Version parity — both servers must agree on unRAID version
|
||
check_unraid_version_parity || exit 1
|
||
|
||
# Remote array and Docker daemon
|
||
check_remote_array || exit 1
|
||
check_remote_docker_daemon || exit 1
|
||
|
||
OWNER_IP=$(tailscale ip -4 "${OWNER,,}" 2>/dev/null)
|
||
MIRROR_IP=$(tailscale ip -4 "${MIRROR,,}" 2>/dev/null)
|
||
|
||
[[ -z "$OWNER_IP" ]] && { error "Cannot resolve $OWNER Tailscale IP"; exit 1; }
|
||
[[ -z "$MIRROR_IP" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; exit 1; }
|
||
|
||
log "Owner IP: $OWNER_IP"
|
||
log "Mirror IP: $MIRROR_IP"
|
||
|
||
# Reconfigure mirror WebUIs → owner IP
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → $OWNER_IP ━━━"
|
||
|
||
WEBUI_FAILURES=0
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
reconfigure_webui "$container" "$port" "$OWNER_IP" \
|
||
"$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ ))
|
||
done
|
||
|
||
# Verify WebUI connectivity
|
||
if [[ "${PARTNERSHIP_ONBOARD_VERIFY:-true}" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_VERIFY WebUI Connectivity ━━━"
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
if curl -sf --max-time 10 "http://${OWNER_IP}:${port}/" >/dev/null 2>&1; then
|
||
log "$container reachable at http://${OWNER_IP}:${port}/ ✅"
|
||
else
|
||
warn "$container not reachable at http://${OWNER_IP}:${port}/ — may not be running"
|
||
fi
|
||
done
|
||
fi
|
||
|
||
# Write state files
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Write State ━━━"
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
|
||
write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "onboard"
|
||
log "Local state: ACTIVE ✅"
|
||
|
||
[[ "$DRY_RUN" == false ]] && remove_from_blocklist "$MIRROR"
|
||
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||
echo "0" > "$OFFLINE_COUNTER"
|
||
|
||
# FolderView3 — create partner folder with this server's failover containers for remote
|
||
if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS FolderView3 Integration ━━━"
|
||
PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR")
|
||
declare -a PARTNER_CONTAINERS=()
|
||
gather_partner_failover_containers PARTNER_CONTAINERS
|
||
if [[ ${#PARTNER_CONTAINERS[@]} -gt 0 ]]; then
|
||
folderview3_create_partner_folder "$PARTNER_FOLDER_NAME" "${PARTNER_CONTAINERS[@]}"
|
||
else
|
||
log "No FAILOVER_${MY_ID}_RUNS_FOR_${REMOTE_ID}_TIER* containers configured — skipping folder creation"
|
||
fi
|
||
fi
|
||
|
||
# Summary
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━"
|
||
echo " Owner: $OWNER_ID ($OWNER_IP)"
|
||
echo " Mirror: $MIRROR_ID ($MIRROR_IP)"
|
||
echo " WebUI failures: $WEBUI_FAILURES"
|
||
echo " Sync interval: ${PARTNERSHIP_SYNC_INTERVAL}min"
|
||
echo ""
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — no changes made"
|
||
elif [[ "$WEBUI_FAILURES" -eq 0 ]]; then
|
||
warn "$ICON_DONE DONE — $MIRROR is now mirroring $OWNER via Tailscale ✅"
|
||
[[ "${PARTNERSHIP_ONBOARD_NOTIFY:-true}" == true ]] && \
|
||
notify "Partnership onboard complete — $MIRROR is now mirroring $OWNER via Tailscale" \
|
||
"Partnership" "normal"
|
||
else
|
||
warn "DONE with $WEBUI_FAILURES WebUI warning(s) — check manually"
|
||
notify "Partnership onboard complete with $WEBUI_FAILURES WebUI warning(s) on $(hostname)" \
|
||
"Partnership" "warning"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Offboard ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "offboard" ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_FAILOVER Offboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
|
||
# Check already offboarded
|
||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||
if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then
|
||
warn "Partnership already INACTIVE — use --status to verify both servers agree"
|
||
exit 0
|
||
fi
|
||
fi
|
||
|
||
# ── Mirror-initiated offboard ─────────────────────────────────────────────────────────────
|
||
if [[ "$AM_MIRROR" == true ]]; then
|
||
warn "$MIRROR_ID ($MIRROR) is initiating offboard"
|
||
warn "Local auth WebUIs will be reconfigured → localhost"
|
||
warn "$OWNER_ID ($OWNER) will finalise on its next --check cycle"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
echo ""
|
||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||
sleep 10
|
||
fi
|
||
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Reconfigure Local WebUIs → localhost ━━━"
|
||
reconfigure_local_webuis "localhost"
|
||
|
||
# FolderView3 — remove partner folder and clean containers
|
||
if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS FolderView3 Cleanup ━━━"
|
||
PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$OWNER")
|
||
folderview3_remove_partner_folder "$PARTNER_FOLDER_NAME"
|
||
fi
|
||
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
write_state_file "$LOCAL_STATE_FILE" \
|
||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||
log "Local state: INACTIVE ✅"
|
||
|
||
[[ "$DRY_RUN" == false ]] && add_to_blocklist "$OWNER" "$REASON"
|
||
|
||
OWNER_IP=$(tailscale ip -4 "${OWNER,,}" 2>/dev/null)
|
||
if [[ -n "$OWNER_IP" ]]; then
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$MIRROR_SSH_KEY"
|
||
notify "Partnership offboard requested by $MIRROR — $OWNER will finalise on next check" \
|
||
"Partnership" "normal"
|
||
else
|
||
warn "$OWNER unreachable — state written locally, owner will see it when reachable"
|
||
fi
|
||
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━"
|
||
echo " Your WebUIs: reconfigured → localhost ✅"
|
||
echo " State: INACTIVE ✅"
|
||
echo " Blocklist: $OWNER blocked — re-onboard to permit access again ✅"
|
||
echo " Owner: will finalise + final sync on next --check ✅"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ── Owner-initiated offboard ──────────────────────────────────────────────────────────────
|
||
warn "Offboarding $MIRROR_ID ($MIRROR) from partnership"
|
||
warn "Final sync will run — mirror leaves with current state"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
echo ""
|
||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||
sleep 10
|
||
echo "Proceeding..."
|
||
fi
|
||
|
||
resolve_remote_ip
|
||
|
||
# Stop any running rsync first
|
||
echo ""
|
||
echo "━━━ $ICON_STOP Stop Running Rsync ━━━"
|
||
bash "$SCRIPT_DIR/../Rsync/rsync_stop.sh" --rsync-only 2>/dev/null || true
|
||
|
||
# Final sync
|
||
echo ""
|
||
echo "━━━ $ICON_SYNC Final Sync ━━━"
|
||
do_final_sync
|
||
|
||
MIRROR_IP=$(tailscale ip -4 "${MIRROR,,}" 2>/dev/null)
|
||
MIRROR_REACHABLE=false
|
||
[[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true
|
||
|
||
# Reconfigure mirror WebUIs → localhost
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → localhost ━━━"
|
||
|
||
WEBUI_FAILURES=0
|
||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
reconfigure_webui "$container" "$port" "localhost" \
|
||
"$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || (( WEBUI_FAILURES++ ))
|
||
done
|
||
else
|
||
warn "$MIRROR unreachable — WebUI reconfiguration skipped"
|
||
warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on --check"
|
||
(( WEBUI_FAILURES++ ))
|
||
fi
|
||
|
||
# Disable critical rsync
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Disable Critical Sync ━━━"
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
update_master_conf "CRITICAL_RSYNC_ENABLED" "false"
|
||
warn "CRITICAL_RSYNC_ENABLED=false ✅"
|
||
else
|
||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
||
fi
|
||
|
||
# Write state files
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Write State ━━━"
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
|
||
write_state_file "$LOCAL_STATE_FILE" \
|
||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||
log "Local state: INACTIVE ✅"
|
||
|
||
[[ "$DRY_RUN" == false ]] && add_to_blocklist "$MIRROR" "$REASON"
|
||
|
||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||
fi
|
||
|
||
# FolderView3 — remove partner folder and clean containers on this (owner) side
|
||
if [[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS FolderView3 Cleanup ━━━"
|
||
PARTNER_FOLDER_NAME=$(derive_partner_folder_name "$MIRROR")
|
||
folderview3_remove_partner_folder "$PARTNER_FOLDER_NAME"
|
||
fi
|
||
|
||
# Tailscale removal
|
||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_NET Tailscale Separation ━━━"
|
||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||
grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-6} * 3600 ))
|
||
warn "Waiting ${PARTNERSHIP_GRACE_HOURS:-6}hr grace — mirror can collect backups..."
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
trap 'warn "Offboard interrupted during grace sleep"; exit 0' SIGTERM SIGINT
|
||
sleep "$grace_seconds"
|
||
trap - SIGTERM SIGINT
|
||
fi
|
||
fi
|
||
remove_tailscale_device "$MIRROR"
|
||
fi
|
||
|
||
# Backup notification
|
||
if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_DISK Backup Handover ━━━"
|
||
log "Backups available for $MIRROR:"
|
||
for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do
|
||
[[ -z "$path" ]] && continue
|
||
echo " $path"
|
||
done
|
||
notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-6}hr. Tailscale access expires then." \
|
||
"Partnership" "warning"
|
||
fi
|
||
|
||
# Summary
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━"
|
||
echo " Owner: $OWNER_ID ($OWNER)"
|
||
echo " Mirror: $MIRROR_ID ($MIRROR)"
|
||
echo " Final sync: complete ✅"
|
||
echo " WebUI failures: $WEBUI_FAILURES"
|
||
echo " Critical rsync: disabled ✅"
|
||
echo " State: INACTIVE ✅"
|
||
echo " Blocklist: $MIRROR blocked — re-onboard to permit access again ✅"
|
||
[[ "${PARTNERSHIP_FOLDERVIEW3:-false}" == true ]] && \
|
||
echo " FolderView3: ${PARTNER_FOLDER_NAME:-} cleaned ✅"
|
||
[[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \
|
||
echo " Tailscale: $MIRROR removed ✅"
|
||
echo ""
|
||
echo " $MIRROR leaves with:"
|
||
echo " ✓ Current auth config (final sync)"
|
||
echo " ✓ Auth WebUIs → localhost"
|
||
echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-6}hr to collect backups"
|
||
echo " ✓ Full ecosystem — just stop the sync"
|
||
echo ""
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes made" || \
|
||
warn "$ICON_DONE DONE — clean separation complete ✅"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Transfer ━━━
|
||
# ==============================================================================================
|
||
if [[ "$MODE" == "transfer" ]]; then
|
||
echo ""
|
||
echo "━━━ $ICON_FAILOVER Transfer Ownership — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||
echo ""
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo "⚠️ WARNING — OWNERSHIP TRANSFER"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " Current owner: $OWNER_ID ($OWNER)"
|
||
echo " Current mirror: $MIRROR_ID ($MIRROR)"
|
||
echo ""
|
||
echo " After transfer:"
|
||
echo " New owner: $MIRROR_ID ($MIRROR)"
|
||
echo " New mirror: $OWNER_ID ($OWNER)"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
# Confirmation check
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
if [[ -z "$TRANSFER_CONFIRM_INPUT" ]]; then
|
||
echo ""
|
||
echo "To proceed pass exactly:"
|
||
echo " --confirm=${PARTNERSHIP_TRANSFER_CONFIRM}"
|
||
echo ""
|
||
error "Transfer cancelled — confirmation required"
|
||
exit 1
|
||
fi
|
||
if [[ "$TRANSFER_CONFIRM_INPUT" != "$PARTNERSHIP_TRANSFER_CONFIRM" ]]; then
|
||
error "Confirmation string does not match — transfer cancelled"
|
||
exit 1
|
||
fi
|
||
log "Confirmation accepted"
|
||
else
|
||
warn "DRY RUN — confirmation check skipped"
|
||
fi
|
||
|
||
# Health strike system
|
||
echo ""
|
||
echo "━━━ $ICON_SHIELD Health Verification ━━━"
|
||
log "Both servers must pass ${PARTNERSHIP_TRANSFER_STRIKES} consecutive health checks"
|
||
|
||
STRIKES=0
|
||
ATTEMPTS=0
|
||
MAX_ATTEMPTS="${PARTNERSHIP_TRANSFER_MAX_ATTEMPTS:-20}"
|
||
|
||
while [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]]; do
|
||
(( ATTEMPTS++ ))
|
||
if [[ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]]; then
|
||
error "Health checks failed after $MAX_ATTEMPTS attempts — servers not stable"
|
||
error "Transfer cancelled — try again when both servers are healthy"
|
||
exit 1
|
||
fi
|
||
|
||
if check_both_healthy; then
|
||
(( STRIKES++ ))
|
||
log "Health check passed ($STRIKES/${PARTNERSHIP_TRANSFER_STRIKES})"
|
||
[[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]] && sleep 10
|
||
else
|
||
warn "Health check failed — resetting (attempt $ATTEMPTS/$MAX_ATTEMPTS)"
|
||
STRIKES=0
|
||
sleep 30
|
||
fi
|
||
done
|
||
warn "Both servers healthy — proceeding with transfer"
|
||
|
||
NEW_OWNER_ID="$MIRROR_ID"
|
||
NEW_MIRROR_ID="$OWNER_ID"
|
||
NEW_OWNER="$MIRROR"
|
||
NEW_MIRROR="$OWNER"
|
||
NEW_OWNER_SSH_KEY_VAR="${NEW_OWNER_ID}_SSH_KEY"
|
||
NEW_MIRROR_SSH_KEY_VAR="${NEW_MIRROR_ID}_SSH_KEY"
|
||
NEW_OWNER_SSH_KEY="${!NEW_OWNER_SSH_KEY_VAR}"
|
||
NEW_MIRROR_SSH_KEY="${!NEW_MIRROR_SSH_KEY_VAR}"
|
||
|
||
NEW_OWNER_IP=$(tailscale ip -4 "${NEW_OWNER,,}" 2>/dev/null)
|
||
NEW_MIRROR_IP=$(tailscale ip -4 "${NEW_MIRROR,,}" 2>/dev/null)
|
||
|
||
[[ -z "$NEW_OWNER_IP" ]] && { error "Cannot resolve new owner Tailscale IP"; exit 1; }
|
||
[[ -z "$NEW_MIRROR_IP" ]] && { error "Cannot resolve new mirror Tailscale IP"; exit 1; }
|
||
|
||
# Final sync in current direction
|
||
echo ""
|
||
echo "━━━ $ICON_SYNC Pre-transfer Sync ━━━"
|
||
do_final_sync
|
||
|
||
# Reconfigure new mirror WebUIs → new owner
|
||
echo ""
|
||
echo "━━━ $ICON_CONTAINERS Reconfigure WebUIs ━━━"
|
||
log "New mirror ($NEW_MIRROR) WebUIs → new owner ($NEW_OWNER_IP)"
|
||
|
||
WEBUI_FAILURES=0
|
||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]}"; do
|
||
[[ -z "$entry" ]] && continue
|
||
container="${entry%%|*}"
|
||
port="${entry##*|}"
|
||
reconfigure_webui "$container" "$port" "$NEW_OWNER_IP" \
|
||
"$NEW_MIRROR_SSH_KEY" "$NEW_MIRROR_IP" "$NEW_MIRROR" || (( WEBUI_FAILURES++ ))
|
||
done
|
||
|
||
# New owner WebUIs → localhost (now manages directly)
|
||
log "New owner ($NEW_OWNER) WebUIs → localhost"
|
||
reconfigure_local_webuis "localhost"
|
||
|
||
# Flip PARTNERSHIP_OWNER_HOST in master.conf on both servers
|
||
echo ""
|
||
echo "━━━ $ICON_GEAR Update Ownership ━━━"
|
||
|
||
if [[ "$DRY_RUN" == false ]]; then
|
||
update_master_conf "PARTNERSHIP_OWNER_HOST" "\"$NEW_OWNER_ID\""
|
||
|
||
# Update remote master.conf
|
||
timeout "$SSH_TIMEOUT" ssh -i "$NEW_MIRROR_SSH_KEY" \
|
||
-o ConnectTimeout="$SSH_TIMEOUT" root@"$NEW_MIRROR_IP" \
|
||
"sed -i 's|^[[:space:]]*PARTNERSHIP_OWNER_HOST=.*| PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_ID\"|' \
|
||
'$SCRIPT_DIR/../master.conf'" 2>/dev/null && \
|
||
log "Remote master.conf updated ✅" || \
|
||
error "Failed to update remote master.conf — update manually"
|
||
else
|
||
warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID on both servers"
|
||
fi
|
||
|
||
# Write state files
|
||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||
write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "transfer"
|
||
push_state_to_remote "$LOCAL_STATE_FILE" "$NEW_MIRROR_IP" "$NEW_MIRROR_SSH_KEY"
|
||
|
||
# Summary
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY TRANSFER SUMMARY ━━━━━"
|
||
echo " New owner: $NEW_OWNER_ID ($NEW_OWNER_IP)"
|
||
echo " New mirror: $NEW_MIRROR_ID ($NEW_MIRROR_IP)"
|
||
echo " WebUI failures: $WEBUI_FAILURES"
|
||
echo " Sync direction: $NEW_OWNER → $NEW_MIRROR"
|
||
echo " Owner host: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_ID"
|
||
echo ""
|
||
if [[ "$DRY_RUN" == true ]]; then
|
||
warn "DRY RUN — no changes made"
|
||
else
|
||
warn "$ICON_DONE DONE — ownership transferred to $NEW_OWNER_ID ✅"
|
||
notify "Partnership ownership transferred — new owner: $NEW_OWNER ($NEW_OWNER_ID)" \
|
||
"Partnership" "normal"
|
||
fi
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi |