padded partnership set up
This commit is contained in:
+717
-780
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,172 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------------------- Critical Sync Maintenance --------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Orchestrator for time-sensitive syncs that run every 15 minutes.
|
||||
# Keeps the mirror current between the less frequent daily and weekly windows.
|
||||
# Schedule: */15 * * * * (every 15 minutes)
|
||||
#
|
||||
# Execution order:
|
||||
# 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides)
|
||||
# 2. emby-failover rsync — dirty Emby sync (watch states, library — Emby stays running)
|
||||
# 3. partnership --check — read both state files, detect changes, act accordingly
|
||||
#
|
||||
# Why every 15 minutes:
|
||||
# Auth stack changes (new users, proxy rules, certs) propagate within 15min ✅
|
||||
# Emby watch states stay in sync — mirror users see correct playback position ✅
|
||||
# Partnership state changes detected and acted on quickly ✅
|
||||
#
|
||||
# Rsync gate:
|
||||
# RSYNC_ENABLED=false → skips all syncs (global gate)
|
||||
# CRITICAL_RSYNC_ENABLED=false → skips critical syncs (per-orchestrator)
|
||||
# partnership --check still runs regardless of rsync gate
|
||||
# (state check doesn't need rsync to work)
|
||||
#
|
||||
# Lock behavior:
|
||||
# acquire_lock "strict" — if previous 15min run still going, skip this cycle
|
||||
# Critical-Data taking > 15min is a problem worth knowing about
|
||||
# Lock prevents pile-up ✅
|
||||
#
|
||||
# Configuration in Master.conf:
|
||||
# CRITICAL_RSYNC_ENABLED — enable/disable rsync section
|
||||
# PARTNERSHIP_ENABLED — enable/disable partnership check
|
||||
# CRITICAL_SYNC_SHARES — shares synced every 15min
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
acquire_lock "strict"
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
START=$(date +%s)
|
||||
RSYNC_OK=false
|
||||
PASS=()
|
||||
FAIL=()
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Critical Shares Sync ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Critical Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
if ! check_rsync_enabled "CRITICAL"; then
|
||||
warn "Critical rsync disabled — skipping sync, running partnership check only"
|
||||
else
|
||||
for share in "${CRITICAL_SYNC_SHARES[@]:-}"; do
|
||||
[[ -z "$share" ]] && continue
|
||||
|
||||
# Parse optional profile flag: "/path/to/share|profile-name"
|
||||
SHARE_PATH="${share%%|*}"
|
||||
SHARE_PROFILE="${share##*|}"
|
||||
SHARE_NAME=$(basename "$SHARE_PATH")
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC $SHARE_NAME ━━━"
|
||||
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then
|
||||
# No profile specified
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH"
|
||||
else
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" --profile="$SHARE_PROFILE"
|
||||
fi
|
||||
|
||||
RSYNC_EXIT=$?
|
||||
SHARE_END=$(date +%s)
|
||||
SHARE_DUR=$(format_duration $(( SHARE_END - SHARE_START )))
|
||||
|
||||
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
|
||||
PASS+=("$SHARE_NAME")
|
||||
success "$SHARE_NAME — done in $SHARE_DUR ✅"
|
||||
RSYNC_OK=true
|
||||
else
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME — failed after $SHARE_DUR"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CLEAN Critical Maintenance Scripts ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Critical Maintenance — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
||||
log "No CRITICAL_MAINTENANCE_SCRIPTS defined — skipping"
|
||||
else
|
||||
for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
# Strip leading comment lines
|
||||
[[ "$script_entry" == \#* ]] && continue
|
||||
|
||||
SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
|
||||
SCRIPT_ARGS="${script_entry#* }"
|
||||
[[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
|
||||
|
||||
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
|
||||
echo " → $SCRIPT_NAME"
|
||||
|
||||
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
||||
warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
bash "$SCRIPT_PATH" $SCRIPT_ARGS
|
||||
EXIT_CODE=$?
|
||||
if [[ "$EXIT_CODE" -ne 0 ]]; then
|
||||
warn "$SCRIPT_NAME exited with code $EXIT_CODE"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SHIELD Partnership Check ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Partnership Check ━━━"
|
||||
|
||||
if [[ "${PARTNERSHIP_ENABLED:-false}" == false ]]; then
|
||||
log "Partnership disabled — skipping check"
|
||||
else
|
||||
# Pass rsync outcome to --check so it can update last_seen_remote
|
||||
if [[ "$RSYNC_OK" == true ]]; then
|
||||
bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-seen
|
||||
else
|
||||
bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-unseen
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
END=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
|
||||
if [[ ${#PASS[@]} -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Synced: ${PASS[*]}"
|
||||
fi
|
||||
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Failed: ${FAIL[*]}"
|
||||
notify "Critical sync failed on $(hostname) — ${FAIL[*]}" "Critical Sync" "warning"
|
||||
fi
|
||||
if [[ ${#PASS[@]} -eq 0 ]] && [[ ${#FAIL[@]} -eq 0 ]]; then
|
||||
echo "$ICON_SKIP Rsync: disabled"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -23,7 +23,7 @@
|
||||
#
|
||||
# Configuration in Master.conf:
|
||||
# DAILY_MAINTENANCE_SCRIPTS — pre/post-sync scripts (git pull, docker restart)
|
||||
# MEDIA_MANAGEMENT_JOBS — media maintenance jobs run after sync
|
||||
# DAILY_MAINTENANCE_SCRIPTS — media maintenance jobs run after sync
|
||||
# HOST1_DAILY_SYNC_SHARES — shares HOST1 pushes to HOST2
|
||||
# HOST2_DAILY_SYNC_SHARES — shares HOST2 pushes to HOST1
|
||||
# HOST1/2_PERSONAL_SHARES — encrypted personal shares
|
||||
@@ -201,14 +201,14 @@ TOTAL_DURATION=$((TOTAL_END - TOTAL_START))
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CLEAN Post-sync Media Jobs ━━━
|
||||
# Reads MEDIA_MANAGEMENT_JOBS from Master.conf — permissions, cleaners, arr cleanup
|
||||
# Reads DAILY_MAINTENANCE_SCRIPTS from Master.conf — permissions, cleaners, arr cleanup
|
||||
# Runs after sync completes — correct ownership available, clean folders guaranteed
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Post-sync Media Jobs ━━━"
|
||||
|
||||
if [[ ${#MEDIA_MANAGEMENT_JOBS[@]} -gt 0 ]]; then
|
||||
for script_entry in "${MEDIA_MANAGEMENT_JOBS[@]}"; do
|
||||
if [[ ${#DAILY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
|
||||
@@ -8,15 +8,15 @@
|
||||
# Execution order:
|
||||
# 1. Stop local containers — Emby + auth stack stopped locally
|
||||
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
|
||||
# 3. Pull updates locally — if CRITICAL_SYNC_UPDATES=true
|
||||
# 4. Pull updates remotely — if CRITICAL_SYNC_UPDATES_REMOTE=true
|
||||
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true
|
||||
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
|
||||
# 5. rsync Emby — full clean mirror, both instances stopped
|
||||
# 6. rsync Critical-Data — auth stack clean sync, databases flushed
|
||||
# 7. Start remote containers — correct order, delayed start respected
|
||||
# 8. Start local containers — correct order, delayed start respected
|
||||
# 9. docker_weekly_restart.sh — weekly container restarts
|
||||
#
|
||||
# Synced shares (WEEKLY_SYNC_JOBS in Master.conf):
|
||||
# Synced shares (WEEKLY_SYNC_SHARES in Master.conf):
|
||||
# /mnt/user/Media_Server/Emby — emby profile — full mirror, cache resets weekly
|
||||
# /mnt/user/appdata-Failover/Critical-Data — critical-data — auth stack clean state
|
||||
#
|
||||
@@ -29,16 +29,16 @@
|
||||
# Container updates during the window:
|
||||
# Containers already stopped for sync — updates pull at zero extra downtime
|
||||
# Both servers start on identical image versions after the window completes
|
||||
# Toggle: CRITICAL_SYNC_UPDATES / CRITICAL_SYNC_UPDATES_REMOTE in Master.conf
|
||||
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in Master.conf
|
||||
#
|
||||
# What triggers weekly_health_digest.sh:
|
||||
# NOT this script — weekly_health_digest.sh runs on its own Saturday schedule
|
||||
#
|
||||
# Configuration in Master.conf:
|
||||
# WEEKLY_SYNC_JOBS — shares synced during the maintenance window
|
||||
# WEEKLY_SYNC_SHARES — shares synced during the maintenance window
|
||||
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync (docker_weekly_restart)
|
||||
# CRITICAL_SYNC_UPDATES — toggle container updates on/off
|
||||
# CRITICAL_SYNC_UPDATES_REMOTE — toggle remote container updates on/off
|
||||
# WEEKLY_SYNC_UPDATES — toggle container updates on/off
|
||||
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates on/off
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# All configuration in Master.conf.
|
||||
# Supports --dry-run to walk through without stopping containers, syncing, or updating.
|
||||
@@ -107,7 +107,7 @@ fi
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Container Updates ━━━"
|
||||
|
||||
if [[ "$CRITICAL_SYNC_UPDATES" == true ]]; then
|
||||
if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull updates for local containers"
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
@@ -133,10 +133,10 @@ if [[ "$CRITICAL_SYNC_UPDATES" == true ]]; then
|
||||
done
|
||||
fi
|
||||
else
|
||||
info "CRITICAL_SYNC_UPDATES=false — skipping local updates"
|
||||
info "WEEKLY_SYNC_UPDATES=false — skipping local updates"
|
||||
fi
|
||||
|
||||
if [[ "$CRITICAL_SYNC_UPDATES_REMOTE" == true ]]; then
|
||||
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
|
||||
else
|
||||
@@ -158,7 +158,7 @@ if [[ "$CRITICAL_SYNC_UPDATES_REMOTE" == true ]]; then
|
||||
done
|
||||
fi
|
||||
else
|
||||
info "CRITICAL_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
||||
info "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
@@ -168,7 +168,7 @@ PASS=()
|
||||
FAIL=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
SYNC_JOBS=("${WEEKLY_SYNC_JOBS[@]}")
|
||||
SYNC_JOBS=("${WEEKLY_SYNC_SHARES[@]}")
|
||||
SHARE_COUNT=${#SYNC_JOBS[@]}
|
||||
|
||||
echo ""
|
||||
@@ -280,7 +280,7 @@ WINDOW_END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $TOTAL_DURATION"
|
||||
echo "$ICON_GEAR Updates: local=$CRITICAL_SYNC_UPDATES remote=$CRITICAL_SYNC_UPDATES_REMOTE"
|
||||
echo "$ICON_GEAR Updates: local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Sync jobs:"
|
||||
@@ -307,7 +307,7 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE"
|
||||
notify "Weekly sync maintenance complete on $(hostname) — synced + updated (local=$CRITICAL_SYNC_UPDATES remote=$CRITICAL_SYNC_UPDATES_REMOTE)" "Weekly Maintenance" "normal"
|
||||
notify "Weekly sync maintenance complete on $(hostname) — synced + updated (local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE)" "Weekly Maintenance" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs"
|
||||
notify "Weekly sync maintenance failed on $(hostname) — sync: ${#FAIL[@]} failed, jobs: ${#JOB_FAIL[@]} failed" "Weekly Maintenance" "warning"
|
||||
|
||||
@@ -0,0 +1,438 @@
|
||||
# Partnership
|
||||
|
||||
Manages the relationship lifecycle between two unRAID servers. Handles onboarding a new mirror, clean separation when someone wants to leave, and transferring ownership when the arrangement changes.
|
||||
|
||||
> This script exists because the partnership between two servers has a lifecycle — and that lifecycle deserves the same engineering discipline as everything else in the ecosystem. A clean exit should be as easy as a clean setup.
|
||||
|
||||
---
|
||||
|
||||
## The Relationship Model
|
||||
|
||||
Two servers. One owns the shared services. One mirrors them.
|
||||
|
||||
```
|
||||
HOST1 (owner) — source of truth
|
||||
Auth stack ← all changes made here
|
||||
NPM proxy rules ← created here, mirrored to mirror
|
||||
Certs ← renewed here, mirrored to mirror
|
||||
Emby ← runs here, dirty-synced every 15min
|
||||
Master.conf ← source of truth, git push propagates
|
||||
|
||||
HOST2 (mirror) — warm copy
|
||||
Auth containers ← running, serving his domain
|
||||
NPM ← running owner's config
|
||||
Certs ← current, mirrored every 15min
|
||||
WebUIs ← click any auth container → opens owner's WebUI via Tailscale
|
||||
Never makes changes directly ← overwritten on next sync
|
||||
```
|
||||
|
||||
**HOST1 is always the owner. HOST2 is always the mirror.**
|
||||
`PARTNERSHIP_OWNER_HOST="HOST1"` in Master.conf — flips to `"HOST2"` only after `--transfer`.
|
||||
Everything derives from HOST1/HOST2 — no duplicate IP or key configuration needed.
|
||||
|
||||
**The mirror never needs to think about auth management.** His Docker UI opens, he clicks an auth container, lands on the owner's WebUI automatically via Tailscale. Changes made there sync to his server in 15 minutes. He benefits from every improvement the owner makes without doing anything.
|
||||
|
||||
**The owner never needs to think about the mirror's server.** Everything propagates automatically. Critical-Data syncs every 15 minutes via `critical_sync_maintenance.sh`. Certs stay current. Config stays consistent.
|
||||
|
||||
---
|
||||
|
||||
## What Makes It Work
|
||||
|
||||
Three things make the mirror transparent to end users:
|
||||
|
||||
**1. Auth stack warm on both servers**
|
||||
|
||||
NPM, LLDAP, Authelia, certs — all running on both servers simultaneously. When traffic hits the mirror's domain, auth is already ready. No cold start, no broken auth window.
|
||||
|
||||
**2. WebUIs pointing to owner**
|
||||
|
||||
Every auth container on the mirror has its WebUI URL configured to point at the owner's Tailscale IP. From the mirror operator's perspective: click container, get owner's UI. From the owner's perspective: one place to manage everything.
|
||||
|
||||
**3. Tailscale-only communication**
|
||||
|
||||
All traffic between servers goes through Tailscale. Encrypted, no open ports, no VPN configuration. The WebUI redirect works because Tailscale keeps both servers permanently connected. HOST1 and HOST2 hostnames must match their exact Tailscale device names — already enforced by the ecosystem.
|
||||
|
||||
---
|
||||
|
||||
## State Files
|
||||
|
||||
Partnership state is tracked in two files on `/boot/config` — survives reboots, available before the array starts, minimal flash wear.
|
||||
|
||||
```
|
||||
/boot/config/partnership_HOST1.db ← HOST1 writes only
|
||||
/boot/config/partnership_HOST2.db ← HOST2 writes only
|
||||
```
|
||||
|
||||
Each server writes **only its own file**. State propagates via SSH — no rsync needed. `critical_sync_maintenance.sh` calls `partnership_manage.sh --check` every 15 minutes, which SSHes to read the remote file and act on any changes.
|
||||
|
||||
**This is how deferred offboard works:** HOST2 offboards while HOST1 is temporarily unreachable → HOST2 writes its state file → next time HOST1 can reach HOST2 → reads HOST2's state → finalises from its side automatically.
|
||||
|
||||
A per-server offline counter (`/boot/config/partnership_offline_days.db`) tracks consecutive missed 15-minute cycles. After `PARTNERSHIP_OFFLINE_THRESHOLD` days either server is unreachable, both independently auto-offboard.
|
||||
|
||||
---
|
||||
|
||||
## The Lifecycle
|
||||
|
||||
### Onboard — Setting Up a New Mirror
|
||||
|
||||
Run `--onboard` from HOST1 (owner) to establish the relationship:
|
||||
|
||||
```bash
|
||||
# Always dry-run first
|
||||
partnership_manage.sh --onboard --dry-run
|
||||
|
||||
# Live onboard
|
||||
partnership_manage.sh --onboard
|
||||
```
|
||||
|
||||
**What happens:**
|
||||
1. Pre-flight — both servers healthy, Tailscale connected
|
||||
2. SSH to HOST2 — reconfigure each auth container's WebUI URL to HOST1's Tailscale IP
|
||||
3. Verify WebUI connectivity — curl each WebUI, confirm reachable
|
||||
4. Write state files — `partnership_HOST1.db=ACTIVE`, push to HOST2
|
||||
5. Reset offline counter
|
||||
6. Notify both servers — partnership active
|
||||
|
||||
**After onboard:**
|
||||
```
|
||||
Mirror opens Docker UI
|
||||
→ clicks NginxProxyManager
|
||||
→ lands on owner's NPM WebUI
|
||||
→ makes changes there
|
||||
→ 15min later synced to his server
|
||||
→ he never touched his own NPM
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Offboard — Clean Separation
|
||||
|
||||
**Either server can initiate offboard.** The process differs slightly depending on who runs it.
|
||||
|
||||
```bash
|
||||
# Always dry-run first
|
||||
partnership_manage.sh --offboard --dry-run
|
||||
|
||||
# Live offboard
|
||||
partnership_manage.sh --offboard
|
||||
```
|
||||
|
||||
#### HOST1 (owner) initiates offboard:
|
||||
|
||||
1. 10 second countdown
|
||||
2. Stop any running rsync (`rsync_stop.sh --rsync-only`)
|
||||
3. Final sync — Critical-Data and Emby pushed one last time
|
||||
4. SSH to HOST2 — reconfigure auth WebUIs back to localhost
|
||||
5. Disable `CRITICAL_RSYNC_ENABLED=false` in Master.conf
|
||||
6. Write `partnership_HOST1.db=INACTIVE`, push to HOST2
|
||||
7. Wait `PARTNERSHIP_GRACE_HOURS` (6hr) — mirror has access to collect anything
|
||||
8. Remove HOST2 from Tailscale tailnet
|
||||
9. Notify HOST2
|
||||
|
||||
#### HOST2 (mirror) initiates offboard:
|
||||
|
||||
1. Reconfigure own auth WebUIs → localhost
|
||||
2. Write `partnership_HOST2.db=INACTIVE`
|
||||
3. SSH to HOST1 — push state file (or write locally if HOST1 unreachable)
|
||||
4. Notify HOST1 — "mirror has requested offboard"
|
||||
|
||||
**HOST1 finalises on next `--check` cycle:**
|
||||
|
||||
5. Reads HOST2's state file — sees INACTIVE
|
||||
6. Runs final sync — HOST2 leaves with current state
|
||||
7. Disables critical sync
|
||||
8. Writes `partnership_HOST1.db=INACTIVE`
|
||||
9. Waits grace period → removes HOST2 from Tailscale
|
||||
|
||||
#### What HOST2 leaves with:
|
||||
|
||||
```
|
||||
Current auth config ✓ — final sync was clean
|
||||
His own DDNS + updater ✓ — always was his, unaffected
|
||||
His own Git mirror ✓ — full ecosystem, always mirrored
|
||||
His anime source of truth ✓ — always was his
|
||||
Auth WebUIs → localhost ✓ — managing his own auth now
|
||||
Full independence ✓ — just stop the sync
|
||||
Certs valid ~60-90 days ✓ — cert_monitor.sh warns at 30 days
|
||||
```
|
||||
|
||||
**Tailscale access and backup window both expire at the same time** (`PARTNERSHIP_GRACE_HOURS=6`). Keeping backups accessible beyond Tailscale removal is meaningless — they expire together by design.
|
||||
|
||||
---
|
||||
|
||||
### Auto-Offboard — 30 Day Offline Threshold
|
||||
|
||||
If either server is unreachable for `PARTNERSHIP_OFFLINE_THRESHOLD` days (default: 30), the other server auto-offboards. Works both directions:
|
||||
|
||||
```
|
||||
HOST1 sees HOST2 offline 30 days:
|
||||
AM_OWNER → full offboard from owner side
|
||||
Tailscale removal ✅ (API call works regardless)
|
||||
|
||||
HOST2 sees HOST1 offline 30 days:
|
||||
AM_MIRROR → mirror offboard
|
||||
Reconfigures own WebUIs → localhost ✅
|
||||
Writes HOST2.db=INACTIVE ✅
|
||||
Fully independent immediately ✅
|
||||
When HOST1 comes back → sees INACTIVE → cleans its side ✅
|
||||
```
|
||||
|
||||
This handles the "partner disappeared" scenario without manual intervention. 30 days is long enough to cover extended outages, short enough that you're not waiting forever for resolution.
|
||||
|
||||
---
|
||||
|
||||
### Transfer — Flipping Ownership
|
||||
|
||||
Run `--transfer` from HOST1 when ownership needs to change:
|
||||
|
||||
```bash
|
||||
# Always dry-run first — this is significant
|
||||
partnership_manage.sh --transfer --dry-run
|
||||
|
||||
# Live transfer — requires explicit confirmation string
|
||||
partnership_manage.sh --transfer --confirm=i-understand-this-transfers-ownership
|
||||
```
|
||||
|
||||
**The confirmation string is long and specific by design.** You cannot type it accidentally.
|
||||
|
||||
**What happens:**
|
||||
1. Display current and future ownership clearly
|
||||
2. Confirmation string check
|
||||
3. Health strike system — both servers must pass `PARTNERSHIP_TRANSFER_STRIKES` consecutive checks (max `PARTNERSHIP_TRANSFER_MAX_ATTEMPTS` attempts before giving up)
|
||||
4. Final sync in current direction — new mirror leaves with current state
|
||||
5. Reconfigure HOST2 WebUIs → new owner's Tailscale IP
|
||||
6. Reconfigure HOST1 WebUIs → localhost (now manages directly)
|
||||
7. Flip `PARTNERSHIP_OWNER_HOST` in Master.conf on both servers
|
||||
8. Write updated state files
|
||||
9. Notify both servers
|
||||
|
||||
**After transfer:**
|
||||
```
|
||||
Before: HOST1 = owner, HOST2 = mirror
|
||||
After: HOST2 = owner, HOST1 = mirror
|
||||
|
||||
HOST2 now:
|
||||
Makes all auth changes ← source of truth
|
||||
Pushes Critical-Data sync ← every 15min
|
||||
Manages NPM, certs, LLDAP ← directly
|
||||
|
||||
HOST1 now:
|
||||
Receives sync ← warm mirror
|
||||
WebUIs point to HOST2 ← transparent
|
||||
Never makes changes directly
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Check — Automated State Monitor
|
||||
|
||||
Called automatically by `critical_sync_maintenance.sh` every 15 minutes. Never run manually.
|
||||
|
||||
```bash
|
||||
# Called internally — do not run manually
|
||||
partnership_manage.sh --check --remote-seen # rsync succeeded, HOST2 reachable
|
||||
partnership_manage.sh --check --remote-unseen # rsync failed, HOST2 unreachable
|
||||
```
|
||||
|
||||
**What it does each cycle:**
|
||||
- `--remote-seen`: resets offline counter, updates `last_seen_remote` timestamp
|
||||
- `--remote-unseen`: increments offline counter, checks auto-offboard threshold
|
||||
- SSHes to remote, reads remote state file
|
||||
- Both ACTIVE → silent, healthy ✅
|
||||
- Remote INACTIVE → owner finalises offboard, mirror cleans up
|
||||
- Threshold exceeded → auto-offboard
|
||||
|
||||
---
|
||||
|
||||
## Role-Based Access
|
||||
|
||||
```
|
||||
HOST1 (owner): --onboard ✅ --offboard ✅ --transfer ✅ --status ✅
|
||||
HOST2 (mirror): --onboard ❌ --offboard ✅ --transfer ❌ --status ✅
|
||||
```
|
||||
|
||||
HOST2 is blocked from onboard and transfer by design — ownership is granted not taken. Either server can initiate offboard — clean exit is always available to both parties.
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
All configuration in `Master.conf` under `── PARTNERSHIP ──`, placed immediately after HOST CONFIGURATION.
|
||||
|
||||
```bash
|
||||
PARTNERSHIP_ENABLED=false
|
||||
PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer
|
||||
|
||||
# Auth containers reconfigured on onboard/offboard
|
||||
# Format: "ContainerName|WebUIPort"
|
||||
PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap-Gmer4Lfe|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# Paths on owner that mirror should collect during grace window
|
||||
PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Failover/Jayred365-Emby"
|
||||
)
|
||||
|
||||
# Timing
|
||||
PARTNERSHIP_GRACE_HOURS=6 # hours after offboard before Tailscale removal
|
||||
# backup access also expires at this time
|
||||
PARTNERSHIP_OFFLINE_THRESHOLD=30 # days unreachable before auto-offboard (both directions)
|
||||
PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale on offboard
|
||||
|
||||
# Tailscale API — required for PARTNERSHIP_REMOVE_TAILSCALE=true
|
||||
TAILSCALE_API_KEY="" # tskey-api-...
|
||||
TAILSCALE_TAILNET="" # your tailnet name
|
||||
|
||||
# Transfer safety
|
||||
PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership"
|
||||
PARTNERSHIP_TRANSFER_STRIKES=3
|
||||
PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20
|
||||
|
||||
# Onboard
|
||||
PARTNERSHIP_ONBOARD_VERIFY=true
|
||||
PARTNERSHIP_ONBOARD_NOTIFY=true
|
||||
PARTNERSHIP_SYNC_INTERVAL=15 # minutes — informational, actual schedule in cron
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## critical_sync_maintenance.sh
|
||||
|
||||
The partnership check runs inside `critical_sync_maintenance.sh` — the 15-minute orchestrator that also handles Critical-Data and Emby failover syncs.
|
||||
|
||||
```
|
||||
Orchestrators/critical_sync_maintenance.sh
|
||||
Schedule: */15 * * * *
|
||||
|
||||
Execution order:
|
||||
1. Critical-Data rsync ← auth stack sync (15min)
|
||||
2. emby-failover rsync ← dirty Emby sync (15min)
|
||||
3. partnership --check ← state check, informed by rsync outcome
|
||||
```
|
||||
|
||||
The rsync outcome directly informs the partnership check — if rsync succeeded, HOST2 was reachable and the offline counter resets. If rsync failed, the counter increments.
|
||||
|
||||
Controlled by `CRITICAL_RSYNC_ENABLED` in Master.conf — set to `false` automatically on offboard.
|
||||
|
||||
---
|
||||
|
||||
## Initial Setup Requirements
|
||||
|
||||
Before `--onboard` can run:
|
||||
|
||||
**1. Tailscale connected on both servers**
|
||||
|
||||
HOST1 and HOST2 hostnames must match their exact Tailscale device names — already enforced by the ecosystem since `HOST1` and `HOST2` in Master.conf are used for Tailscale IP resolution everywhere.
|
||||
|
||||
**2. SSH keys configured — passwordless both directions**
|
||||
|
||||
```bash
|
||||
ssh -i /root/.ssh/your_key root@[HOST2-tailscale-ip] "hostname"
|
||||
```
|
||||
|
||||
**3. Auth containers exist on HOST2**
|
||||
|
||||
Containers must exist before onboard. They can be stopped — onboard only reconfigures WebUI URLs. Critical-Data sync brings them live config.
|
||||
|
||||
**4. Critical-Data rsync profile configured**
|
||||
|
||||
The `critical-data` rsync profile must be set up in Master.conf. This is what keeps the mirror current after onboard. Configured in `CRITICAL_SYNC_SHARES`.
|
||||
|
||||
**5. Tailscale API key configured** (if `PARTNERSHIP_REMOVE_TAILSCALE=true`)
|
||||
|
||||
```
|
||||
https://login.tailscale.com/admin/settings/keys
|
||||
Scope: Devices write
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Independence — Always One Stop Away
|
||||
|
||||
The partnership is designed with a clean exit built in from day one.
|
||||
|
||||
**To stop the relationship — from either server:**
|
||||
```bash
|
||||
partnership_manage.sh --offboard
|
||||
```
|
||||
|
||||
That's it. The mirror's containers keep running. His domain keeps working. His DDNS keeps pointing to him. His git repo has the full ecosystem. His certs run for another 60-90 days.
|
||||
|
||||
**What the owner keeps after separation:**
|
||||
```
|
||||
His hardware ← always was his
|
||||
His media shares ← his source of truth shares stay his
|
||||
His services ← unaffected
|
||||
His domain ← unaffected
|
||||
His Emby users ← continue as before
|
||||
```
|
||||
|
||||
**What the owner loses:**
|
||||
```
|
||||
Offsite backup ← was the mirror's hardware
|
||||
Geographic redundancy ← can't script that, it was the person
|
||||
```
|
||||
|
||||
The arrangement was always mutual. Both parties contributed something the scripts couldn't provide — hardware in a different location on a different power utility. That's irreplaceable. Everything else in the ecosystem can be reconfigured.
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**WebUI still pointing to old IP after onboard:**
|
||||
- Check template was found: `partnership_manage.sh --status`
|
||||
- Some containers may need a restart to pick up the new WebUI URL
|
||||
- Verify with `--status` on both servers
|
||||
|
||||
**Transfer health checks failing:**
|
||||
- Both arrays must be fully started
|
||||
- Both Docker daemons must be responding
|
||||
- Tailscale must be connected on both servers
|
||||
- Max attempts: `PARTNERSHIP_TRANSFER_MAX_ATTEMPTS` (default 20) before giving up
|
||||
|
||||
**Mirror's domain broken after offboard:**
|
||||
- Auth WebUIs reconfigured to localhost — mirror manages auth directly now
|
||||
- Verify auth containers are running: `docker ps`
|
||||
- Check cert expiry: run `cert_monitor.sh`
|
||||
|
||||
**State mismatch between servers:**
|
||||
- Run `--status` on both servers to see both state files
|
||||
- If one shows ACTIVE and other INACTIVE — run `--offboard` to resolve
|
||||
- If HOST2 unreachable — HOST2 will self-resolve on next `--check` when reachable
|
||||
|
||||
**Auto-offboard triggered unexpectedly:**
|
||||
- Check `/boot/config/partnership_offline_days.db` for counter value
|
||||
- Extended outage or Tailscale connectivity issue may have incremented counter
|
||||
- Re-onboard if partnership should continue: `--onboard`
|
||||
|
||||
---
|
||||
|
||||
## Design Notes
|
||||
|
||||
**Why HOST1/HOST2 instead of PARTNERSHIP_OWNER/MIRROR?**
|
||||
|
||||
HOST1 and HOST2 are already defined in Master.conf with SSH keys, Tailscale names, and all connection details. Duplicating them as PARTNERSHIP_OWNER and PARTNERSHIP_MIRROR would require maintaining the same values twice. A single `PARTNERSHIP_OWNER_HOST` var flips ownership direction — everything else derives from the existing HOST1/HOST2 configuration.
|
||||
|
||||
**Why does transfer require a confirmation string?**
|
||||
|
||||
Transferring ownership touches Master.conf on both servers, reconfigures WebUIs, and flips sync direction. The confirmation string is the same philosophy as `--i-know-what-im-doing` in the arr cleanup scripts — make accidental execution impossible, not just unlikely.
|
||||
|
||||
**Why do Tailscale removal and backup access expire at the same time?**
|
||||
|
||||
Keeping backups accessible beyond Tailscale removal is meaningless — if the mirror can't reach the owner's server via Tailscale, he can't access the backups anyway. `PARTNERSHIP_GRACE_HOURS` controls both simultaneously. One var, consistent behavior, no misleading "30 days available" when access is gone in 6 hours.
|
||||
|
||||
**Why does either server auto-offboard after 30 days offline?**
|
||||
|
||||
30 consecutive days of missed sync cycles means the relationship has effectively ended regardless of intent. The auto-offboard makes the state official and cleans up both sides without requiring manual intervention from a server that may genuinely be gone. Each server acts independently — they don't need to coordinate to offboard.
|
||||
|
||||
**Why does WebUI reconfiguration use the unRAID template system?**
|
||||
|
||||
unRAID stores container WebUI URLs in `/boot/config/plugins/dockerMan/templates-user/`. Editing templates is the correct way to change WebUI URLs — it persists across container restarts and array reboots. Direct Docker label manipulation would be lost on the next container recreation.
|
||||
|
||||
**Why is Tailscale WebUI left on the mirror's local server?**
|
||||
|
||||
Tailscale WebUI shows that server's network state from its own perspective. When diagnosing connectivity issues between the servers, you need to see the mirror's network view — not the owner's. It's the one WebUI that genuinely needs to stay local.
|
||||
@@ -0,0 +1,943 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------------------- Partnership Manager --------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Manages the relationship lifecycle between two unRAID servers.
|
||||
# HOST1 is always the owner (source of truth). HOST2 is always the mirror.
|
||||
# PARTNERSHIP_OWNER_HOST flips to "HOST2" after --transfer.
|
||||
#
|
||||
# Modes:
|
||||
# --onboard ← owner only — set up mirror relationship
|
||||
# Reconfigures HOST2 auth WebUIs → HOST1 Tailscale IP
|
||||
# HOST2 clicks NPM → gets HOST1's NPM automatically
|
||||
#
|
||||
# --offboard ← either server — clean separation
|
||||
# Either party can initiate
|
||||
# HOST2 offboard: writes state, reconfigures own WebUIs → localhost
|
||||
# HOST1 sees it on next --check → finalises, runs final sync
|
||||
# HOST1 offboard: final sync, reconfigures HOST2 WebUIs, removes Tailscale
|
||||
# Both leave with current state, clean exit ✅
|
||||
#
|
||||
# --transfer ← owner only — flip ownership
|
||||
# Requires confirmation string + health strike system
|
||||
# Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in Master.conf
|
||||
#
|
||||
# --check ← called by critical_sync_maintenance.sh every 15min
|
||||
# Reads both state files via SSH
|
||||
# Detects offboard requests → finalises from owner side
|
||||
# Updates last_seen_remote timestamp
|
||||
# Increments offline counter → auto-offboards after threshold
|
||||
# Silent when healthy
|
||||
#
|
||||
# --status ← either server — show current state
|
||||
#
|
||||
# State files on /boot/config (survives reboots, available before array):
|
||||
# /boot/config/partnership_HOST1.db ← HOST1 writes only
|
||||
# /boot/config/partnership_HOST2.db ← HOST2 writes only
|
||||
# Propagated via SSH — no rsync needed
|
||||
#
|
||||
# All configuration in Master.conf under PARTNERSHIP section.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
# Parse flags before parse_args
|
||||
MODE=""
|
||||
TRANSFER_CONFIRM_INPUT=""
|
||||
REMOTE_SEEN=false
|
||||
REMOTE_UNSEEN=false
|
||||
FILTERED_ARGS=()
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--onboard) MODE="onboard" ;;
|
||||
--offboard) MODE="offboard" ;;
|
||||
--transfer) MODE="transfer" ;;
|
||||
--check) MODE="check" ;;
|
||||
--status) MODE="status" ;;
|
||||
--confirm=*) TRANSFER_CONFIRM_INPUT="${arg#--confirm=}" ;;
|
||||
--remote-seen) REMOTE_SEEN=true ;;
|
||||
--remote-unseen) REMOTE_UNSEEN=true ;;
|
||||
*) FILTERED_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
detect_hosts
|
||||
|
||||
# Derive owner and mirror from PARTNERSHIP_OWNER_HOST
|
||||
if [[ "${PARTNERSHIP_OWNER_HOST:-HOST1}" == "HOST1" ]]; then
|
||||
OWNER="$HOST1"
|
||||
MIRROR="$HOST2"
|
||||
OWNER_SSH_KEY="$HOST1_SSH_KEY"
|
||||
MIRROR_SSH_KEY="$HOST2_SSH_KEY"
|
||||
else
|
||||
OWNER="$HOST2"
|
||||
MIRROR="$HOST1"
|
||||
OWNER_SSH_KEY="$HOST2_SSH_KEY"
|
||||
MIRROR_SSH_KEY="$HOST1_SSH_KEY"
|
||||
fi
|
||||
|
||||
AM_OWNER=false
|
||||
AM_MIRROR=false
|
||||
[[ "$LOCAL_SERVER_NAME" == "$OWNER" ]] && AM_OWNER=true
|
||||
[[ "$LOCAL_SERVER_NAME" == "$MIRROR" ]] && AM_MIRROR=true
|
||||
|
||||
# State files
|
||||
LOCAL_STATE_FILE="/boot/config/partnership_${LOCAL_SERVER_NAME}.db"
|
||||
REMOTE_STATE_FILE="/boot/config/partnership_${REMOTE_SERVER_NAME}.db"
|
||||
OWNER_STATE_FILE="/boot/config/partnership_${OWNER}.db"
|
||||
MIRROR_STATE_FILE="/boot/config/partnership_${MIRROR}.db"
|
||||
|
||||
# Offline counter file
|
||||
OFFLINE_COUNTER="/boot/config/partnership_offline_days.db"
|
||||
|
||||
if [[ -z "$MODE" ]]; then
|
||||
error "No mode specified"
|
||||
echo "Usage:"
|
||||
echo " partnership_manage.sh --onboard [--dry-run]"
|
||||
echo " partnership_manage.sh --offboard [--dry-run]"
|
||||
echo " partnership_manage.sh --transfer --confirm=... [--dry-run]"
|
||||
echo " partnership_manage.sh --check --remote-seen|--remote-unseen"
|
||||
echo " partnership_manage.sh --status"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Role-based access control
|
||||
if [[ "$AM_MIRROR" == true ]]; then
|
||||
case "$MODE" in
|
||||
onboard|transfer)
|
||||
error "Only the owner ($OWNER) can run --$MODE"
|
||||
error "Run from $OWNER or use --offboard to separate cleanly"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Acquire lock for all modes except check (check is called frequently, lock would pile up)
|
||||
[[ "$MODE" != "check" ]] && acquire_lock "strict"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# HELPERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
read_state_file() {
|
||||
local file="$1"
|
||||
local key="$2"
|
||||
grep "^${key}=" "$file" 2>/dev/null | cut -d= -f2
|
||||
}
|
||||
|
||||
write_state_file() {
|
||||
local file="$1"
|
||||
shift
|
||||
# flock to prevent concurrent writes
|
||||
(
|
||||
flock -x 200
|
||||
cat > "$file" << EOF
|
||||
state=${1:-UNKNOWN}
|
||||
owner=${OWNER}
|
||||
mirror=${MIRROR}
|
||||
onboarded=${2:-}
|
||||
offboarded=${3:-}
|
||||
triggered_by=${4:-}
|
||||
reason=${5:-}
|
||||
last_seen_remote=${6:-}
|
||||
updated=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
EOF
|
||||
) 200>"${file}.lock"
|
||||
}
|
||||
|
||||
push_state_to_remote() {
|
||||
local local_file="$1"
|
||||
local remote_ip="$2"
|
||||
local ssh_key="$3"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would push state file to remote"
|
||||
return 0
|
||||
fi
|
||||
|
||||
scp -i "$ssh_key" -o ConnectTimeout=10 \
|
||||
"$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \
|
||||
success "State file pushed to remote ✅" || \
|
||||
warn "Could not push state file to remote — will propagate on next sync"
|
||||
}
|
||||
|
||||
read_remote_state() {
|
||||
local remote_ip="$1"
|
||||
local ssh_key="$2"
|
||||
local remote_file="$3"
|
||||
|
||||
ssh -i "$ssh_key" -o ConnectTimeout=10 root@"$remote_ip" \
|
||||
"cat '$remote_file' 2>/dev/null" 2>/dev/null
|
||||
}
|
||||
|
||||
reconfigure_webui() {
|
||||
local container="$1"
|
||||
local port="$2"
|
||||
local target_ip="$3"
|
||||
local ssh_key="$4"
|
||||
local remote_ip="$5"
|
||||
local label="${6:-remote}"
|
||||
|
||||
info "Reconfiguring $container WebUI → ${target_ip}:${port} on $label..."
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would reconfigure $container WebUI to http://${target_ip}:${port}/"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local template
|
||||
template=$(ssh -i "$ssh_key" -o ConnectTimeout=10 root@"$remote_ip" \
|
||||
"grep -rl '<WebUI>' /boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \
|
||||
xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null)
|
||||
|
||||
if [[ -z "$template" ]]; then
|
||||
warn "$container template not found on $label — WebUI needs manual reconfiguration"
|
||||
return 1
|
||||
fi
|
||||
|
||||
ssh -i "$ssh_key" -o ConnectTimeout=10 root@"$remote_ip" \
|
||||
"sed -i 's|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g' '$template'" \
|
||||
2>/dev/null && \
|
||||
success "$container → http://${target_ip}:${port}/ ✅" || {
|
||||
error "Failed to reconfigure $container WebUI"
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
reconfigure_local_webuis() {
|
||||
local target_ip="$1"
|
||||
info "Reconfiguring local auth WebUIs → ${target_ip}..."
|
||||
|
||||
local failures=0
|
||||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
local container="${entry%%|*}"
|
||||
local port="${entry##*|}"
|
||||
|
||||
local template
|
||||
template=$(grep -rl '<WebUI>' /boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \
|
||||
xargs grep -l "\"$container\"" 2>/dev/null | head -1)
|
||||
|
||||
if [[ -z "$template" ]]; then
|
||||
warn "$container template not found locally"
|
||||
((failures++))
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would reconfigure $container → http://${target_ip}:${port}/"
|
||||
continue
|
||||
fi
|
||||
|
||||
sed -i "s|<WebUI>.*</WebUI>|<WebUI>http://${target_ip}:${port}/</WebUI>|g" \
|
||||
"$template" 2>/dev/null && \
|
||||
success "$container → http://${target_ip}:${port}/ ✅" || \
|
||||
{ error "Failed to reconfigure $container"; ((failures++)); }
|
||||
done
|
||||
return $failures
|
||||
}
|
||||
|
||||
get_tailscale_device_id() {
|
||||
local hostname="$1"
|
||||
curl -sf --max-time 10 \
|
||||
-H "Authorization: Bearer $TAILSCALE_API_KEY" \
|
||||
"https://api.tailscale.com/api/v2/tailnet/${TAILSCALE_TAILNET}/devices" \
|
||||
2>/dev/null | \
|
||||
grep -o "\"id\":\"[^\"]*\"[^}]*\"hostname\":\"${hostname}\"" | \
|
||||
grep -o '"id":"[^"]*"' | \
|
||||
grep -o '[^"]*"$' | tr -d '"'
|
||||
}
|
||||
|
||||
remove_tailscale_device() {
|
||||
local hostname="$1"
|
||||
|
||||
if [[ -z "${TAILSCALE_API_KEY:-}" ]] || [[ -z "${TAILSCALE_TAILNET:-}" ]]; then
|
||||
warn "TAILSCALE_API_KEY or TAILSCALE_TAILNET not configured — skipping Tailscale removal"
|
||||
return 1
|
||||
fi
|
||||
|
||||
info "Looking up Tailscale device ID for $hostname..."
|
||||
local device_id
|
||||
device_id=$(get_tailscale_device_id "$hostname")
|
||||
|
||||
if [[ -z "$device_id" ]]; then
|
||||
warn "Device $hostname not found in Tailscale — may already be removed"
|
||||
return 0
|
||||
fi
|
||||
|
||||
info "Removing $hostname (device $device_id) from Tailscale..."
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove $hostname from Tailscale tailnet"
|
||||
return 0
|
||||
fi
|
||||
|
||||
curl -sf --max-time 10 -X DELETE \
|
||||
-H "Authorization: Bearer $TAILSCALE_API_KEY" \
|
||||
"https://api.tailscale.com/api/v2/devices/${device_id}" 2>/dev/null && \
|
||||
success "$hostname removed from Tailscale ✅" || \
|
||||
error "Failed to remove $hostname from Tailscale — remove manually"
|
||||
}
|
||||
|
||||
check_both_healthy() {
|
||||
mountpoint -q /mnt/user 2>/dev/null || { error "Local array not healthy"; return 1; }
|
||||
|
||||
local mirror_ip
|
||||
mirror_ip=$(tailscale ip -4 "$MIRROR" 2>/dev/null)
|
||||
[[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; }
|
||||
|
||||
ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout=10 root@"$mirror_ip" \
|
||||
"mountpoint -q /mnt/user && timeout 10 docker ps" >/dev/null 2>&1 || {
|
||||
error "Mirror not healthy"
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
do_final_sync() {
|
||||
info "Running final Critical-Data sync..."
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
|
||||
"/mnt/user/appdata-Failover/Critical-Data" --log
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" \
|
||||
"/mnt/user/Media_Server/Emby" --profile=emby-failover --log
|
||||
else
|
||||
warn "DRY RUN — would run final Critical-Data and emby-failover sync"
|
||||
fi
|
||||
success "Final sync complete — mirror has current state ✅"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ STATUS ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$MODE" == "status" ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY PARTNERSHIP STATUS ━━━━━"
|
||||
|
||||
OWNER_IP=$(tailscale ip -4 "$OWNER" 2>/dev/null || echo "unreachable")
|
||||
MIRROR_IP=$(tailscale ip -4 "$MIRROR" 2>/dev/null || echo "unreachable")
|
||||
|
||||
echo " Owner: $OWNER ($OWNER_IP)"
|
||||
echo " Mirror: $MIRROR ($MIRROR_IP)"
|
||||
echo " Enabled: ${PARTNERSHIP_ENABLED:-false}"
|
||||
echo " Running as: $LOCAL_SERVER_NAME ($( [[ "$AM_OWNER" == true ]] && echo "owner" || echo "mirror"))"
|
||||
echo ""
|
||||
|
||||
# Local state
|
||||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||||
LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||||
LOCAL_ONBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "onboarded")
|
||||
LOCAL_OFFBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "offboarded")
|
||||
LOCAL_LAST_SEEN=$(read_state_file "$LOCAL_STATE_FILE" "last_seen_remote")
|
||||
echo " Local state: $LOCAL_STATE"
|
||||
[[ -n "$LOCAL_ONBOARDED" ]] && echo " Onboarded: $LOCAL_ONBOARDED"
|
||||
[[ -n "$LOCAL_OFFBOARDED" ]] && echo " Offboarded: $LOCAL_OFFBOARDED"
|
||||
[[ -n "$LOCAL_LAST_SEEN" ]] && echo " Remote last seen: $LOCAL_LAST_SEEN"
|
||||
else
|
||||
echo " Local state: no state file found"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Remote state via SSH
|
||||
REMOTE_IP=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
|
||||
if [[ -n "$REMOTE_IP" ]]; then
|
||||
REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE")
|
||||
if [[ -n "$REMOTE_CONTENT" ]]; then
|
||||
REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2)
|
||||
REMOTE_ONBOARDED=$(echo "$REMOTE_CONTENT" | grep "^onboarded=" | cut -d= -f2)
|
||||
REMOTE_OFFBOARDED=$(echo "$REMOTE_CONTENT" | grep "^offboarded=" | cut -d= -f2)
|
||||
echo " Remote state: $REMOTE_STATE"
|
||||
[[ -n "$REMOTE_ONBOARDED" ]] && echo " Onboarded: $REMOTE_ONBOARDED"
|
||||
[[ -n "$REMOTE_OFFBOARDED" ]] && echo " Offboarded: $REMOTE_OFFBOARDED"
|
||||
|
||||
# Check agreement
|
||||
if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]]; then
|
||||
echo ""
|
||||
echo " ✅ Both servers agree: $LOCAL_STATE"
|
||||
else
|
||||
echo ""
|
||||
echo " ⚠️ State mismatch — local: $LOCAL_STATE remote: $REMOTE_STATE"
|
||||
echo " Run --onboard or --offboard to resolve"
|
||||
fi
|
||||
else
|
||||
echo " Remote state: state file not found on $REMOTE_SERVER_NAME"
|
||||
fi
|
||||
else
|
||||
echo " Remote state: $REMOTE_SERVER_NAME unreachable"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo " Auth WebUIs configured:"
|
||||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
echo " ${entry%%|*} → port ${entry##*|}"
|
||||
done
|
||||
|
||||
# Offline counter
|
||||
if [[ -f "$OFFLINE_COUNTER" ]]; then
|
||||
OFFLINE_DAYS=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0)
|
||||
[[ "$OFFLINE_DAYS" -gt 0 ]] && \
|
||||
echo "" && \
|
||||
echo " ⚠️ Remote offline counter: ${OFFLINE_DAYS}/${PARTNERSHIP_OFFLINE_THRESHOLD} days"
|
||||
fi
|
||||
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ CHECK (called every 15min by critical_sync_maintenance.sh) ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$MODE" == "check" ]]; then
|
||||
|
||||
# Update last_seen_remote and offline counter based on rsync outcome
|
||||
if [[ "$REMOTE_SEEN" == true ]]; then
|
||||
OFFLINE_COUNT=0
|
||||
echo "0" > "$OFFLINE_COUNTER"
|
||||
# Update last_seen_remote in local state file
|
||||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||||
sed -i "s|^last_seen_remote=.*|last_seen_remote=$(date '+%Y-%m-%d %H:%M:%S')|" \
|
||||
"$LOCAL_STATE_FILE" 2>/dev/null
|
||||
fi
|
||||
elif [[ "$REMOTE_UNSEEN" == true ]]; then
|
||||
OFFLINE_COUNT=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0)
|
||||
OFFLINE_COUNT=$(( OFFLINE_COUNT + 1 ))
|
||||
echo "$OFFLINE_COUNT" > "$OFFLINE_COUNTER"
|
||||
|
||||
# Check auto-offboard threshold
|
||||
# Convert 15min intervals to days: threshold_days * 96 intervals/day
|
||||
THRESHOLD_INTERVALS=$(( ${PARTNERSHIP_OFFLINE_THRESHOLD:-30} * 96 ))
|
||||
if [[ "$OFFLINE_COUNT" -ge "$THRESHOLD_INTERVALS" ]]; then
|
||||
warn "Remote offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days — triggering auto-offboard"
|
||||
notify "Partnership auto-offboard on $(hostname) — $REMOTE_SERVER_NAME offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days. Both servers will offboard independently." \
|
||||
"Partnership" "warning"
|
||||
bash "$0" --offboard --reason="auto-offboard-timeout"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "Partnership check — remote unseen (count: $OFFLINE_COUNT/$THRESHOLD_INTERVALS)"
|
||||
fi
|
||||
|
||||
# Read remote state file
|
||||
REMOTE_IP=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
|
||||
if [[ -z "$REMOTE_IP" ]]; then
|
||||
log "Partnership check — remote unreachable, skipping state check"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE")
|
||||
[[ -z "$REMOTE_CONTENT" ]] && log "Partnership check — remote state file not found" && exit 0
|
||||
|
||||
REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2)
|
||||
LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state" 2>/dev/null || echo "UNKNOWN")
|
||||
|
||||
# Both agree — healthy
|
||||
if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then
|
||||
log "Partnership check — ACTIVE, both servers agree ✅"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Remote requested offboard
|
||||
if [[ "$REMOTE_STATE" == "INACTIVE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then
|
||||
info "Partnership check — $REMOTE_SERVER_NAME requested offboard"
|
||||
|
||||
if [[ "$AM_OWNER" == true ]]; then
|
||||
info "Owner finalising offboard request from mirror..."
|
||||
# Final sync — mirror leaves with current state
|
||||
do_final_sync
|
||||
# Update local state
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$REMOTE_SERVER_NAME" "mirror-requested"
|
||||
# Disable critical rsync
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sed -i 's/CRITICAL_RSYNC_ENABLED=true/CRITICAL_RSYNC_ENABLED=false/' \
|
||||
"$SCRIPT_DIR/../Master.conf" 2>/dev/null
|
||||
fi
|
||||
# Tailscale removal after grace period
|
||||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||||
local grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-3} * 3600 ))
|
||||
info "Waiting ${PARTNERSHIP_GRACE_HOURS:-3}hr grace period for mirror to see notification..."
|
||||
[[ "$DRY_RUN" == false ]] && sleep "$grace_seconds"
|
||||
remove_tailscale_device "$MIRROR"
|
||||
fi
|
||||
notify "Partnership offboard finalised on $(hostname) — $MIRROR requested separation" \
|
||||
"Partnership" "normal"
|
||||
else
|
||||
# Mirror sees owner is INACTIVE — clean up own side
|
||||
info "Owner has offboarded — cleaning up mirror side..."
|
||||
reconfigure_local_webuis "localhost"
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$OWNER" "owner-offboarded"
|
||||
notify "Partnership ended on $(hostname) — $OWNER has offboarded. Auth WebUIs reconfigured to localhost." \
|
||||
"Partnership" "normal"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Both inactive — nothing to do
|
||||
if [[ "$LOCAL_STATE" == "INACTIVE" ]] && [[ "$REMOTE_STATE" == "INACTIVE" ]]; then
|
||||
log "Partnership check — INACTIVE on both servers"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "Partnership check — local: $LOCAL_STATE remote: $REMOTE_STATE"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ ONBOARD ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$MODE" == "onboard" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_FAILOVER Onboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
# Check already onboarded
|
||||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||||
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||||
if [[ "$CURRENT_STATE" == "ACTIVE" ]]; then
|
||||
info "Partnership already ACTIVE — no action needed"
|
||||
info "Use --status for detail or --offboard to separate"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
info "Owner: $OWNER"
|
||||
info "Mirror: $MIRROR"
|
||||
|
||||
# Pre-flight
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
resolve_remote_ip
|
||||
check_connectivity
|
||||
check_remote_array || exit 1
|
||||
check_remote_docker || exit 1
|
||||
|
||||
OWNER_IP=$(tailscale ip -4 "$OWNER" 2>/dev/null)
|
||||
MIRROR_IP=$(tailscale ip -4 "$MIRROR" 2>/dev/null)
|
||||
|
||||
[[ -z "$OWNER_IP" ]] && error "Cannot resolve $OWNER Tailscale IP" && exit 1
|
||||
[[ -z "$MIRROR_IP" ]] && error "Cannot resolve $MIRROR Tailscale IP" && exit 1
|
||||
|
||||
success "Owner IP: $OWNER_IP"
|
||||
success "Mirror IP: $MIRROR_IP"
|
||||
|
||||
# Reconfigure mirror WebUIs → owner IP
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → $OWNER_IP ━━━"
|
||||
|
||||
WEBUI_FAILURES=0
|
||||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
container="${entry%%|*}"
|
||||
port="${entry##*|}"
|
||||
reconfigure_webui "$container" "$port" "$OWNER_IP" \
|
||||
"$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || ((WEBUI_FAILURES++))
|
||||
done
|
||||
|
||||
# Verify WebUI connectivity
|
||||
if [[ "${PARTNERSHIP_ONBOARD_VERIFY:-true}" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_HEALTH Verify WebUI Connectivity ━━━"
|
||||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
container="${entry%%|*}"
|
||||
port="${entry##*|}"
|
||||
if curl -sf --max-time 10 "http://${OWNER_IP}:${port}/" >/dev/null 2>&1; then
|
||||
success "$container reachable at http://${OWNER_IP}:${port}/ ✅"
|
||||
else
|
||||
warn "$container not reachable at http://${OWNER_IP}:${port}/ — may not be running"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Write state files
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Write State ━━━"
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "onboard"
|
||||
success "Local state: ACTIVE ✅"
|
||||
|
||||
# Push to remote
|
||||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||||
|
||||
# Reset offline counter
|
||||
echo "0" > "$OFFLINE_COUNTER"
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━"
|
||||
echo " Owner: $OWNER ($OWNER_IP)"
|
||||
echo " Mirror: $MIRROR ($MIRROR_IP)"
|
||||
echo " WebUI failures: $WEBUI_FAILURES"
|
||||
echo " Sync interval: ${PARTNERSHIP_SYNC_INTERVAL}min"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ "$WEBUI_FAILURES" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
[[ "${PARTNERSHIP_ONBOARD_NOTIFY:-true}" == true ]] && \
|
||||
notify "Partnership onboard complete — $MIRROR is now mirroring $OWNER via Tailscale" \
|
||||
"Partnership" "normal"
|
||||
else
|
||||
echo "$ICON_WARN Status: DONE with $WEBUI_FAILURES WebUI warning(s)"
|
||||
notify "Partnership onboard complete with $WEBUI_FAILURES WebUI warning(s) on $(hostname)" \
|
||||
"Partnership" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ OFFBOARD ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$MODE" == "offboard" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_FAILOVER Offboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
# Check already offboarded
|
||||
if [[ -f "$LOCAL_STATE_FILE" ]]; then
|
||||
CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state")
|
||||
if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then
|
||||
info "Partnership already INACTIVE"
|
||||
info "Use --status to verify both servers agree"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
REASON="${REASON:-manual}"
|
||||
|
||||
# ── MIRROR-initiated offboard ────────────────────────────────────────────────────────────
|
||||
if [[ "$AM_MIRROR" == true ]]; then
|
||||
warn "$MIRROR is initiating offboard"
|
||||
warn "This will reconfigure your local auth WebUIs → localhost"
|
||||
warn "HOST1 will finalise the offboard on its next --check cycle"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
echo ""
|
||||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||||
sleep 10
|
||||
fi
|
||||
|
||||
# Reconfigure own WebUIs → localhost
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Reconfigure Local WebUIs → localhost ━━━"
|
||||
reconfigure_local_webuis "localhost"
|
||||
|
||||
# Write own state file
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||||
success "Local state: INACTIVE ✅"
|
||||
|
||||
# Try to push state to owner immediately
|
||||
OWNER_IP=$(tailscale ip -4 "$OWNER" 2>/dev/null)
|
||||
if [[ -n "$OWNER_IP" ]]; then
|
||||
push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$MIRROR_SSH_KEY"
|
||||
notify "Partnership offboard requested by $MIRROR — owner will finalise on next check cycle" \
|
||||
"Partnership" "normal"
|
||||
else
|
||||
warn "$OWNER unreachable — state written locally, owner will see it when reachable"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━"
|
||||
echo " Your WebUIs: reconfigured → localhost ✅"
|
||||
echo " State: INACTIVE written ✅"
|
||||
echo " Owner: will finalise + run final sync on next --check ✅"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── OWNER-initiated offboard ─────────────────────────────────────────────────────────────
|
||||
warn "Offboarding $MIRROR from partnership"
|
||||
warn "Final sync will run — mirror leaves with current state"
|
||||
warn "$MIRROR auth WebUIs will be reconfigured → localhost"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
echo ""
|
||||
echo "You have 10 seconds to cancel (Ctrl+C)..."
|
||||
sleep 10
|
||||
echo "Proceeding..."
|
||||
fi
|
||||
|
||||
resolve_remote_ip
|
||||
|
||||
# Stop any running rsync first
|
||||
echo ""
|
||||
echo "━━━ $ICON_STOP Stop Running Rsync ━━━"
|
||||
bash "$SCRIPT_DIR/../unRAID_Essentials/rsync_stop.sh" --rsync-only 2>/dev/null || true
|
||||
|
||||
# Final sync
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Final Sync ━━━"
|
||||
do_final_sync
|
||||
|
||||
MIRROR_IP=$(tailscale ip -4 "$MIRROR" 2>/dev/null)
|
||||
MIRROR_REACHABLE=false
|
||||
[[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true
|
||||
|
||||
# Reconfigure mirror WebUIs → localhost
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → localhost ━━━"
|
||||
|
||||
WEBUI_FAILURES=0
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
container="${entry%%|*}"
|
||||
port="${entry##*|}"
|
||||
reconfigure_webui "$container" "$port" "localhost" \
|
||||
"$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || ((WEBUI_FAILURES++))
|
||||
done
|
||||
else
|
||||
warn "$MIRROR unreachable — WebUI reconfiguration skipped"
|
||||
warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on next --check"
|
||||
((WEBUI_FAILURES++))
|
||||
fi
|
||||
|
||||
# Disable critical rsync
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Disable Critical Sync ━━━"
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sed -i 's/CRITICAL_RSYNC_ENABLED=true/CRITICAL_RSYNC_ENABLED=false/' \
|
||||
"$SCRIPT_DIR/../Master.conf" 2>/dev/null
|
||||
success "CRITICAL_RSYNC_ENABLED=false ✅"
|
||||
else
|
||||
warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false"
|
||||
fi
|
||||
|
||||
# Write state files
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Write State ━━━"
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
write_state_file "$LOCAL_STATE_FILE" \
|
||||
"INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON"
|
||||
success "Local state: INACTIVE ✅"
|
||||
|
||||
# Push to mirror
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY"
|
||||
fi
|
||||
|
||||
# Tailscale removal
|
||||
if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET Tailscale Separation ━━━"
|
||||
if [[ "$MIRROR_REACHABLE" == true ]]; then
|
||||
local grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-3} * 3600 ))
|
||||
info "Waiting ${PARTNERSHIP_GRACE_HOURS:-3}hr grace period for mirror to see state..."
|
||||
[[ "$DRY_RUN" == false ]] && sleep "$grace_seconds"
|
||||
fi
|
||||
remove_tailscale_device "$MIRROR"
|
||||
fi
|
||||
|
||||
# Backup notification
|
||||
if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_DISK Backup Handover ━━━"
|
||||
info "Backups available for $MIRROR:"
|
||||
for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do
|
||||
[[ -z "$path" ]] && continue
|
||||
echo " $path"
|
||||
done
|
||||
info "Available for ${PARTNERSHIP_GRACE_HOURS:-3} hours — Tailscale access removed after this window"
|
||||
notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-3}hr. Tailscale access expires then. Certs valid ~60-90 days. Run tailscale up to rejoin." \
|
||||
"Partnership" "warning"
|
||||
fi
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━"
|
||||
echo " Owner: $OWNER"
|
||||
echo " Mirror: $MIRROR"
|
||||
echo " Final sync: complete ✅"
|
||||
echo " WebUI failures: $WEBUI_FAILURES"
|
||||
echo " Critical rsync: disabled ✅"
|
||||
echo " State: INACTIVE ✅"
|
||||
[[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \
|
||||
echo " Tailscale: $MIRROR removed ✅"
|
||||
echo ""
|
||||
echo " $MIRROR leaves with:"
|
||||
echo " ✓ Current auth config (final sync)"
|
||||
echo " ✓ His own DDNS and updater"
|
||||
echo " ✓ His own Git mirror"
|
||||
echo " ✓ Auth WebUIs → localhost"
|
||||
echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-3}hr to collect backups before Tailscale removed"
|
||||
echo " ✓ Certs valid ~60-90 more days"
|
||||
echo " ✓ Full ecosystem — just stop the sync"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo ""
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
else
|
||||
echo ""
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE — clean separation complete"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ TRANSFER ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$MODE" == "transfer" ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_FAILOVER Transfer Ownership — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "⚠️ WARNING — OWNERSHIP TRANSFER"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " Current owner: $OWNER"
|
||||
echo " Current mirror: $MIRROR"
|
||||
echo ""
|
||||
echo " After transfer:"
|
||||
echo " New owner: $MIRROR"
|
||||
echo " New mirror: $OWNER"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Confirmation string
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
if [[ -z "$TRANSFER_CONFIRM_INPUT" ]]; then
|
||||
echo ""
|
||||
echo "To proceed type exactly:"
|
||||
echo " --confirm=${PARTNERSHIP_TRANSFER_CONFIRM}"
|
||||
echo ""
|
||||
error "Transfer cancelled — confirmation required"
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$TRANSFER_CONFIRM_INPUT" != "$PARTNERSHIP_TRANSFER_CONFIRM" ]]; then
|
||||
error "Confirmation string does not match — transfer cancelled"
|
||||
exit 1
|
||||
fi
|
||||
success "Confirmation accepted"
|
||||
else
|
||||
warn "DRY RUN — confirmation check skipped"
|
||||
fi
|
||||
|
||||
# Health strike system
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Health Verification ━━━"
|
||||
info "Both servers must pass ${PARTNERSHIP_TRANSFER_STRIKES} consecutive health checks"
|
||||
|
||||
STRIKES=0
|
||||
ATTEMPTS=0
|
||||
MAX_ATTEMPTS="${PARTNERSHIP_TRANSFER_MAX_ATTEMPTS:-20}"
|
||||
|
||||
while [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]]; do
|
||||
((ATTEMPTS++))
|
||||
if [[ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]]; then
|
||||
error "Health checks failed after $MAX_ATTEMPTS attempts — servers not stable"
|
||||
error "Transfer cancelled — try again when both servers are healthy"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if check_both_healthy; then
|
||||
((STRIKES++))
|
||||
success "Health check passed ($STRIKES/${PARTNERSHIP_TRANSFER_STRIKES})"
|
||||
[[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]] && sleep 10
|
||||
else
|
||||
warn "Health check failed — resetting (attempt $ATTEMPTS/$MAX_ATTEMPTS)"
|
||||
STRIKES=0
|
||||
sleep 30
|
||||
fi
|
||||
done
|
||||
success "Both servers healthy — proceeding"
|
||||
|
||||
NEW_OWNER="$MIRROR"
|
||||
NEW_MIRROR="$OWNER"
|
||||
NEW_OWNER_IP=$(tailscale ip -4 "$NEW_OWNER" 2>/dev/null)
|
||||
NEW_MIRROR_IP=$(tailscale ip -4 "$NEW_MIRROR" 2>/dev/null)
|
||||
|
||||
[[ -z "$NEW_OWNER_IP" ]] && error "Cannot resolve new owner Tailscale IP" && exit 1
|
||||
|
||||
# Final sync in current direction
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Pre-transfer Sync ━━━"
|
||||
do_final_sync
|
||||
|
||||
# Reconfigure new mirror WebUIs → new owner
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Reconfigure WebUIs ━━━"
|
||||
info "New mirror ($NEW_MIRROR) WebUIs → new owner ($NEW_OWNER_IP)"
|
||||
|
||||
NEW_MIRROR_SSH="${HOST1_SSH_KEY}"
|
||||
[[ "$NEW_MIRROR" == "$HOST2" ]] && NEW_MIRROR_SSH="${HOST2_SSH_KEY}"
|
||||
NEW_OWNER_SSH="${HOST1_SSH_KEY}"
|
||||
[[ "$NEW_OWNER" == "$HOST2" ]] && NEW_OWNER_SSH="${HOST2_SSH_KEY}"
|
||||
|
||||
WEBUI_FAILURES=0
|
||||
for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
container="${entry%%|*}"
|
||||
port="${entry##*|}"
|
||||
reconfigure_webui "$container" "$port" "$NEW_OWNER_IP" \
|
||||
"$NEW_MIRROR_SSH" "$NEW_MIRROR_IP" "$NEW_MIRROR" || ((WEBUI_FAILURES++))
|
||||
done
|
||||
|
||||
# New owner WebUIs → localhost (now manages directly)
|
||||
info "New owner ($NEW_OWNER) WebUIs → localhost"
|
||||
reconfigure_local_webuis "localhost"
|
||||
|
||||
# Flip PARTNERSHIP_OWNER_HOST in Master.conf on both servers
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Update Ownership ━━━"
|
||||
NEW_OWNER_HOST="HOST1"
|
||||
[[ "$NEW_OWNER" == "$HOST2" ]] && NEW_OWNER_HOST="HOST2"
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
sed -i "s/PARTNERSHIP_OWNER_HOST=.*/PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_HOST\"/" \
|
||||
"$SCRIPT_DIR/../Master.conf" 2>/dev/null
|
||||
success "Local Master.conf updated: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_HOST ✅"
|
||||
|
||||
# Update remote
|
||||
ssh -i "$NEW_MIRROR_SSH" -o ConnectTimeout=10 root@"$NEW_MIRROR_IP" \
|
||||
"sed -i 's/PARTNERSHIP_OWNER_HOST=.*/PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_HOST\"/' \
|
||||
'$SCRIPT_DIR/../Master.conf'" 2>/dev/null && \
|
||||
success "Remote Master.conf updated ✅" || \
|
||||
error "Failed to update remote Master.conf — update manually"
|
||||
else
|
||||
warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_HOST on both servers"
|
||||
fi
|
||||
|
||||
# Write state files
|
||||
NOW=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "transfer"
|
||||
push_state_to_remote "$LOCAL_STATE_FILE" "$NEW_MIRROR_IP" "$NEW_MIRROR_SSH"
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY TRANSFER SUMMARY ━━━━━"
|
||||
echo " New owner: $NEW_OWNER ($NEW_OWNER_IP)"
|
||||
echo " New mirror: $NEW_MIRROR ($NEW_MIRROR_IP)"
|
||||
echo " WebUI failures: $WEBUI_FAILURES"
|
||||
echo " Sync direction: $NEW_OWNER → $NEW_MIRROR"
|
||||
echo " Master.conf: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_HOST"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE — ownership transferred"
|
||||
notify "Partnership ownership transferred — new owner: $NEW_OWNER new mirror: $NEW_MIRROR" \
|
||||
"Partnership" "normal"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
+125
-22
@@ -98,8 +98,8 @@
|
||||
|
||||
# SSH keys for server-to-server rsync and failover container operations.
|
||||
# Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/your_key"
|
||||
HOST2_SSH_KEY="/root/.ssh/your_key"
|
||||
HOST1_SSH_KEY="/root/.ssh/your_host1_key"
|
||||
HOST2_SSH_KEY="/root/.ssh/your_host2_key"
|
||||
|
||||
# ━━━ Emby ━━━
|
||||
# Defined once here — referenced by transcode_manager.sh, emby_session_report.sh,
|
||||
@@ -113,6 +113,75 @@
|
||||
HOST2_EMBY_URL="http://localhost:8096" # same port — different server, different key
|
||||
HOST2_EMBY_API_KEY="your-emby-api-key"
|
||||
|
||||
# ==============================================================================================
|
||||
# ── PARTNERSHIP ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Manages the relationship lifecycle between two unRAID servers.
|
||||
# HOST1 is always the owner (source of truth) — HOST2 is always the mirror.
|
||||
# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation.
|
||||
# All other vars derive from HOST1/HOST2 — no duplicate IP or key configuration needed.
|
||||
#
|
||||
# State files live on /boot/config — survives reboots, available before array starts:
|
||||
# /boot/config/partnership_HOST1.db ← HOST1 writes only
|
||||
# /boot/config/partnership_HOST2.db ← HOST2 writes only
|
||||
# Each server SSHes to write/read the other's file directly — no rsync needed
|
||||
#
|
||||
# critical_sync_maintenance.sh runs --check every 15min:
|
||||
# Reads both state files via SSH
|
||||
# Detects offboard requests → finalises from owner side
|
||||
# Increments offline counter → auto-offboards after threshold
|
||||
# Silent when healthy ✅
|
||||
|
||||
PARTNERSHIP_ENABLED=false
|
||||
PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer
|
||||
|
||||
# Auth containers whose WebUI URL gets reconfigured on onboard/offboard
|
||||
# Format: "ContainerName|Port"
|
||||
# These are the containers the mirror operator clicks in his Docker UI
|
||||
# On onboard → pointed at owner's Tailscale IP
|
||||
# On offboard → pointed back at localhost
|
||||
PARTNERSHIP_AUTH_WEBUIS=(
|
||||
"NginxProxyManager|81"
|
||||
"Lldap|17170"
|
||||
"Authelia|9091"
|
||||
"Authelia-Secondary|9092"
|
||||
)
|
||||
|
||||
# Paths on HOST1 that HOST2 operator should collect before backup retention expires
|
||||
# Notified on offboard — no auto-deletion, manual collection
|
||||
PARTNERSHIP_MIRROR_BACKUPS=(
|
||||
# "/mnt/user/appdata-Failover/Host2-Emby"
|
||||
# add any paths HOST2 wants to collect after separation
|
||||
)
|
||||
|
||||
# Offboard settings
|
||||
PARTNERSHIP_GRACE_HOURS=3 # hours after offboard before Tailscale removal
|
||||
# backup access also expires at this time
|
||||
# collect anything needed before this window closes
|
||||
PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale on offboard
|
||||
# false = skip removal (manual or testing)
|
||||
|
||||
# Tailscale API — used to remove HOST2 from tailnet on offboard
|
||||
# API key: https://login.tailscale.com/admin/settings/keys
|
||||
# Tailnet: your tailnet name (e.g. "yourname.github" or "youremail.com")
|
||||
TAILSCALE_API_KEY="" # tskey-api-...
|
||||
TAILSCALE_TAILNET="" # your tailnet name
|
||||
|
||||
# Transfer safety
|
||||
PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership"
|
||||
PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required
|
||||
PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 # max attempts before giving up
|
||||
|
||||
# Onboard settings
|
||||
PARTNERSHIP_ONBOARD_VERIFY=true # verify WebUI reachable after reconfiguration
|
||||
PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on completion
|
||||
PARTNERSHIP_SYNC_INTERVAL=15 # minutes — Critical-Data sync frequency
|
||||
|
||||
# Auto-offboard threshold
|
||||
PARTNERSHIP_OFFLINE_THRESHOLD=30 # days either server unreachable before auto-offboard
|
||||
# works both directions — mirror offline 30d → owner offboards
|
||||
# owner offline 30d → mirror offboards itself
|
||||
|
||||
# ==============================================================================================
|
||||
# ── LOGGING ───────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
@@ -142,7 +211,7 @@
|
||||
# Falls back to GITEA_DOMAIN if local and Tailscale both fail.
|
||||
GITEA_CONTAINER="Gitea" # exact Docker container name
|
||||
GITEA_REPO_PATH="youruser/Unraid_Scripts.git" # repo path on Gitea server
|
||||
GITEA_DOMAIN="" # e.g. git.gmer4lfe.com — requires NPM + DNS setup
|
||||
GITEA_DOMAIN="" # e.g. git.yourdomain.com — requires NPM + DNS setup
|
||||
TARGET_DIR="/mnt/user/appdata/unraid_scripts" # where scripts are cloned to
|
||||
GITEA_SSH_KEY="/root/.ssh/your_gitea_key" # SSH key for authenticating to Gitea
|
||||
SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 222/221)
|
||||
@@ -238,6 +307,15 @@ WEEKLY_SYNC_JOBS=(
|
||||
"/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack
|
||||
)
|
||||
|
||||
# ━━━ Critical Sync Maintenance ━━━
|
||||
# Shares synced every 15 minutes by critical_sync_maintenance.sh
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then emby-failover (dirty sync)
|
||||
CRITICAL_SYNC_SHARES=(
|
||||
"/mnt/user/appdata-Failover/Critical-Data" # auth stack — full sync
|
||||
"/mnt/user/Media_Server/Emby|emby-failover" # Emby dirty sync — stays running
|
||||
)
|
||||
|
||||
# Container update toggles for the weekly sync window.
|
||||
# Containers are already stopped for the sync — updates pull at no extra downtime.
|
||||
# Both false → sync only, no updates.
|
||||
@@ -245,6 +323,14 @@ WEEKLY_SYNC_JOBS=(
|
||||
CRITICAL_SYNC_UPDATES=true # pull container updates locally
|
||||
CRITICAL_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH
|
||||
|
||||
# Orchestrators that use rsync — read by rsync_stop.sh for auto-detection
|
||||
# Add any new orchestrator that calls rsync.sh here — no script changes needed
|
||||
# rsync_stop.sh will detect and handle it automatically on stop
|
||||
RSYNC_ORCHESTRATORS=(
|
||||
"daily_sync_maintenance"
|
||||
"weekly_sync_maintenance"
|
||||
)
|
||||
|
||||
# ━━━ Media Management ━━━
|
||||
# Job list run directly by daily_sync_maintenance.sh after the media share sync.
|
||||
# Runs sequentially — permissions first, then cleaners, then arr cleanup.
|
||||
@@ -265,6 +351,31 @@ MEDIA_MANAGEMENT_JOBS=(
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Rsync Enable/Disable ━━━
|
||||
# Two-tier toggle system — Tier 1 overrides Tier 2.
|
||||
#
|
||||
# Tier 1 — Global gate:
|
||||
# RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions
|
||||
# Use when: remote server completely offline, major maintenance, disaster recovery
|
||||
#
|
||||
# Tier 2 — Per-orchestrator (only applies when Tier 1 is true):
|
||||
# Fine grained control — disable specific orchestrators while keeping others
|
||||
# Use when: rebuilding secondary, testing, per-window bandwidth management
|
||||
#
|
||||
# Real world example (HOST2 data rebuild — your current situation):
|
||||
# RSYNC_ENABLED=true ← rsync works, individual scripts run fine
|
||||
# DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild
|
||||
# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe, separate BW)
|
||||
# FAILOVER_RSYNC_ENABLED=true ← handback writeback still works when needed
|
||||
# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies (test each share manually)
|
||||
# → When ready: DAILY_RSYNC_ENABLED=true
|
||||
|
||||
RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below
|
||||
DAILY_RSYNC_ENABLED=false # Tier 2 — daily_sync_maintenance.sh rsync section
|
||||
WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section
|
||||
FAILOVER_RSYNC_ENABLED=true # Tier 2 — failover.sh writeback jobs on handback
|
||||
CRITICAL_RSYNC_ENABLED=true # Tier 2 — critical_sync_maintenance.sh (every 15min)
|
||||
|
||||
# ━━━ Rsync Defaults ━━━
|
||||
# Global fallback values used when no profile match is found.
|
||||
# Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed.
|
||||
@@ -300,7 +411,7 @@ MEDIA_MANAGEMENT_JOBS=(
|
||||
# Current profiles:
|
||||
# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency
|
||||
# critical-data — auth stack — containers stopped both sides, Authelia delayed start
|
||||
# gmer4lfe — server-specific appdata — no container stops needed
|
||||
# host1 — server-specific appdata — no container stops needed
|
||||
# important-data — NextCloud + Postgres — NextCloud delayed start after Postgres
|
||||
# emby — weekly clean sync — both Emby stopped, full mirror, minimal excludes
|
||||
# called by weekly_sync_maintenance.sh only — do NOT schedule separately
|
||||
@@ -310,7 +421,7 @@ MEDIA_MANAGEMENT_JOBS=(
|
||||
declare -A PROFILE_RSYNC_OPTS=(
|
||||
[arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
|
||||
[critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete"
|
||||
[gmer4lfe]="-av --info=progress2 --bwlimit=$BW_LIMIT"
|
||||
[host1]="-av --info=progress2 --bwlimit=$BW_LIMIT"
|
||||
[important-data]="-av --human-readable --bwlimit=$BW_LIMIT"
|
||||
[emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
|
||||
[emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
|
||||
@@ -321,7 +432,7 @@ declare -A PROFILE_RSYNC_OPTS=(
|
||||
declare -A PROFILE_BW_LIMIT=(
|
||||
[arrs_stack]=5000 # lower — runs alongside other syncs, avoids saturating link
|
||||
[critical-data]=9500 # high — small dataset, get it synced fast and clean
|
||||
[gmer4lfe]=8000
|
||||
[host1]=8000
|
||||
[important-data]=9500 # high — database sync needs to be fast
|
||||
[emby]=8000 # medium — large full mirror, steady transfer
|
||||
[emby-failover]=9500 # high — small critical dataset, sync as fast as possible
|
||||
@@ -331,7 +442,7 @@ declare -A PROFILE_BW_LIMIT=(
|
||||
declare -A PROFILE_RETRY_COUNT=(
|
||||
[arrs_stack]=3
|
||||
[critical-data]=3
|
||||
[gmer4lfe]=3
|
||||
[host1]=3
|
||||
[important-data]=3
|
||||
[emby]=3
|
||||
[emby-failover]=3
|
||||
@@ -342,7 +453,7 @@ declare -A PROFILE_RETRY_COUNT=(
|
||||
declare -A PROFILE_SLEEP=(
|
||||
[arrs_stack]=300
|
||||
[critical-data]=300
|
||||
[gmer4lfe]=300
|
||||
[host1]=300
|
||||
[important-data]=300
|
||||
[emby]=300
|
||||
[emby-failover]=120 # shorter — frequent dirty sync, retry faster
|
||||
@@ -358,7 +469,7 @@ declare -A PROFILE_SLEEP=(
|
||||
declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
|
||||
[arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
|
||||
[critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap NginxProxyManager Authelia Authelia-Secondary"
|
||||
[gmer4lfe]="Organizrv2 UptimeKuma VaultWarden"
|
||||
[host1]="Organizrv2 UptimeKuma VaultWarden"
|
||||
[important-data]="Postgres-NextCloud NextCloud"
|
||||
[emby]="Emby" # weekly clean sync — both Emby instances stopped, WAL checkpointed
|
||||
[emby-failover]="" # dirty sync — Emby stays running both sides, WAL excluded from sync
|
||||
@@ -371,7 +482,7 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
|
||||
declare -A PROFILE_DELAYED_CONTAINERS=(
|
||||
[arrs_stack]=""
|
||||
[critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis to be ready
|
||||
[gmer4lfe]=""
|
||||
[host1]=""
|
||||
[important-data]="NextCloud" # wait for Postgres to accept connections
|
||||
[emby]=""
|
||||
[emby-failover]=""
|
||||
@@ -382,7 +493,7 @@ declare -A PROFILE_DELAYED_CONTAINERS=(
|
||||
declare -A PROFILE_CONTAINER_DELAY=(
|
||||
[arrs_stack]=5
|
||||
[critical-data]=15 # Mariadb + Redis need time to accept connections
|
||||
[gmer4lfe]=5
|
||||
[host1]=5
|
||||
[important-data]=10 # Postgres needs time before NextCloud
|
||||
[emby]=5
|
||||
[emby-failover]=5
|
||||
@@ -394,7 +505,7 @@ declare -A PROFILE_CONTAINER_DELAY=(
|
||||
# SPACE-SEPARATED STRINGS — converted to array at runtime
|
||||
declare -A PROFILE_EXCLUDE_DIRS=(
|
||||
[arrs_stack]="logs *.tmp"
|
||||
[gmer4lfe]="logs *.tmp"
|
||||
[host1]="logs *.tmp"
|
||||
[important-data]="logs *.tmp"
|
||||
[critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt"
|
||||
[emby]="logs transcodes cache crash*"
|
||||
@@ -403,16 +514,8 @@ declare -A PROFILE_EXCLUDE_DIRS=(
|
||||
[emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root"
|
||||
)
|
||||
|
||||
# Skip per-disk space check for these profiles — appdata syncs go to cache/appdata
|
||||
# not to array disks, so disk space check is irrelevant and just slows things down
|
||||
declare -A PROFILE_SKIP_DISK_CHECK=(
|
||||
[arrs_stack]=true
|
||||
[critical-data]=true
|
||||
[gmer4lfe]=true
|
||||
[important-data]=true
|
||||
[emby]=true
|
||||
[emby-failover]=true
|
||||
)
|
||||
# Skip per-disk space check is no longer needed — check_remote_disks() auto-detects
|
||||
# XFS and ZFS filesystem types from disks.ini, no manual configuration required
|
||||
|
||||
# ==============================================================================================
|
||||
# ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- System Watchdog --------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Last line of defense — reboots the system cleanly if it is about to become unstable.
|
||||
# Last line of defense — reb on oots the system cleanly if it is about to become unstable.
|
||||
# Runs continuously as a background process — started by array_start.sh at array start.
|
||||
# Works alongside docker_watchdog.sh which handles container-level healing first.
|
||||
#
|
||||
|
||||
Reference in New Issue
Block a user