padded partnership set up

2026-04-29 20:09:28 -04:00
parent 95ce62e663
commit 94f90a4579
8 changed files with 2414 additions and 821 deletions
@@ -98,8 +98,8 @@

 # SSH keys for server-to-server rsync and failover container operations.
 # Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys.
-    HOST1_SSH_KEY="/root/.ssh/your_key"
-    HOST2_SSH_KEY="/root/.ssh/your_key"
+    HOST1_SSH_KEY="/root/.ssh/your_host1_key"
+    HOST2_SSH_KEY="/root/.ssh/your_host2_key"

 # ━━━ Emby ━━━
 # Defined once here — referenced by transcode_manager.sh, emby_session_report.sh,
@@ -113,6 +113,75 @@
    HOST2_EMBY_URL="http://localhost:8096"       # same port — different server, different key
    HOST2_EMBY_API_KEY="your-emby-api-key"

+# ==============================================================================================
+# ── PARTNERSHIP ────────────────────────────────────────────────────────────────────────────────
+# ==============================================================================================
+# Manages the relationship lifecycle between two unRAID servers.
+# HOST1 is always the owner (source of truth) — HOST2 is always the mirror.
+# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation.
+# All other vars derive from HOST1/HOST2 — no duplicate IP or key configuration needed.
+#
+# State files live on /boot/config — survives reboots, available before array starts:
+#   /boot/config/partnership_HOST1.db ← HOST1 writes only
+#   /boot/config/partnership_HOST2.db ← HOST2 writes only
+#   Each server SSHes to write/read the other's file directly — no rsync needed
+#
+# critical_sync_maintenance.sh runs --check every 15min:
+#   Reads both state files via SSH
+#   Detects offboard requests → finalises from owner side
+#   Increments offline counter → auto-offboards after threshold
+#   Silent when healthy ✅
+
+    PARTNERSHIP_ENABLED=false
+    PARTNERSHIP_OWNER_HOST="HOST1"          # "HOST1" or "HOST2" — flips on --transfer
+
+# Auth containers whose WebUI URL gets reconfigured on onboard/offboard
+# Format: "ContainerName|Port"
+# These are the containers the mirror operator clicks in his Docker UI
+# On onboard → pointed at owner's Tailscale IP
+# On offboard → pointed back at localhost
+PARTNERSHIP_AUTH_WEBUIS=(
+    "NginxProxyManager|81"
+    "Lldap|17170"
+    "Authelia|9091"
+    "Authelia-Secondary|9092"
+)
+
+# Paths on HOST1 that HOST2 operator should collect before backup retention expires
+# Notified on offboard — no auto-deletion, manual collection
+PARTNERSHIP_MIRROR_BACKUPS=(
+    # "/mnt/user/appdata-Failover/Host2-Emby"
+    # add any paths HOST2 wants to collect after separation
+)
+
+# Offboard settings
+    PARTNERSHIP_GRACE_HOURS=3               # hours after offboard before Tailscale removal
+                                            # backup access also expires at this time
+                                            # collect anything needed before this window closes
+    PARTNERSHIP_REMOVE_TAILSCALE=true       # remove mirror from Tailscale on offboard
+                                            # false = skip removal (manual or testing)
+
+# Tailscale API — used to remove HOST2 from tailnet on offboard
+# API key: https://login.tailscale.com/admin/settings/keys
+# Tailnet: your tailnet name (e.g. "yourname.github" or "youremail.com")
+    TAILSCALE_API_KEY=""                    # tskey-api-...
+    TAILSCALE_TAILNET=""                    # your tailnet name
+
+# Transfer safety
+    PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership"
+    PARTNERSHIP_TRANSFER_STRIKES=3          # consecutive health checks required
+    PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20    # max attempts before giving up
+
+# Onboard settings
+    PARTNERSHIP_ONBOARD_VERIFY=true         # verify WebUI reachable after reconfiguration
+    PARTNERSHIP_ONBOARD_NOTIFY=true         # notify both servers on completion
+    PARTNERSHIP_SYNC_INTERVAL=15            # minutes — Critical-Data sync frequency
+
+# Auto-offboard threshold
+    PARTNERSHIP_OFFLINE_THRESHOLD=30        # days either server unreachable before auto-offboard
+                                            # works both directions — mirror offline 30d → owner offboards
+                                            # owner offline 30d → mirror offboards itself
+
 # ==============================================================================================
 # ── LOGGING ───────────────────────────────────────────────────────────────────────────────────
 # ==============================================================================================
@@ -142,7 +211,7 @@
 # Falls back to GITEA_DOMAIN if local and Tailscale both fail.
    GITEA_CONTAINER="Gitea"                             # exact Docker container name
    GITEA_REPO_PATH="youruser/Unraid_Scripts.git"   # repo path on Gitea server
-    GITEA_DOMAIN=""                                     # e.g. git.gmer4lfe.com — requires NPM + DNS setup
+    GITEA_DOMAIN=""                                     # e.g. git.yourdomain.com — requires NPM + DNS setup
    TARGET_DIR="/mnt/user/appdata/unraid_scripts"       # where scripts are cloned to
    GITEA_SSH_KEY="/root/.ssh/your_gitea_key"             # SSH key for authenticating to Gitea
    SSH_PORT=221                                        # Gitea SSH port (default 22, Gitea often uses 222/221)
@@ -238,6 +307,15 @@ WEEKLY_SYNC_JOBS=(
    "/mnt/user/appdata-Failover/Critical-Data"  # critical-data profile — auth stack
 )

+# ━━━ Critical Sync Maintenance ━━━
+# Shares synced every 15 minutes by critical_sync_maintenance.sh
+# Format: "/path/to/share" or "/path/to/share|profile-name"
+# Order matters — Critical-Data first (auth stack), then emby-failover (dirty sync)
+CRITICAL_SYNC_SHARES=(
+    "/mnt/user/appdata-Failover/Critical-Data"              # auth stack — full sync
+    "/mnt/user/Media_Server/Emby|emby-failover"             # Emby dirty sync — stays running
+)
+
 # Container update toggles for the weekly sync window.
 # Containers are already stopped for the sync — updates pull at no extra downtime.
 # Both false → sync only, no updates.
@@ -245,6 +323,14 @@ WEEKLY_SYNC_JOBS=(
    CRITICAL_SYNC_UPDATES=true          # pull container updates locally
    CRITICAL_SYNC_UPDATES_REMOTE=true   # pull container updates on remote via SSH

+# Orchestrators that use rsync — read by rsync_stop.sh for auto-detection
+# Add any new orchestrator that calls rsync.sh here — no script changes needed
+# rsync_stop.sh will detect and handle it automatically on stop
+RSYNC_ORCHESTRATORS=(
+    "daily_sync_maintenance"
+    "weekly_sync_maintenance"
+)
+
 # ━━━ Media Management ━━━
 # Job list run directly by daily_sync_maintenance.sh after the media share sync.
 # Runs sequentially — permissions first, then cleaners, then arr cleanup.
@@ -265,6 +351,31 @@ MEDIA_MANAGEMENT_JOBS=(
 # ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
 # ==============================================================================================

+# ━━━ Rsync Enable/Disable ━━━
+# Two-tier toggle system — Tier 1 overrides Tier 2.
+#
+# Tier 1 — Global gate:
+#   RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions
+#   Use when: remote server completely offline, major maintenance, disaster recovery
+#
+# Tier 2 — Per-orchestrator (only applies when Tier 1 is true):
+#   Fine grained control — disable specific orchestrators while keeping others
+#   Use when: rebuilding secondary, testing, per-window bandwidth management
+#
+# Real world example (HOST2 data rebuild — your current situation):
+#   RSYNC_ENABLED=true           ← rsync works, individual scripts run fine
+#   DAILY_RSYNC_ENABLED=false    ← skip daily HDD syncs during rebuild
+#   WEEKLY_RSYNC_ENABLED=true    ← Emby + Critical-Data still sync (NVMe, separate BW)
+#   FAILOVER_RSYNC_ENABLED=true  ← handback writeback still works when needed
+#   → Run individual: bash Rsync/rsync.sh /mnt/user/Movies (test each share manually)
+#   → When ready: DAILY_RSYNC_ENABLED=true
+
+    RSYNC_ENABLED=true              # Tier 1 — global gate, overrides everything below
+    DAILY_RSYNC_ENABLED=false       # Tier 2 — daily_sync_maintenance.sh rsync section
+    WEEKLY_RSYNC_ENABLED=true       # Tier 2 — weekly_sync_maintenance.sh rsync section
+    FAILOVER_RSYNC_ENABLED=true     # Tier 2 — failover.sh writeback jobs on handback
+    CRITICAL_RSYNC_ENABLED=true     # Tier 2 — critical_sync_maintenance.sh (every 15min)
+
 # ━━━ Rsync Defaults ━━━
 # Global fallback values used when no profile match is found.
 # Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed.
@@ -300,7 +411,7 @@ MEDIA_MANAGEMENT_JOBS=(
 # Current profiles:
 #   arrs_stack    — arr databases — lower bandwidth, containers stopped for consistency
 #   critical-data — auth stack — containers stopped both sides, Authelia delayed start
-#   gmer4lfe      — server-specific appdata — no container stops needed
+#   host1         — server-specific appdata — no container stops needed
 #   important-data — NextCloud + Postgres — NextCloud delayed start after Postgres
 #   emby          — weekly clean sync — both Emby stopped, full mirror, minimal excludes
 #                   called by weekly_sync_maintenance.sh only — do NOT schedule separately
@@ -310,7 +421,7 @@ MEDIA_MANAGEMENT_JOBS=(
 declare -A PROFILE_RSYNC_OPTS=(
    [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
    [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete"
-    [gmer4lfe]="-av --info=progress2 --bwlimit=$BW_LIMIT"
+    [host1]="-av --info=progress2 --bwlimit=$BW_LIMIT"
    [important-data]="-av --human-readable --bwlimit=$BW_LIMIT"
    [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
    [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file"
@@ -321,7 +432,7 @@ declare -A PROFILE_RSYNC_OPTS=(
 declare -A PROFILE_BW_LIMIT=(
    [arrs_stack]=5000       # lower — runs alongside other syncs, avoids saturating link
    [critical-data]=9500    # high — small dataset, get it synced fast and clean
-    [gmer4lfe]=8000
+    [host1]=8000
    [important-data]=9500   # high — database sync needs to be fast
    [emby]=8000             # medium — large full mirror, steady transfer
    [emby-failover]=9500    # high — small critical dataset, sync as fast as possible
@@ -331,7 +442,7 @@ declare -A PROFILE_BW_LIMIT=(
 declare -A PROFILE_RETRY_COUNT=(
    [arrs_stack]=3
    [critical-data]=3
-    [gmer4lfe]=3
+    [host1]=3
    [important-data]=3
    [emby]=3
    [emby-failover]=3
@@ -342,7 +453,7 @@ declare -A PROFILE_RETRY_COUNT=(
 declare -A PROFILE_SLEEP=(
    [arrs_stack]=300
    [critical-data]=300
-    [gmer4lfe]=300
+    [host1]=300
    [important-data]=300
    [emby]=300
    [emby-failover]=120     # shorter — frequent dirty sync, retry faster
@@ -358,7 +469,7 @@ declare -A PROFILE_SLEEP=(
 declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
    [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
    [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap NginxProxyManager Authelia Authelia-Secondary"
-    [gmer4lfe]="Organizrv2 UptimeKuma VaultWarden"
+    [host1]="Organizrv2 UptimeKuma VaultWarden"
    [important-data]="Postgres-NextCloud NextCloud"
    [emby]="Emby"           # weekly clean sync — both Emby instances stopped, WAL checkpointed
    [emby-failover]=""      # dirty sync — Emby stays running both sides, WAL excluded from sync
@@ -371,7 +482,7 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
 declare -A PROFILE_DELAYED_CONTAINERS=(
    [arrs_stack]=""
    [critical-data]="Authelia Authelia-Secondary"   # wait for Mariadb + Redis to be ready
-    [gmer4lfe]=""
+    [host1]=""
    [important-data]="NextCloud"                    # wait for Postgres to accept connections
    [emby]=""
    [emby-failover]=""
@@ -382,7 +493,7 @@ declare -A PROFILE_DELAYED_CONTAINERS=(
 declare -A PROFILE_CONTAINER_DELAY=(
    [arrs_stack]=5
    [critical-data]=15      # Mariadb + Redis need time to accept connections
-    [gmer4lfe]=5
+    [host1]=5
    [important-data]=10     # Postgres needs time before NextCloud
    [emby]=5
    [emby-failover]=5
@@ -394,7 +505,7 @@ declare -A PROFILE_CONTAINER_DELAY=(
 # SPACE-SEPARATED STRINGS — converted to array at runtime
 declare -A PROFILE_EXCLUDE_DIRS=(
    [arrs_stack]="logs *.tmp"
-    [gmer4lfe]="logs *.tmp"
+    [host1]="logs *.tmp"
    [important-data]="logs *.tmp"
    [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt"
    [emby]="logs transcodes cache crash*"
@@ -403,16 +514,8 @@ declare -A PROFILE_EXCLUDE_DIRS=(
    [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root"
 )

-# Skip per-disk space check for these profiles — appdata syncs go to cache/appdata
-# not to array disks, so disk space check is irrelevant and just slows things down
-declare -A PROFILE_SKIP_DISK_CHECK=(
-    [arrs_stack]=true
-    [critical-data]=true
-    [gmer4lfe]=true
-    [important-data]=true
-    [emby]=true
-    [emby-failover]=true
-)
+# Skip per-disk space check is no longer needed — check_remote_disks() auto-detects
+# XFS and ZFS filesystem types from disks.ini, no manual configuration required

 # ==============================================================================================
 # ── FAILOVER ──────────────────────────────────────────────────────────────────────────────────