diff --git a/Master.conf b/Master.conf index f29d780..b96acaf 100644 --- a/Master.conf +++ b/Master.conf @@ -9,7 +9,7 @@ # Every script sources Master.conf and common.sh at startup. # Change a value here and it affects all scripts that use it — no hunting through files. # To disable something: comment it out with # rather than deleting it. -# To add a new rsync profile: add a key to each PROFILE_* array. +# To add a new rsync profile: add a key to each PROFILE_* array in the RSYNC section. # To add or remove orchestrator jobs: edit the arrays in the ORCHESTRATORS section. # # ── INDEX ───────────────────────────────────────────────────────────────────────────────────── @@ -17,6 +17,7 @@ # Section Description # ─────────────────────────────────────────────────────────────────────────────────────────── # HOST CONFIGURATION Server hostnames, SSH keys, Emby connection details, DATA_DIR +# PARTNERSHIP Mirror relationship lifecycle — onboard/offboard/transfer # LOGGING Enable or disable verbose logging # NOTIFICATIONS unRAID native and Discord webhook settings # GIT / REPO Gitea repository and SSH settings @@ -24,10 +25,11 @@ # ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────── # ARRAY START Scripts launched at array start (array_start.sh) # DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh) -# WEEKLY SYNC MAINTENANCE Job list + sync jobs + sync settings (weekly_sync_maintenance.sh) -# MEDIA MANAGEMENT Job list for media_management.sh +# WEEKLY SYNC MAINTENANCE Job list + sync shares + update toggles (weekly_sync_maintenance.sh) +# CRITICAL SYNC MAINTENANCE 15-minute jobs + sync shares + partnership check (critical_sync_maintenance.sh) # # ── RSYNC ────────────────────────────────────────────────────────────────────────────────── +# RSYNC ENABLE/DISABLE Two-tier toggle system — global gate + per-orchestrator # RSYNC DEFAULTS Global fallback rsync options and limits # REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort # RSYNC PROFILE SYSTEM Per-profile overrides for appdata syncs @@ -42,12 +44,15 @@ # RSYNC WRITEBACK JOBS Appdata synced back to primary on handback # # ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── +# DOWNLOADERS RESET Daily reset for slskd, SABnzbd, qBittorrent # DOCKER DAILY RESTART Containers restarted daily # DOCKER WEEKLY RESTART Containers restarted weekly # DOCKER WATCHDOG Continuous two-tier self-healing container monitoring -# DOCKER NETWORK CONNECT Connect containers to extra networks on array start +# DOCKER NETWORK CONNECT Ensure networks exist + connect containers on array start # # ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── +# INOTIFY TUNING inotify limits — raised at array start by inotify_tuning.sh +# SYSTEM TUNING MONITOR Tracks inotify + php-fpm usage over time # REBOOT User warning delay before scheduled reboot # MOVER Mover stop timeout # SYSLOG FILTER Docker veth noise filter file path @@ -84,7 +89,7 @@ # ============================================================================================== # ━━━ Hosts ━━━ -# Hostnames must match exact Docker/unRAID hostnames — case sensitive. +# Hostnames must match exact Docker/unRAID hostnames AND Tailscale device names — case sensitive. # Used by detect_hosts() in common.sh to determine which server is local and which is remote. # Both servers run identical scripts — host detection makes them bidirectional. HOST1="unRAID-Gmer4Lfe" @@ -107,12 +112,80 @@ # API key: Emby Dashboard → API Keys → + New Key HOST1_EMBY_CONTAINER="Emby" HOST1_EMBY_URL="http://localhost:8096" - HOST1_EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" + HOST1_EMBY_API_KEY="your-host1-emby-api-key" HOST2_EMBY_CONTAINER="Emby-Jayred365" - HOST2_EMBY_URL="http://localhost:8096" # same port — different server, different key + HOST2_EMBY_URL="http://localhost:8096" HOST2_EMBY_API_KEY="your-host2-emby-api-key" +# ============================================================================================== +# ── PARTNERSHIP ─────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Manages the relationship lifecycle between two unRAID servers. +# HOST1 is always the owner (source of truth) — HOST2 is always the mirror. +# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation. +# All vars derive from HOST1/HOST2 — no duplicate IP or key configuration needed. +# Hostnames already match Tailscale device names — IP resolution is automatic. +# +# State files on /boot/config — survives reboots, available before array starts: +# /boot/config/partnership_HOST1.db ← HOST1 writes only +# /boot/config/partnership_HOST2.db ← HOST2 writes only +# Propagated via SSH — no rsync needed +# +# critical_sync_maintenance.sh runs --check every 15min: +# Reads both state files via SSH +# Detects offboard requests → finalises from owner side +# Increments offline counter → auto-offboards after threshold +# Silent when healthy ✅ +# +# See README-Partnership.md for full lifecycle documentation. + + PARTNERSHIP_ENABLED=false + PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer + +# Auth containers reconfigured on onboard/offboard +# Format: "ContainerName|WebUIPort" +# On onboard → WebUI pointed at owner's Tailscale IP (mirror clicks NPM, gets owner's) +# On offboard → WebUI pointed back at localhost + PARTNERSHIP_AUTH_WEBUIS=( + "NginxProxyManager|81" + "Lldap-Gmer4Lfe|17170" + "Authelia|9091" + "Authelia-Secondary|9092" + ) + +# Paths on HOST1 that HOST2 should collect during the grace window after offboard +# Notified on offboard — no auto-deletion, manual collection + PARTNERSHIP_MIRROR_BACKUPS=( + # "/mnt/user/appdata-Failover/Jayred365-Emby" + ) + +# Timing — single var controls both Tailscale removal and backup access expiry +# Both expire at the same time — keeping backups accessible beyond Tailscale removal is pointless + PARTNERSHIP_GRACE_HOURS=6 # hours after offboard before Tailscale removal + # backup access expires at the same time + PARTNERSHIP_OFFLINE_THRESHOLD=30 # days either server unreachable before auto-offboard + # works both directions independently + +# Tailscale removal on offboard + PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale tailnet on offboard + # false = skip removal (manual or testing) + +# Tailscale API — required when PARTNERSHIP_REMOVE_TAILSCALE=true +# API key: https://login.tailscale.com/admin/settings/keys → Devices write scope + TAILSCALE_API_KEY="" # tskey-api-... + TAILSCALE_TAILNET="" # your tailnet name (e.g. yourname.github) + +# Transfer safety + PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership" + PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required + PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 # max health check attempts before giving up + +# Onboard settings + PARTNERSHIP_ONBOARD_VERIFY=true # verify WebUI reachable after reconfiguration + PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on completion + PARTNERSHIP_SYNC_INTERVAL=15 # minutes — informational, actual schedule in cron + # ============================================================================================== # ── LOGGING ─────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -140,15 +213,15 @@ # Gitea self-hosted repository — used by git_pull_execute.sh. # Detects Gitea container location at runtime — works through failover automatically. # Falls back to GITEA_DOMAIN if local and Tailscale both fail. - GITEA_CONTAINER="Gitea" # exact Docker container name - GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git" # repo path on Gitea server - GITEA_DOMAIN="" # e.g. git.gmer4lfe.com — requires NPM + DNS setup - TARGET_DIR="/mnt/user/appdata/unraid_scripts" # where scripts are cloned to - GITEA_SSH_KEY="/root/.ssh/unraid_gitea" # SSH key for authenticating to Gitea - SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 222/221) + GITEA_CONTAINER="Gitea" + GITEA_REPO_PATH="FailedProxy/Unraid_Scripts.git" + GITEA_DOMAIN="" # e.g. git.yourdomain.com — requires NPM + DNS + TARGET_DIR="/mnt/user/appdata/unraid_scripts" + GITEA_SSH_KEY="/root/.ssh/unraid_gitea" + SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 221/222) # ============================================================================================== -# ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────────── +# ── ORCHESTRATORS ───────────────────────────────────────────────────────────────────────────── # ============================================================================================== # All orchestrator job lists live here — edit arrays to add/remove scripts. # No changes to orchestrator scripts needed when adding or removing jobs. @@ -156,110 +229,112 @@ # ━━━ Array Start ━━━ # Scripts launched by array_start.sh when the array comes online. # Launched in order — each as a background process. -# One-shot scripts (ramdisk, syslog, fpm, network) run and exit naturally. +# One-shot scripts (ramdisk, syslog, fpm, inotify, network) run and exit naturally. # Continuous scripts (watchdogs, failover) run until array stops. - -ARRAY_START_SCRIPTS=( - "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts - "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill - "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning - "unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted - "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers - "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop - "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop - "Failover/failover.sh" # mutual failover — continuous loop -) + ARRAY_START_SCRIPTS=( + "Transcodes/ramdisk_setup.sh" # creates ramdisk + symlink before Emby starts + "unRAID_Essentials/docker_syslog_filter.sh" # suppress veth noise before logs fill + "unRAID_Essentials/php_fpm_max_children.sh" # WebGUI performance tuning + "unRAID_Essentials/inotify_tuning.sh" # bump inotify limits — containers miss events if exhausted + "Docker_Essentials/docker_network_connect.sh" # ensure networks exist + connect containers + "unRAID_Essentials/system_watchdog.sh" # system health monitor — continuous loop + "Docker_Essentials/docker_watchdog.sh" # container health monitor — continuous loop +# "Failover/failover.sh" # mutual failover — enable when HOST2 ready + ) # ━━━ Daily Sync Maintenance ━━━ -# daily_sync_maintenance.sh runs the media share sync built into the script first, -# then iterates DAILY_MAINTENANCE_SCRIPTS for additional jobs. +# daily_sync_maintenance.sh runs media share sync first, then iterates +# DAILY_MAINTENANCE_SCRIPTS for all pre and post sync jobs. # Schedule: 0 1 * * * (1am daily) - -DAILY_MAINTENANCE_SCRIPTS=( - "git_pull_execute.sh" # pull latest scripts — always runs first - "Docker_Essentials/docker_daily_restart.sh" # daily container restarts -) + DAILY_MAINTENANCE_SCRIPTS=( + "git_pull_execute.sh" # pull latest scripts — always runs first + "Media/media_shares_permissions.sh" # apply permissions + "Media/media_cleaner.sh anime" # remove junk from anime shares + "Media/media_cleaner.sh media" # remove junk from media shares + #"Media/lidarr_cleanup.sh" # remove orphaned music files — enable when ready + #"Media/sonarr_cleanup.sh" # remove orphaned TV files — enable when ready + #"Media/radarr_cleanup.sh" # remove orphaned movie files — enable when ready + "Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last + ) # Media shares synced daily by daily_sync_maintenance.sh. -# Each server syncs only the shares it owns (source of truth) — direction is automatic. -# HOST1 pushes its truth shares to HOST2. HOST2 pushes its truth shares to HOST1. +# Each server syncs only the shares it owns — direction is automatic. +# HOST1 pushes its shares to HOST2. HOST2 pushes its shares to HOST1. # Never both pushing the same share — one server is always the truth holder. # These shares use DEFAULT_RSYNC_OPTS — no profile entry needed. -# For shares needing custom options or container stops — create a profile in the RSYNC section. +# For shares needing custom options or container stops — create a profile in RSYNC section. + HOST1_DAILY_SYNC_SHARES=( + /mnt/user/Books + /mnt/user/Intros + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Music_Videos + /mnt/user/Nextcloud + /mnt/user/stand-up_comedy + /mnt/user/Sports + /mnt/user/Tv_Shows + /mnt/user/Anime_Shows-Old + /mnt/user/Anime_Movies-Old + ) -HOST1_DAILY_SYNC_SHARES=( - /mnt/user/Books - /mnt/user/Intros - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Music_Videos - /mnt/user/Nextcloud - /mnt/user/stand-up_comedy - /mnt/user/Sports - /mnt/user/Tv_Shows - /mnt/user/Anime_Shows-Old - /mnt/user/Anime_Movies-Old -) - -HOST2_DAILY_SYNC_SHARES=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Shows -) + HOST2_DAILY_SYNC_SHARES=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Shows + ) # Personal encrypted shares — synced for offsite backup, independent of media shares. # ZFS encrypted at dataset level — remote receives encrypted blocks, cannot read content. # See README-Rsync_Setup.md for ZFS encryption setup before uncommenting. -HOST1_PERSONAL_SHARES=( - # /mnt/user/Gmer4Lfe-Personal # uncomment after creating encrypted dataset -) + HOST1_PERSONAL_SHARES=( + # /mnt/user/HOST1-Personal # uncomment after creating encrypted dataset + ) -HOST2_PERSONAL_SHARES=( - # /mnt/user/Jayred365-Personal # uncomment after creating encrypted dataset -) + HOST2_PERSONAL_SHARES=( + # /mnt/user/HOST2-Personal # uncomment after creating encrypted dataset + ) # ━━━ Weekly Sync Maintenance ━━━ -# weekly_sync_maintenance.sh handles the critical sync built into the script first: -# stop containers both sides → pull updates → sync Emby + Critical-Data → restart -# Then iterates WEEKLY_MAINTENANCE_SCRIPTS for additional jobs. +# weekly_sync_maintenance.sh stops containers both sides → pulls updates → +# syncs WEEKLY_SYNC_SHARES → restarts → then iterates WEEKLY_MAINTENANCE_SCRIPTS. # Schedule: 30 2 * * 0 (Sunday 2:30am) + WEEKLY_MAINTENANCE_SCRIPTS=( + "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync + ) -WEEKLY_MAINTENANCE_SCRIPTS=( - "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync -) - -# Shares synced by weekly_sync_maintenance.sh during the maintenance window. -# Containers are stopped both sides before these sync — full clean state guaranteed. +# Shares synced during the weekly maintenance window. +# Containers stopped both sides before sync — full clean state guaranteed. # Profiles drive container stops, excludes, and options — configure in RSYNC section. -# Order matters — Emby first, then auth stack. -WEEKLY_SYNC_JOBS=( - "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror - "/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack -) +# Order matters — Emby first (larger), then Critical-Data (auth stack). + WEEKLY_SYNC_SHARES=( + "/mnt/user/Media_Server/Emby" # emby profile — full clean mirror + "/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack + ) # Container update toggles for the weekly sync window. -# Containers are already stopped for the sync — updates pull at no extra downtime. +# Containers already stopped for sync — updates pull at no extra downtime. # Both false → sync only, no updates. -# Toggle false temporarily to skip updates without changing the schedule. - CRITICAL_SYNC_UPDATES=true # pull container updates locally - CRITICAL_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH + WEEKLY_SYNC_UPDATES=true # pull container updates locally during weekly window + WEEKLY_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH -# ━━━ Media Management ━━━ -# Job list run directly by daily_sync_maintenance.sh after the media share sync. -# Runs sequentially — permissions first, then cleaners, then arr cleanup. -# Comment out any job to disable without removing it. -# Each individual script can still be run manually for one-off maintenance. +# ━━━ Critical Sync Maintenance ━━━ +# critical_sync_maintenance.sh runs every 15 minutes. +# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check +# partnership --check always runs last regardless of rsync gate. +# Format for shares: "/path/to/share" or "/path/to/share|profile-name" -MEDIA_MANAGEMENT_JOBS=( - "Media/media_shares_permissions.sh" # apply permissions — runs first - "Media/media_cleaner.sh anime" # remove junk from anime shares - "Media/media_cleaner.sh media" # remove junk from media shares - "Media/lidarr_cleanup.sh" # remove orphaned music files - "Media/sonarr_cleanup.sh" # remove orphaned TV files - "Media/radarr_cleanup.sh" # remove orphaned movie files - "Docker_Essentials/downloaders_reset.sh" # clear stuck states + purge old history -) +# Jobs run every 15 minutes before the rsync shares. +# Comment out to disable without removing. + CRITICAL_MAINTENANCE_SCRIPTS=( + "Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 15min + ) + +# Shares synced during the critical sync window. + CRITICAL_SYNC_SHARES=( + "/mnt/user/appdata-Failover/Critical-Data" # auth stack — full sync + "/mnt/user/Media_Server/Emby|emby-failover" # Emby dirty sync — stays running + ) # ============================================================================================== # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── @@ -270,47 +345,46 @@ MEDIA_MANAGEMENT_JOBS=( # # Tier 1 — Global gate: # RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions -# Use when: remote server completely offline, major maintenance, disaster recovery +# Use when: remote completely offline, major maintenance, disaster recovery # # Tier 2 — Per-orchestrator (only applies when Tier 1 is true): # Fine grained control — disable specific orchestrators while keeping others # Use when: rebuilding secondary, testing, per-window bandwidth management # -# Real world example (HOST2 data rebuild — your current situation): +# Example — HOST2 data rebuild: # RSYNC_ENABLED=true ← rsync works, individual scripts run fine # DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild -# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe, separate BW) -# FAILOVER_RSYNC_ENABLED=true ← handback writeback still works when needed -# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies (test each share manually) +# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe) +# CRITICAL_RSYNC_ENABLED=true ← 15min auth stack sync still runs +# FAILOVER_RSYNC_ENABLED=true ← handback writeback still works +# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies # → When ready: DAILY_RSYNC_ENABLED=true - RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below - DAILY_RSYNC_ENABLED=false # Tier 2 — daily_sync_maintenance.sh rsync section + DAILY_RSYNC_ENABLED=false # Tier 2 — HOST2 rebuild in progress, re-enable when ready WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section + CRITICAL_RSYNC_ENABLED=true # Tier 2 — critical_sync_maintenance.sh (every 15min) FAILOVER_RSYNC_ENABLED=true # Tier 2 — failover.sh writeback jobs on handback # ━━━ Rsync Defaults ━━━ # Global fallback values used when no profile match is found. # Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed. # Appdata shares match profiles by directory basename (lowercased). -# If a profile key exists it overrides the global. If missing the global is used. + BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit + RETRY_COUNT=3 # retry attempts before giving up + SLEEP=300 # seconds between retry attempts + CRITICAL_CONTAINER_NAMES=() # containers stopped on REMOTE before rsync — profiles override + DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override + CONTAINER_DELAY=5 # seconds before starting delayed containers + EXCLUDE_DIRS=() # directories excluded from transfer — profiles override - BW_LIMIT=12500 # KB/s — 12500 ≈ 100Mbit — network transfer speed cap - RETRY_COUNT=3 # retry attempts if rsync fails before giving up - SLEEP=300 # seconds between retry attempts - CRITICAL_CONTAINER_NAMES=() # containers to stop on REMOTE before rsync — profiles override - DELAYED_CONTAINERS=() # containers needing delay before starting — profiles override - CONTAINER_DELAY=5 # seconds to wait before starting delayed containers - EXCLUDE_DIRS=() # directories to exclude from transfer — profiles override - -# --delete removes files on remote that no longer exist on source (mirror behaviour) -# --inplace writes directly to destination — better for large files, avoids temp copies -# --no-whole-file forces delta transfer even on fast connections — sends only changed blocks +# --delete removes files on remote not on source (mirror behaviour) +# --inplace writes directly to destination — better for large files +# --no-whole-file forces delta transfer — sends only changed blocks DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) # ━━━ Remote Health Checks ━━━ -# Pre-flight check — aborts if remote rootfs (/) usage is at or above this percentage. -# When remote array is down, rsync writes land on rootfs — fills fast and crashes the server. +# Pre-flight — aborts if remote rootfs (/) usage is at or above this percentage. +# When remote array is down, rsync writes land on rootfs and fill it rapidly. ROOTFS_WARN=75 # ━━━ Rsync Profile System ━━━ @@ -322,113 +396,101 @@ MEDIA_MANAGEMENT_JOBS=( # List ALL desired options explicitly when defining a profile. # # Current profiles: -# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency -# critical-data — auth stack — containers stopped both sides, Authelia delayed start -# gmer4lfe — server-specific appdata — no container stops needed +# arrs_stack — arr databases — lower bandwidth, containers stopped for consistency +# critical-data — auth stack — containers stopped both sides, Authelia delayed start +# gmer4lfe — server-specific appdata — no container stops needed # important-data — NextCloud + Postgres — NextCloud delayed start after Postgres -# emby — weekly clean sync — both Emby stopped, full mirror, minimal excludes -# called by weekly_sync_maintenance.sh only — do NOT schedule separately -# emby-failover — frequent dirty sync — Emby stays running, WAL excluded, critical data only -# also used for failover writeback on handback +# emby — weekly clean sync — both Emby stopped, full mirror +# called by weekly_sync_maintenance.sh only — do NOT schedule separately +# emby-failover — frequent dirty sync — Emby stays running, WAL excluded +# also used for failover writeback on handback -declare -A PROFILE_RSYNC_OPTS=( - [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" - [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" - [gmer4lfe]="-av --info=progress2 --bwlimit=$BW_LIMIT" - [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" - [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" - [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" -) + declare -A PROFILE_RSYNC_OPTS=( + [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" + [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" + [gmer4lfe]="-av --info=progress2 --bwlimit=$BW_LIMIT" + [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" + [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" + [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" + ) -# Per-profile bandwidth limits in KB/s — overrides global BW_LIMIT for that profile only -# Lower for shares running alongside other jobs, higher for time-sensitive critical data -declare -A PROFILE_BW_LIMIT=( - [arrs_stack]=5000 # lower — runs alongside other syncs, avoids saturating link - [critical-data]=9500 # high — small dataset, get it synced fast and clean - [gmer4lfe]=8000 - [important-data]=9500 # high — database sync needs to be fast - [emby]=8000 # medium — large full mirror, steady transfer - [emby-failover]=9500 # high — small critical dataset, sync as fast as possible -) +# Per-profile bandwidth limits in KB/s + declare -A PROFILE_BW_LIMIT=( + [arrs_stack]=5000 # lower — runs alongside other syncs + [critical-data]=9500 # high — small dataset, sync fast + [gmer4lfe]=8000 + [important-data]=9500 # high — database sync + [emby]=8000 # medium — large full mirror + [emby-failover]=9500 # high — small critical dataset + ) -# Retry attempts per profile — how many times to retry before giving up on a failed sync -declare -A PROFILE_RETRY_COUNT=( - [arrs_stack]=3 - [critical-data]=3 - [gmer4lfe]=3 - [important-data]=3 - [emby]=3 - [emby-failover]=3 -) +# Retry attempts per profile + declare -A PROFILE_RETRY_COUNT=( + [arrs_stack]=3 + [critical-data]=3 + [gmer4lfe]=3 + [important-data]=3 + [emby]=3 + [emby-failover]=3 + ) -# Seconds to wait between retry attempts -# emby-failover shorter — frequent sync, faster retry on transient failures -declare -A PROFILE_SLEEP=( - [arrs_stack]=300 - [critical-data]=300 - [gmer4lfe]=300 - [important-data]=300 - [emby]=300 - [emby-failover]=120 # shorter — frequent dirty sync, retry faster -) +# Seconds between retry attempts + declare -A PROFILE_SLEEP=( + [arrs_stack]=300 + [critical-data]=300 + [gmer4lfe]=300 + [important-data]=300 + [emby]=300 + [emby-failover]=120 # shorter — frequent dirty sync, retry faster + ) -# Containers stopped on BOTH LOCAL and REMOTE servers before rsync. -# Local stops first — flushes databases cleanly before pushing data out. -# Remote stops next — prevents writes to destination while receiving. -# Only containers that were running get restarted — stopped containers stay stopped. -# Same container names on both servers — consistent naming is required by this ecosystem. -# If a container is not found on a server it is skipped gracefully, not errored. +# Containers stopped on BOTH LOCAL and REMOTE before rsync. +# Local stops first — flushes databases cleanly. Remote stops next — prevents writes. +# Only running containers get restarted — stopped containers stay stopped. # SPACE-SEPARATED STRINGS — converted to array at runtime -declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( - [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" - [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary" - [gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe" - [important-data]="Postgres-NextCloud NextCloud" - [emby]="Emby" # weekly clean sync — both Emby instances stopped, WAL checkpointed - [emby-failover]="" # dirty sync — Emby stays running both sides, WAL excluded from sync -) + declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( + [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" + [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary" + [gmer4lfe]="Organizrv2-Gmer4Lfe UptimeKuma-Gmer4Lfe VaultWarden-Gmer4Lfe" + [important-data]="Postgres-NextCloud NextCloud" + [emby]="Emby" + [emby-failover]="" # dirty sync — Emby stays running both sides + ) -# Containers that need a delay before starting after rsync completes. -# Database containers must be accepting connections before dependent apps start. -# Authelia waits for Mariadb + Redis. NextCloud waits for Postgres. +# Containers needing a delay after rsync before starting. # SPACE-SEPARATED STRINGS — converted to array at runtime -declare -A PROFILE_DELAYED_CONTAINERS=( - [arrs_stack]="" - [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis to be ready - [gmer4lfe]="" - [important-data]="NextCloud" # wait for Postgres to accept connections - [emby]="" - [emby-failover]="" -) + declare -A PROFILE_DELAYED_CONTAINERS=( + [arrs_stack]="" + [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis + [gmer4lfe]="" + [important-data]="NextCloud" # wait for Postgres + [emby]="" + [emby-failover]="" + ) -# Seconds to wait before starting delayed containers -# 15s gives Mariadb, Redis, and LLDAP time to accept connections before Authelia starts -declare -A PROFILE_CONTAINER_DELAY=( - [arrs_stack]=5 - [critical-data]=15 # Mariadb + Redis need time to accept connections - [gmer4lfe]=5 - [important-data]=10 # Postgres needs time before NextCloud - [emby]=5 - [emby-failover]=5 -) +# Seconds before starting delayed containers + declare -A PROFILE_CONTAINER_DELAY=( + [arrs_stack]=5 + [critical-data]=15 # Mariadb + Redis need time to accept connections + [gmer4lfe]=5 + [important-data]=10 # Postgres needs time before NextCloud + [emby]=5 + [emby-failover]=5 + ) -# Directories excluded from rsync transfer per profile -# emby-failover excludes WAL files — safe to sync while Emby is running -# emby clean sync only excludes logs, transcodes, cache — full metadata mirror +# Directories excluded from rsync per profile. # SPACE-SEPARATED STRINGS — converted to array at runtime -declare -A PROFILE_EXCLUDE_DIRS=( - [arrs_stack]="logs *.tmp" - [gmer4lfe]="logs *.tmp" - [important-data]="logs *.tmp" - [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" - [emby]="logs transcodes cache crash*" - # emby-failover: Emby running, WAL excluded — only safe critical data synced - # users.db, library.db, authentication.db, config/ — everything else excluded - [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" -) + declare -A PROFILE_EXCLUDE_DIRS=( + [arrs_stack]="logs *.tmp" + [gmer4lfe]="logs *.tmp" + [important-data]="logs *.tmp" + [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" + [emby]="logs transcodes cache crash*" + [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" + ) -# Skip per-disk space check is no longer needed — check_remote_disks() auto-detects -# XFS and ZFS filesystem types from disks.ini, no manual configuration required +# Note: disk check is auto-detected from disks.ini — no PROFILE_SKIP_DISK_CHECK needed. +# check_remote_disks() reads fsType per disk and handles XFS and ZFS automatically. # ============================================================================================== # ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── @@ -454,405 +516,350 @@ declare -A PROFILE_EXCLUDE_DIRS=( FAILOVER_CHECK_INTERVAL=120 FAILOVER_HANDBACK_STRIKES=2 FAILOVER_STATE_FILE="/boot/config/failover_state.db" - FAILOVER_ENABLED=false # set true when HOST2 is back online and tested - # false = suppresses "not running" warnings in status scripts + FAILOVER_ENABLED=false # HOST2 being rebuilt — set true when back online and tested + # false = suppresses "not running" warnings in status scripts # ━━━ Failover Test ━━━ FAILOVER_TEST_BLOCK_WAIT=150 FAILOVER_TEST_HANDBACK_WAIT=360 # ━━━ DDNS ━━━ -HOST1_DDNS_CONTAINERS=( - "Gmer4Lfe.com" -) + HOST1_DDNS_CONTAINERS=( + "Gmer4Lfe.com" + ) -HOST2_DDNS_CONTAINERS=( - "Gmer4Lfe.us" -) + HOST2_DDNS_CONTAINERS=( + "Gmer4Lfe.us" + ) # ━━━ Internet Loss ━━━ -FAILOVER_HOST1_STOP_ON_NO_NET=( - "Gmer4Lfe.com" -) + FAILOVER_HOST1_STOP_ON_NO_NET=( + "Gmer4Lfe.com" + ) -FAILOVER_HOST2_STOP_ON_NO_NET=( - "Gmer4Lfe.us" -) + FAILOVER_HOST2_STOP_ON_NO_NET=( + "Gmer4Lfe.us" + ) # ━━━ Tiered Container Lists ━━━ -# HOST1 runs for HOST2 -FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE=( - "Gmer4Lfe.us" - "VaultWarden-Jayred365" - # "container-placeholder" -) +# HOST1 runs for HOST2 (HOST2 is down) + FAILOVER_HOST1_RUNS_FOR_HOST2_IMMEDIATE=( + "Gmer4Lfe.us" + "VaultWarden-Jayred365" + ) -FAILOVER_HOST1_RUNS_FOR_HOST2_2HR=( - # "container-placeholder" -) + FAILOVER_HOST1_RUNS_FOR_HOST2_2HR=( + # "container-placeholder" + ) -FAILOVER_HOST1_RUNS_FOR_HOST2_6HR=( - # "container-placeholder" -) + FAILOVER_HOST1_RUNS_FOR_HOST2_6HR=( + # "container-placeholder" + ) -FAILOVER_HOST1_RUNS_FOR_HOST2_18HR=( - # "container-placeholder" -) + FAILOVER_HOST1_RUNS_FOR_HOST2_18HR=( + # "container-placeholder" + ) -# HOST2 runs for HOST1 -FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE=( - "Gmer4Lfe.com" - "Emby" - "VaultWarden-Gmer4Lfe" - "Dispatcharr" - "Dispatcharr-Basic" - "Dispatcharr-Iptv-Users" - "ErsatzTV-Emby" -) +# HOST2 runs for HOST1 (HOST1 is down) + FAILOVER_HOST2_RUNS_FOR_HOST1_IMMEDIATE=( + "Gmer4Lfe.com" + "Emby" + "VaultWarden-Gmer4Lfe" + "Dispatcharr" + "Dispatcharr-Basic" + "Dispatcharr-Iptv-Users" + "ErsatzTV-Emby" + ) -FAILOVER_HOST2_RUNS_FOR_HOST1_2HR=( - "Postgres-NextCloud" - "NextCloud" - "PostgreSQL_Immich" - "Immich-Gmer4Lfe" - # "container-placeholder" -) + FAILOVER_HOST2_RUNS_FOR_HOST1_2HR=( + "Postgres-NextCloud" + "NextCloud" + "PostgreSQL_Immich" + "Immich-Gmer4Lfe" + ) -FAILOVER_HOST2_RUNS_FOR_HOST1_6HR=( - "Gitea" - # "container-placeholder" -) + FAILOVER_HOST2_RUNS_FOR_HOST1_6HR=( + "Gitea" + ) -FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=( - "Sonarr" - "Radarr" - "Lidarr" - "Readarr" - "Prowlarr" - "Bazarr" - "SABnzbd-Gmer4Lfe" - "Qbittorrent-Gmer4Lfe" - "LidaTube" - "Pinchflat" - "ChannelTube" - # "container-placeholder" -) + FAILOVER_HOST2_RUNS_FOR_HOST1_18HR=( + "Sonarr" + "Radarr" + "Lidarr" + "Readarr" + "Prowlarr" + "Bazarr" + "SABnzbd-Gmer4Lfe" + "Qbittorrent-Gmer4Lfe" + "LidaTube" + "Pinchflat" + "ChannelTube" + ) # ━━━ Tier Delay Settings ━━━ -# How long the primary server must be down before each tier activates — in minutes. -# Tier 1 is always immediate — Live TV and media can't wait. -# Set independently per host — adjust based on hardware and what's worth starting. -# Longer delays = less resource usage on covering server but slower recovery. -# -# HOST1's containers running on HOST2 (HOST1 is down): -HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich — can wait -HOST1_TIER3_DELAY=720 # 12 hours — secondary services — Gitea etc. -HOST1_TIER4_DELAY=1440 # 24 hours — full workflow — arrs and downloaders +# How long the primary must be down before each tier activates — in minutes. +# Tier 1 is always immediate. Set independently per host. -# HOST2's containers running on HOST1 (HOST2 is down): -HOST2_TIER2_DELAY=240 -HOST2_TIER3_DELAY=720 -HOST2_TIER4_DELAY=1440 +# HOST1's containers running on HOST2 (HOST1 is down) + HOST1_TIER2_DELAY=240 # 4 hours — NextCloud, Immich + HOST1_TIER3_DELAY=720 # 12 hours — secondary services + HOST1_TIER4_DELAY=1440 # 24 hours — arrs + downloaders + +# HOST2's containers running on HOST1 (HOST2 is down) + HOST2_TIER2_DELAY=240 + HOST2_TIER3_DELAY=720 + HOST2_TIER4_DELAY=1440 # ━━━ Rsync Writeback Jobs ━━━ -# Syncs critical appdata BACK to primary server during handback after failover. -# Containers are stopped before writeback runs — clean source, no competing writes. -# Purpose: primary comes back online with the state that built up during its outage -# (watch states, auth changes, library updates that happened on HOST2) +# Syncs critical appdata BACK to primary during handback after failover. +# Containers stopped before writeback runs — clean source, no competing writes. # # HOST*_TIER1_WRITEBACK_DELAY: # Short outages skip Tier 1 writeback — primary state is more reliable than dirty sync data # Only writeback if outage lasted longer than this many minutes -# 60 minutes = if HOST1 was down less than 1hr, don't bother writing back Emby # -# Tier 4 writeback automatically syncs HOST*_DAILY_SYNC_SHARES back — no need to list those here -# Only add paths that are NOT in DAILY_SYNC_SHARES and need writeback after extended outage - -HOST1_TIER1_WRITEBACK_DELAY=60 # minutes — skip Emby writeback if outage under 1hr -HOST2_TIER1_WRITEBACK_DELAY=60 +# Tier 4 writeback automatically syncs HOST*_DAILY_SYNC_SHARES — no need to list those here. +# Only add paths NOT in DAILY_SYNC_SHARES that need writeback after extended outage. + HOST1_TIER1_WRITEBACK_DELAY=60 # skip Emby writeback if outage under 1hr + HOST2_TIER1_WRITEBACK_DELAY=60 # HOST1 writeback — run by HOST2 during HOST1 handback -FAILOVER_HOST1_WRITEBACK_TIER1=( - "/mnt/user/Media_Server/Emby" # watch states, playstates built up during outage -) + FAILOVER_HOST1_WRITEBACK_TIER1=( + "/mnt/user/Media_Server/Emby" # watch states built up during outage + ) -FAILOVER_HOST1_WRITEBACK_TIER2=( - "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres — files added during outage -) + FAILOVER_HOST1_WRITEBACK_TIER2=( + "/mnt/user/appdata-Failover/Important-Data" # NextCloud + Postgres + ) -FAILOVER_HOST1_WRITEBACK_TIER3=( - # "location-placeholder" -) + FAILOVER_HOST1_WRITEBACK_TIER3=( + # "location-placeholder" + ) -FAILOVER_HOST1_WRITEBACK_TIER4=( - # Edge cases outside HOST1_DAILY_SYNC_SHARES - "/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage -) + FAILOVER_HOST1_WRITEBACK_TIER4=( + "/mnt/user/appdata-Failover/Arrs_Stack" # arr databases — downloads queued during outage + ) # HOST2 writeback — run by HOST1 during HOST2 handback -FAILOVER_HOST2_WRITEBACK_TIER1=( - # "/mnt/user/appdata-Failover/Jayred365-Emby" -) + FAILOVER_HOST2_WRITEBACK_TIER1=( + # "/mnt/user/appdata-Failover/Jayred365-Emby" + ) -FAILOVER_HOST2_WRITEBACK_TIER2=( - # "/mnt/user/appdata-Failover/Jayred365-Important" -) + FAILOVER_HOST2_WRITEBACK_TIER2=( + # "/mnt/user/appdata-Failover/Jayred365-Important" + ) -FAILOVER_HOST2_WRITEBACK_TIER3=( - # "location-placeholder" -) + FAILOVER_HOST2_WRITEBACK_TIER3=( + # "location-placeholder" + ) -FAILOVER_HOST2_WRITEBACK_TIER4=( - # Edge cases outside HOST2_DAILY_SYNC_SHARES - "/mnt/user/appdata-Failover/Arrs_Stack" -) + FAILOVER_HOST2_WRITEBACK_TIER4=( + "/mnt/user/appdata-Failover/Arrs_Stack" + ) # ============================================================================================== # ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ Downloaders Reset ━━━ -# Daily maintenance reset for all download clients. -# Called by daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS before container restarts. -# Clears stuck states, purges old history, prepares each downloader for a clean daily cycle. -# -# Retention period — applies to: slskd failed imports, SABnzbd completed and failed history +# Runs every 15 minutes via CRITICAL_MAINTENANCE_SCRIPTS. +# Clears stuck states, purges old history, prepares each client for a clean cycle. DOWNLOADER_RETENTION_DAYS=7 # ── slskd ── -# Clears stuck/errored searches, dead transfer records, purges expired failed imports -# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected +# Clears stuck searches, dead transfers, purges expired failed imports. +# SLSKD_FAILED_IMPORTS_DIR: where Soularr moves albums Lidarr rejected. HOST1_SLSKD_URL="http://localhost:8980" - HOST1_SLSKD_API_KEY="4bF9kL2mNpQrT7vWxYz1A3dEgHjKoRsU" + HOST1_SLSKD_API_KEY="your-slskd-api-key" HOST1_SLSKD_FAILED_IMPORTS_DIR="/mnt/user/Temp_Storage/Slskd/completed/failed_imports" # ── SABnzbd ── -# Clears completed history, failed history, and stalled paused queue items HOST1_SABNZBD_URL="http://localhost:8180" - HOST1_SABNZBD_API_KEY="8bfefe41d83b4d50883e32859b55ca9a" + HOST1_SABNZBD_API_KEY="your-sabnzbd-api-key" # ── qBittorrent ── -# Last chance failsafe — deletes torrents older than QBIT_FAILSAFE_MIN_DAYS -# qBittorrent's own rules handle normal cleanup (ratio >= 1.25 OR 45 days inactive) -# This catches anything missed after extended time -# deleteFiles=false — removes torrent from qBit but leaves files on disk -# Radarr/Sonarr manage actual files independently -# QBIT_FAILSAFE_MIN_RATIO=0 disables ratio gate — age is the only condition +# deleteFiles=false — removes torrent from qBit but leaves files on disk. +# Radarr/Sonarr manage actual files independently. HOST1_QBIT_URL="http://localhost:8080" HOST1_QBIT_USERNAME="admin" - HOST1_QBIT_PASSWORD="changeme" + HOST1_QBIT_PASSWORD="your-qbit-password" QBIT_FAILSAFE_MIN_DAYS=180 - QBIT_FAILSAFE_MIN_RATIO=0 # 0 = age only, no ratio requirement + QBIT_FAILSAFE_MIN_RATIO=0 # 0 = age only, no ratio requirement # ━━━ Docker Daily Restart ━━━ -# Containers restarted every day by docker_daily_restart.sh via daily_sync_maintenance.sh. -# These containers run better with a daily restart — not just "keeping things fresh". -# Dispatcharr specifically degrades over time without restart — daily is intentional. -# Schedule is set in daily_sync_maintenance.sh — runs at 1am as part of daily window. -# Case-sensitive — must match exact Docker container names. -DAILY_RESTART_CONTAINERS=( - "NginxProxyManager" - "Authelia" - "Dispatcharr-Iptv-Users" - "Dispatcharr" # Live TV scheduler — degrades without daily restart - "Dispatcharr-Basic" - "ErsatzTV-Emby" -) +# Containers restarted every day via daily_sync_maintenance.sh. +# Dispatcharr degrades over time without restart — daily is intentional. + DAILY_RESTART_CONTAINERS=( + "NginxProxyManager" + "Lldap-Gmer4Lfe" + "Authelia" + "Authelia-Secondary" + "Dispatcharr-Iptv-Users" + "Dispatcharr" # Live TV scheduler — degrades without daily restart + "Dispatcharr-Basic" + "ErsatzTV-Emby" + ) # ━━━ Docker Weekly Restart ━━━ -# Less critical services restarted weekly by docker_weekly_restart.sh. -# Called by weekly_sync_maintenance.sh Sunday 2:30am — containers already stopped -# for the weekly sync window so restart adds zero extra downtime. -# Weekly restarts also catch any pending image updates not applied during weekly sync. -WEEKLY_RESTART_CONTAINERS=( - "NextCloud" - "Organizrv2-Gmer4Lfe" - "AdGuard-Home" - "Immich-Gmer4Lfe" -) +# Less critical services restarted weekly via weekly_sync_maintenance.sh (Sunday 2:30am). +# Containers already stopped for weekly sync — restart adds zero extra downtime. + WEEKLY_RESTART_CONTAINERS=( + "NextCloud" + "Organizrv2-Gmer4Lfe" + "AdGuard-Home" + "Immich-Gmer4Lfe" + ) # ━━━ Docker Watchdog ━━━ # Continuous two-tier self-healing container monitoring. # Started by array_start.sh — runs until array stops. # Re-sources Master.conf each cycle — add/remove containers without restarting watchdog. -# Silent when all healthy — only logs when something needs attention. -# Heartbeat fires periodically as proof of life even when everything is healthy. # # Tier 1 — strict monitoring of explicitly configured containers: -# Memory hard limits — immediate restart if container exceeds limit +# Memory hard limits — immediate restart if exceeded # CPU thresholds — strike system, restart after CPU_FAIL_LIMIT sustained strikes # HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT failed checks # Required containers — must always be running, strike + skip list with auto-clear # # Tier 2 — global health scan of ALL running containers: # Unhealthy status — Docker HEALTHCHECK unhealthy → restart -# OOM killed — kernel killed container → restart + notify +# OOM killed — kernel killed → restart + notify # Crash loop detection — RestartCount climbing → notify, critical above limit # Dead containers — remove and restart # Unexpected exits — non-zero exit code → restart # Memory hard limits in MB — immediate restart if exceeded -# Container restarted the moment it crosses this line — no strike system -# 20GB=20480 16GB=16384 12GB=12288 10GB=10240 -# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024 -declare -A WATCHDOG_CONTAINERS=( - ["Emby"]=16384 - ["LidaTube"]=6144 - ["Tdarr"]=6144 - ["Code-Server"]=1024 -) +# 20GB=20480 16GB=16384 12GB=12288 10GB=10240 8GB=8192 4GB=4096 2GB=2048 1GB=1024 + declare -A WATCHDOG_CONTAINERS=( + ["Emby"]=16384 + ["LidaTube"]=6144 + ["Tdarr"]=6144 + ["Code-Server"]=1024 + ) # HTTP health check URLs — checked every cycle, strike system before restart -# Container must respond with HTTP 200 within CURL_TIMEOUT seconds -# Per-host — HOST1 and HOST2 may run different containers on different ports -declare -A HOST1_WATCHDOG_CONTAINER_URLS=( - ["Emby"]="http://localhost:8096" -) + declare -A HOST1_WATCHDOG_CONTAINER_URLS=( + ["Emby"]="http://localhost:8096" + ) -declare -A HOST2_WATCHDOG_CONTAINER_URLS=( - ["Emby"]="http://localhost:8096" -) + declare -A HOST2_WATCHDOG_CONTAINER_URLS=( + ["Emby"]="http://localhost:8096" + ) # Required containers — must always be running -# Strike system: SYS_WATCHDOG_STRIKE_LIMIT strikes before restart attempt -# Persistent skip list: added after WATCHDOG_CONTAINER_RESTART_LIMIT restarts in window -# Skip list auto-clears when container recovers — no manual intervention needed -# Per-host — each server has different critical containers -HOST1_WATCHDOG_REQUIRED_CONTAINERS=( - "NginxProxyManager" - "Lldap-Gmer4Lfe" - "Authelia" - "Mariadb-Authelia" - "Redis-Authelia" - "Authelia-Secondary" - "Redis-Authelia-Secondary" -) +# Strike system before restart, skip list after repeated failures, auto-clears on recovery + HOST1_WATCHDOG_REQUIRED_CONTAINERS=( + "NginxProxyManager" + "Lldap-Gmer4Lfe" + "Authelia" + "Authelia-Secondary" + "Mariadb-Authelia" + "Mariadb-Authelia-Secondary" + "Redis-Authelia" + "Redis-Authelia-Secondary" + ) -HOST2_WATCHDOG_REQUIRED_CONTAINERS=( - "NginxProxyManager" - # add HOST2 required containers here -) + HOST2_WATCHDOG_REQUIRED_CONTAINERS=( + "NginxProxyManager" + # add HOST2 required containers here + ) -# Strike state file — /tmp resets on reboot which is correct -# Fresh start after reboot means no stale strikes carrying over +# Strike state file — /tmp resets on reboot (correct — no stale strikes after reboot) WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" -# CPU thresholds — normalised against total core count automatically at runtime -# SOFT = warn only, HARD = strike toward restart -# CPU_FAIL_LIMIT = consecutive HARD strikes before restart - SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU - HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU - CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart +# CPU thresholds — normalised against total core count at runtime + SOFT_CPU_THRESHOLD=80 + HARD_CPU_THRESHOLD=85 + CPU_FAIL_LIMIT=2 # consecutive hard CPU strikes before container restart -# Memory soft threshold — warn when container reaches this % of its WATCHDOG_CONTAINERS hard limit -# Does not trigger restart — informational only +# Memory soft threshold — warn when container reaches this % of its hard limit SOFT_MEM_THRESHOLD=80 -# HTTP responsiveness — consecutive failed checks before restart -# CURL_TIMEOUT = seconds before curl gives up on a single check - RESP_FAIL_LIMIT=2 # consecutive failed checks before restart - CURL_TIMEOUT=5 # seconds per check before timeout +# HTTP responsiveness + RESP_FAIL_LIMIT=2 # consecutive failed checks before restart + CURL_TIMEOUT=5 # seconds per check -# How often the watchdog runs its checks -# 900 = 15 minutes — long enough to not be noisy, short enough to catch issues quickly -# Containers have this long to recover before next check - DOCKER_WATCHDOG_INTERVAL=900 # seconds between watchdog cycles +# Watchdog cycle interval + DOCKER_WATCHDOG_INTERVAL=900 # seconds between cycles (15 minutes) -# Heartbeat — proof of life logged periodically even when everything is healthy -# Useful to confirm the watchdog is still running without flooding logs - DOCKER_WATCHDOG_HEARTBEAT=true # true = log heartbeat / false = completely silent - DOCKER_WATCHDOG_HEARTBEAT_HOURS=1 # how often to log heartbeat (hours) +# Heartbeat + DOCKER_WATCHDOG_HEARTBEAT=true + DOCKER_WATCHDOG_HEARTBEAT_HOURS=1 -# Tier 2 master toggle — set false to disable global container scanning entirely -# When false only WATCHDOG_CONTAINERS and required containers are monitored - WATCHDOG_SCAN_ALL=true +# Tier 2 master toggle + WATCHDOG_SCAN_ALL=true # false = only WATCHDOG_CONTAINERS + required containers -# Containers to skip in Tier 2 scan entirely -# Useful for containers that legitimately exit/restart frequently -WATCHDOG_SCAN_IGNORE=( - "DashGate" - "PIA-WG-Config-Generator" - "Aperture" - "Aperture-Kids" - "pgvector-18-Apeture-Kids" - "Pgvector18-Aperture" -) +# Containers to skip in Tier 2 scan — legitimately stopped or frequently restarting + WATCHDOG_SCAN_IGNORE=( + "DashGate" + "PIA-WG-Config-Generator" + "Aperture" + "Aperture-Kids" + "pgvector-18-Apeture-Kids" + "Pgvector18-Aperture" + ) -# Individual Tier 2 check toggles — disable specific checks without disabling Tier 2 - WATCHDOG_RESTART_UNHEALTHY=true # restart containers with Docker HEALTHCHECK = unhealthy - WATCHDOG_RESTART_DEAD=true # restart containers in dead state - WATCHDOG_RESTART_CRASHED=true # restart containers that exited with non-zero code - WATCHDOG_NOTIFY_OOM=true # notify + restart OOM killed containers - WATCHDOG_NOTIFY_CRASHLOOP=true # notify when Docker RestartCount keeps climbing +# Individual Tier 2 check toggles + WATCHDOG_RESTART_UNHEALTHY=true + WATCHDOG_RESTART_DEAD=true + WATCHDOG_RESTART_CRASHED=true + WATCHDOG_NOTIFY_OOM=true + WATCHDOG_NOTIFY_CRASHLOOP=true -# Crash loop threshold — notify critical if Docker has restarted this many times total -# Above this number the notification escalates to critical — manual intervention needed +# Crash loop threshold WATCHDOG_CRASH_LIMIT=5 -# Startup grace period — skip restarts while system is still booting after array start -# Prevents watchdog from restarting containers that are legitimately still initializing - WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts on failures +# Startup grace period — skip restarts while system is still booting + WATCHDOG_STARTUP_GRACE=600 # seconds after boot before watchdog acts -# Restart loop protection — stops hammering a broken container -# If watchdog restarts a container more than LIMIT times in WINDOW hours → skip list -# Skip list auto-clears when container recovers healthy - WATCHDOG_CONTAINER_RESTART_LIMIT=3 # max watchdog restarts allowed in window +# Restart loop protection + WATCHDOG_CONTAINER_RESTART_LIMIT=3 WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db" - # rolling restart history for loop detection -# Notification batching — one clean summary per cycle instead of one ping per event -# true = batch all events into one notification at end of cycle -# false = send one notification per event (noisy on busy systems) +# Notification batching — one summary per cycle instead of one ping per event WATCHDOG_BATCH_NOTIFY=true # Dependency ordering — skip restarting a container if its dependency is also down -# Prevents restarting Authelia before its database is ready -# Space-separated list of dependencies per container -declare -A WATCHDOG_DEPENDENCIES=( - ["Authelia"]="Mariadb-Authelia Redis-Authelia" - ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" - ["NextCloud"]="Postgres-NextCloud" -) + declare -A WATCHDOG_DEPENDENCIES=( + ["Authelia"]="Mariadb-Authelia Redis-Authelia" + ["Authelia-Secondary"]="Mariadb-Authelia Redis-Authelia-Secondary" + ["NextCloud"]="Postgres-NextCloud" + ) # ━━━ Docker Network Connect ━━━ -# Connects containers to extra Docker networks on array start via array_start.sh. -# Useful for containers that need their own custom network but also need to be -# reachable from your main custom bridge network. -# Every container in NETWORK_CONNECT_CONTAINERS is connected to every network in -# NETWORK_CONNECT_NETWORKS — containers not found are skipped gracefully. -NETWORK_CONNECT_CONTAINERS=( - "memcached" - "Npm-CrowdSec" -) +# Ensures custom networks exist and connects containers at array start. + NETWORK_CONNECT_CONTAINERS=( + "memcached" + "Npm-CrowdSec" + ) -NETWORK_CONNECT_NETWORKS=( - "high-availability" # must exist before array start — create in Docker settings -) + NETWORK_CONNECT_NETWORKS=( + "high-availability" + ) # ============================================================================================== # ── UNRAID ESSENTIALS ───────────────────────────────────────────────────────────────────────── # ============================================================================================== # ━━━ inotify Tuning ━━━ -# Linux inotify limits — applied at every array start by inotify_tuning.sh +# Linux inotify limits — applied at every array start by inotify_tuning.sh. # Default unRAID values are very low — with many Docker containers watching files -# (Sonarr, Radarr, Lidarr, NextCloud etc.) you can silently exhaust the limit. -# Symptoms: containers miss file events, downloads not detected, library not updated. +# you can silently exhaust the limit causing containers to miss file events. # These settings are lost on reboot — reapplied automatically at array start. INOTIFY_MAX_INSTANCES=1024 # default: 128 — max inotify instances per user INOTIFY_MAX_WATCHES=524288 # default: 8192 — max files watched per instance INOTIFY_MAX_QUEUED_EVENTS=32768 # default: 16384 — max events queued before dropping # ━━━ System Tuning Monitor ━━━ -# Tracks inotify and php-fpm usage over time — read by sunday_morning_coffee_report.sh -# Snapshot written every 6 hours by system_tuning_monitor.sh -# Log bounded to TUNING_LOG_RETENTION days — auto-purges on each write +# Tracks inotify and php-fpm usage over time. +# Snapshot written every 6 hours by system_tuning_monitor.sh. +# Read by sunday_morning_coffee_report.sh for weekly peak/avg/warning summary. INOTIFY_WARN_PCT=80 # warn if inotify instances exceed this % of limit PHP_FPM_WARN_PCT=80 # warn if php-fpm workers exceed this % of max_children TUNING_MONITOR_LOG="$DATA_DIR/system_tuning_history.db" - TUNING_LOG_RETENTION=30 # days — enough for monthly trend visibility + TUNING_LOG_RETENTION=30 # days before old entries are purged # ━━━ Reboot ━━━ # Seconds of warning broadcast to logged-in users before server_reboot.sh reboots. @@ -897,223 +904,206 @@ NETWORK_CONNECT_NETWORKS=( # ============================================================================================== # ━━━ Media Permissions ━━━ -# Applied recursively to all shares in MEDIA_PERMISSION_SHARES by media_shares_permissions.sh. -# Runs first in MEDIA_MANAGEMENT_JOBS — arr cleanup scripts depend on correct ownership. -# 777 mode = read/write/execute for all users — standard for unRAID media shares -# nobody:users = standard unRAID media share ownership +# Applied recursively by media_shares_permissions.sh. +# Runs first in DAILY_MAINTENANCE_SCRIPTS — arr cleanup depends on correct ownership. PERMISSIONS_MODE="777" PERMISSIONS_OWNER="nobody:users" -MEDIA_PERMISSION_SHARES=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Movies-Old - /mnt/user/Anime_Shows - /mnt/user/Anime_Shows-Old - /mnt/user/appcache - /mnt/user/Books - /mnt/user/Downloads - /mnt/user/Games - /mnt/user/Intros - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movie_Recordings - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Music_Videos - /mnt/user/Photo - /mnt/user/Sports - /mnt/user/stand-up_comedy - /mnt/user/Temp_Storage - /mnt/user/Tv_Recordings - /mnt/user/Tv_Shows - /mnt/user/YouTube -) + MEDIA_PERMISSION_SHARES=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Shows + /mnt/user/Anime_Shows-Old + /mnt/user/appcache + /mnt/user/Books + /mnt/user/Downloads + /mnt/user/Games + /mnt/user/Intros + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movie_Recordings + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Music_Videos + /mnt/user/Photo + /mnt/user/Sports + /mnt/user/stand-up_comedy + /mnt/user/Temp_Storage + /mnt/user/Tv_Recordings + /mnt/user/Tv_Shows + /mnt/user/YouTube + ) # ━━━ Media Cleaner ━━━ # Removes junk files from media shares — two profiles: anime and media. -# Called via MEDIA_MANAGEMENT_JOBS. Run manually: Media/media_cleaner.sh anime|media +# Called via DAILY_MAINTENANCE_SCRIPTS. Run manually: Media/media_cleaner.sh anime|media + ANIME_CLEAN_FOLDERS=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Shows + /mnt/user/Anime_Shows-Old + ) -ANIME_CLEAN_FOLDERS=( - /mnt/user/Anime_Movies - /mnt/user/Anime_Movies-Old - /mnt/user/Anime_Shows - /mnt/user/Anime_Shows-Old -) + MEDIA_CLEAN_FOLDERS=( + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Sports + /mnt/user/stand-up_comedy + /mnt/user/Tv_Shows + ) -MEDIA_CLEAN_FOLDERS=( - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movies - /mnt/user/Music - /mnt/user/Sports - /mnt/user/stand-up_comedy - /mnt/user/Tv_Shows -) + ANIME_FILE_PATTERNS=( + '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' + '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' + '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' + '*.log' '*.json' + ) -ANIME_FILE_PATTERNS=( - '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' - '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' - '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' - '*.log' '*.json' -) - -MEDIA_FILE_PATTERNS=( - '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' - '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' - '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' - '*.log' '*.json' '*.iso' '*.lrc' -) + MEDIA_FILE_PATTERNS=( + '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' + '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' + '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' + '*.log' '*.json' '*.iso' '*.lrc' + ) # ━━━ Arr Cleanup ━━━ # Orphan file cleanup via Lidarr, Sonarr, and Radarr APIs. -# Compares tracked file paths from API against disk — deletes untracked files older than ORPHAN_AGE. # detect_hosts() selects correct URL, API key, and root path at runtime. # -# Protected patterns are NEVER deleted — cover art, metadata, subtitles generated by the arr -# are not included in the tracked file API response but must not be deleted. +# API versions: +# Sonarr v4 → /api/v3/series → /api/v3/episodefile?seriesId=X +# Radarr v6 → /api/v3/movie → /api/v3/moviefile?movieId=X +# Lidarr v3 → /api/v1/artist → /api/v1/trackFile?artistId=X # -# API versions and endpoint patterns: -# Sonarr v4 → /api/v3/series (get IDs) → /api/v3/episodefile?seriesId=X per series -# Radarr v5 → /api/v3/movie (get IDs) → /api/v3/moviefile?movieId=X per movie -# Lidarr v3 → /api/v1/artist (get IDs) → /api/v1/trackFile?artistId=X per artist -# All require per-ID loops — bulk endpoints removed in newer versions -# -# Version checking — scripts verify the arr major version matches before running -# If the arr updates and breaks the API the script exits safely before touching files -# Update the MAJOR version here when the script is updated to support a new version -# MINOR = 0 means any minor version within that major is accepted - - SONARR_VERSION_MAJOR=4 # tested major version — script exits if major differs - RADARR_VERSION_MAJOR=6 # tested major version — script exits if major differs - LIDARR_VERSION_MAJOR=3 # tested major version — script exits if major differs +# Version checking — scripts verify major version matches before running. +# Update the MAJOR version here when script is updated to support a new arr version. + SONARR_VERSION_MAJOR=4 + RADARR_VERSION_MAJOR=6 + LIDARR_VERSION_MAJOR=3 # ── Lidarr ──────────────────────────────────────────────────────────────────────────────────── -HOST1_LIDARR_URL="http://192.168.50.2:8686" -HOST1_LIDARR_API_KEY="b2977e71ef074bc0a0529d9fcce3b2dc" -HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New" -LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck + HOST1_LIDARR_URL="http://localhost:8686" + HOST1_LIDARR_API_KEY="your-lidarr-api-key" + HOST1_LIDARR_MUSIC_ROOT="/mnt/user/Music-New" + LIDARR_LOCK_WARN_AGE=3600 # 1hr — large libraries take time, not stuck # Container path → host path translation # Lidarr stores file paths using container paths — script scans host paths # Add one entry per root folder configured in Lidarr Settings → Media Management → Root Folders -declare -A HOST1_LIDARR_PATH_MAP=( - ["/ext-music"]="/mnt/user/Music-New" -) -declare -A HOST2_LIDARR_PATH_MAP=( - # HOST2 does not run Lidarr — fill in if that changes - # ["/ext-music"]="/mnt/user/Music-New" -) + declare -A HOST1_LIDARR_PATH_MAP=( + ["/ext-music"]="/mnt/user/Music-New" + ) + declare -A HOST2_LIDARR_PATH_MAP=( + # HOST2 does not run Lidarr — fill in if that changes + ) -LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion - # protects files that may still be mid-import or recently downloaded -LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") -LIDARR_PROTECTED_PATTERNS=( - # Metadata - "*.nfo" "*.tbn" - # Images — album art, artist images, Emby artwork - "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" - "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" - "banner.*" "thumb.*" "landscape.*" - "folder.*" "cover.*" "album.*" "artist.*" "disc.*" - # Lyrics - "*.lrc" -) - # NEVER deleted — cover art, metadata, lyrics - # Lidarr generates these but doesn't include them in trackFile API - # Without this protection cleanup would delete all your artwork -LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this -LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run - # protects against API returning partial data on a bad day -LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" - # persists last known tracked count for percentage comparison + LIDARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion + LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this + LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run + # protects against API returning partial data on a bad day + LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" + LIDARR_EXTENSIONS=("flac" "mp3" "m4a" "wav" "aac" "ogg" "opus" "wma") + LIDARR_PROTECTED_PATTERNS=( + # Metadata + "*.nfo" "*.tbn" + # Images — album art, artist images, Emby artwork + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + "folder.*" "cover.*" "album.*" "artist.*" "disc.*" + # Lyrics + "*.lrc" + ) # ── Sonarr ──────────────────────────────────────────────────────────────────────────────────── -HOST1_SONARR_URL="http://192.168.50.2:8989" -HOST1_SONARR_API_KEY="130decd3db5b4c25afad64864cd03f9f" -HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" + HOST1_SONARR_URL="http://localhost:8989" + HOST1_SONARR_API_KEY="your-sonarr-api-key" + HOST1_SONARR_TV_ROOT="/mnt/user/Tv_Shows" # Container path → host path translation # Add one entry per root folder configured in Sonarr Settings → Media Management → Root Folders # Note: stand-up_comedy in both Sonarr + Radarr — TV specials and movie specials, one folder -declare -A HOST1_SONARR_PATH_MAP=( - ["/tv"]="/mnt/user/Tv_Shows" - ["/ext-standup-comedy"]="/mnt/user/stand-up_comedy" - ["/kids tv"]="/mnt/user/Kids_Tv_Shows" - ["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old" -) + declare -A HOST1_SONARR_PATH_MAP=( + ["/tv"]="/mnt/user/Tv_Shows" + ["/ext-standup-comedy"]="/mnt/user/stand-up_comedy" + ["/kids tv"]="/mnt/user/Kids_Tv_Shows" + ["/ext-anime-shows"]="/mnt/user/Anime_Shows-Old" + ) -HOST2_SONARR_URL="http://localhost:8989" -HOST2_SONARR_API_KEY="your-host2-sonarr-api-key" -HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows" + HOST2_SONARR_URL="http://localhost:8989" + HOST2_SONARR_API_KEY="your-host2-sonarr-api-key" + HOST2_SONARR_TV_ROOT="/mnt/user/Anime_Shows" -declare -A HOST2_SONARR_PATH_MAP=( - # Fill in when HOST2 is back online - # ["/tv"]="/mnt/user/Anime_Shows" -) + declare -A HOST2_SONARR_PATH_MAP=( + # fill in when HOST2 is back online + # ["/tv"]="/mnt/user/Anime_Shows" + ) -SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion -SONARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this -SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") -SONARR_PROTECTED_PATTERNS=( - # Subtitles - "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" - # Metadata - "*.nfo" "*.tbn" - # Images — cover art, posters, fanart, Emby artwork - "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" - "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" - "banner.*" "thumb.*" "landscape.*" - # Kodi/Emby extras — not tracked by Sonarr API - "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" - "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" - "*-clip.*" "*-other.*" - # Theme songs — stored in show folder, not tracked - "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" -) + SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion + SONARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this + SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") + SONARR_PROTECTED_PATTERNS=( + # Subtitles + "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" + # Metadata + "*.nfo" "*.tbn" + # Images + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + # Kodi/Emby extras — not tracked by Sonarr API + "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" + "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" + "*-clip.*" "*-other.*" + # Theme songs — stored in show folder, not tracked + "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" + ) # ── Radarr ──────────────────────────────────────────────────────────────────────────────────── -HOST1_RADARR_URL="http://192.168.50.2:7878" -HOST1_RADARR_API_KEY="d43a3ec6cf1549edb4af0cc63f98b2a9" -HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" + HOST1_RADARR_URL="http://localhost:7878" + HOST1_RADARR_API_KEY="your-radarr-api-key" + HOST1_RADARR_MOVIES_ROOT="/mnt/user/Movies" # Container path → host path translation # Add one entry per root folder configured in Radarr Settings → Media Management → Root Folders # Note: stand-up_comedy in both Radarr + Sonarr — movie specials and TV specials, one folder -declare -A HOST1_RADARR_PATH_MAP=( - ["/movies"]="/mnt/user/Movies" - ["/kids movies"]="/mnt/user/Kids_Movies" - ["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy" - ["/anime-movies"]="/mnt/user/Anime_Movies-Old" -) + declare -A HOST1_RADARR_PATH_MAP=( + ["/movies"]="/mnt/user/Movies" + ["/kids movies"]="/mnt/user/Kids_Movies" + ["/ext-stand-up-comedy"]="/mnt/user/stand-up_comedy" + ["/anime-movies"]="/mnt/user/Anime_Movies-Old" + ) -HOST2_RADARR_URL="http://localhost:7878" -HOST2_RADARR_API_KEY="your-host2-radarr-api-key" -HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies" + HOST2_RADARR_URL="http://localhost:7878" + HOST2_RADARR_API_KEY="your-host2-radarr-api-key" + HOST2_RADARR_MOVIES_ROOT="/mnt/user/Anime_Movies" -declare -A HOST2_RADARR_PATH_MAP=( - # Fill in when HOST2 is back online - # ["/anime-movies"]="/mnt/user/Anime_Movies" -) + declare -A HOST2_RADARR_PATH_MAP=( + # fill in when HOST2 is back online + # ["/anime-movies"]="/mnt/user/Anime_Movies" + ) -RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion -RADARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this -RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") -RADARR_PROTECTED_PATTERNS=( - # Subtitles - "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" - # Metadata - "*.nfo" "*.tbn" - # Images — cover art, posters, fanart, Emby artwork - "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" - "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" - "banner.*" "thumb.*" "landscape.*" - # Kodi/Emby extras — not tracked by Radarr API - "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" - "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" - "*-clip.*" "*-other.*" - # Theme songs — stored in movie folder, not tracked - "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" -) + RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion + RADARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this + RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") + RADARR_PROTECTED_PATTERNS=( + # Subtitles + "*.srt" "*.sub" "*.ass" "*.ssa" "*.idx" "*.vtt" + # Metadata + "*.nfo" "*.tbn" + # Images + "*.jpg" "*.jpeg" "*.png" "*.webp" "*.svg" + "poster.*" "fanart.*" "backdrop.*" "clearlogo.*" + "banner.*" "thumb.*" "landscape.*" + # Kodi/Emby extras — not tracked by Radarr API + "*-trailer.*" "*-featurette.*" "*-behindthescenes.*" + "*-interview.*" "*-scene.*" "*-short.*" "*-deleted.*" + "*-clip.*" "*-other.*" + # Theme songs — stored in movie folder, not tracked + "theme.mp3" "theme.flac" "theme.wav" "theme.m4a" "theme.mka" + ) # ━━━ Arr Failed/Stalled Recovery ━━━ # Auto blocklist + re-search failed imports and stalled downloads. @@ -1126,36 +1116,24 @@ RADARR_PROTECTED_PATTERNS=( # stalled — download stuck with no connections or progress # # Items newer than ARR_IMPORT_RECOVERY_AGE are skipped — gives arr time to retry first. -# API versions: Sonarr /api/v3/ — Radarr /api/v3/ — Lidarr /api/v1/ # Lidarr runs on HOST1 only — exits cleanly on HOST2. + ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this + # matches cron interval — items eligible after one missed cycle -ARR_IMPORT_RECOVERY_AGE=6 # hours — skip items newer than this - # gives the arr time to retry on its own before we intervene - # matches cron interval — items are eligible after one missed cycle - -# Per-arr enable/disable toggles — set false to temporarily disable without removing from cron -# Useful if an arr is having issues and you want to skip it for a few runs -HOST1_SONARR_RECOVERY=true # Tv_Shows import recovery -HOST1_RADARR_RECOVERY=true # Movies import recovery -HOST1_LIDARR_RECOVERY=true # Music import recovery — HOST1 only, exits cleanly on HOST2 -HOST2_SONARR_RECOVERY=true # Anime_Shows import recovery -HOST2_RADARR_RECOVERY=true # Anime_Movies import recovery + HOST1_SONARR_RECOVERY=true + HOST1_RADARR_RECOVERY=true + HOST1_LIDARR_RECOVERY=true # HOST1 only — exits cleanly on HOST2 + HOST2_SONARR_RECOVERY=true + HOST2_RADARR_RECOVERY=true # ============================================================================================== # ── TRANSCODES ──────────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Session-based storage allocator using filesystem symlink indirection. -# ffmpeg resolves the symlink ONCE at session start — existing sessions are never affected. -# -# How it works: -# ramdisk_setup.sh — creates tmpfs and symlink at array start via array_start.sh -# transcode_management.sh — every 3min, runs cleanup then manager in correct order -# transcode_cleanup.sh — removes old inactive files -# transcode_manager.sh — manages symlink direction based on usage thresholds +# ffmpeg resolves the symlink ONCE at session start — existing sessions never affected. # # ⚠️ Docker mount — must use shared propagation: # --mount type=bind,source=/mnt/ram-transcode,target=/ext-ram-transcode,bind-propagation=shared -# Standard rprivate bind mounts lock the inode — sessions drift to SSD permanently. # ━━━ Transcode Manager ━━━ # tmpfs mount point — created at array start by ramdisk_setup.sh @@ -1163,7 +1141,6 @@ HOST2_RADARR_RECOVERY=true # Anime_Movies import recovery RAMDISK_PATH="/mnt/ramdisk_transcodes" # Ramdisk size ceiling — tmpfs only uses RAM actually needed, not the full size upfront -# Set this to a comfortable limit based on your typical concurrent stream count # Real-world: 9 streams peaked at ~5.5GB — 8G gives comfortable headroom RAMDISK_SIZE="8G" @@ -1173,63 +1150,40 @@ HOST2_RADARR_RECOVERY=true # Anime_Movies import recovery TRANSCODE_LINK="/mnt/ram-transcode" # SSD fallback location — where transcodes land when ramdisk is too full -# Must have enough free space to handle peak session load TRANSCODE_SSD="/mnt/cache/Temp_Storage/Emby/Transcodes/" # Usage thresholds in GB — hysteresis gap between WARN and LOW prevents flip-flop -# RAMDISK_WARN_GB: flip symlink to SSD when ramdisk usage reaches this -# RAMDISK_LOW_GB: flip symlink back to ramdisk when usage drops to this # Gap (6.8 - 5.5 = 1.3GB) means ramdisk must drop 1.3GB before flipping back -# Without hysteresis a session right at the threshold causes rapid flipping - RAMDISK_WARN_GB=6.8 - RAMDISK_LOW_GB=5.5 + RAMDISK_WARN_GB=6.8 # flip to SSD when ramdisk usage reaches this + RAMDISK_LOW_GB=5.5 # flip back to ramdisk when usage drops to this + RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD before allowing flip to SSD -# Minimum free GB on SSD before allowing a flip to SSD -# Prevents flipping to SSD when it's almost full — that would be worse than a full ramdisk - RAMDISK_SSD_MIN_GB=20 + TRANSCODE_MAX_AGE=20 # minutes — HLS segment age before cleanup eligibility + TRANSCODE_ORPHAN_AGE=30 # minutes — files with no matching session at all + TRANSCODE_FLIP_WARN=3 # notify if symlink flips this many times in one hour -# File age thresholds in minutes before cleanup eligibility -# TRANSCODE_MAX_AGE: HLS segment files older than this with no active session = clean up -# TRANSCODE_ORPHAN_AGE: files with no matching session at all = clean up - TRANSCODE_MAX_AGE=20 - TRANSCODE_ORPHAN_AGE=30 - -# Notify if symlink flips this many times in one hour -# Frequent flips indicate the ramdisk is too small or thresholds need adjustment - TRANSCODE_FLIP_WARN=3 - -# Permissions applied to ramdisk and SSD transcode directories TRANSCODE_OWNER="nobody:users" TRANSCODE_CHMOD="755" # Operating mode — controls symlink direction behaviour -# smart — auto-flips between ramdisk and SSD based on RAMDISK_WARN_GB / RAMDISK_LOW_GB -# hysteresis gap prevents flip-flop — default for production -# ramdisk — always uses ramdisk, never flips to SSD -# warns if RAMDISK_WARN_GB exceeded but holds position -# use during SSD maintenance or when SSD space is low +# smart — auto-flips between ramdisk and SSD based on thresholds (default) +# ramdisk — always uses ramdisk, warns if RAMDISK_WARN_GB exceeded but holds # ssd — always uses SSD, never flips to ramdisk -# use during ramdisk maintenance or after a ramdisk issue - TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd + TRANSCODE_MANAGER_MODE="smart" # smart | ramdisk | ssd # Daily statistics log — read by weekly_health_digest.sh for transcode summary -# Tracks peak usage, flip count, session ratio, files cleaned per day -# Bounded to TRANSCODE_LOG_RETENTION days — auto-purges old entries on each write TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" - TRANSCODE_LOG_RETENTION=90 # days before old entries are purged + TRANSCODE_LOG_RETENTION=90 # days before old entries purged # ━━━ Transcode Server Array ━━━ # All media servers sharing the ramdisk transcode space. # Format: "ContainerName|URL|APIKey|Type" — Type: emby | jellyfin | plex # Entries with placeholder API keys are skipped automatically. # ⚠️ Tdarr does NOT belong here — keep Tdarr on SSD. - -TRANSCODE_SERVERS=( - "${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby" - # "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby" - # "Jellyfin|http://localhost:8097|jellyfin-api-key|jellyfin" - # "Plex|http://localhost:32400|plex-token|plex" -) + TRANSCODE_SERVERS=( + "${HOST1_EMBY_CONTAINER}|${HOST1_EMBY_URL}|${HOST1_EMBY_API_KEY}|emby" + # "${HOST2_EMBY_CONTAINER}|${HOST2_EMBY_URL}|${HOST2_EMBY_API_KEY}|emby" + ) TRANSCODE_CHECK_EMBY=true @@ -1243,38 +1197,36 @@ TRANSCODE_SERVERS=( # CERT_WARN_DAYS = notify this many days before expiry # CERT_CRIT_DAYS = escalate to critical this many days before expiry # CERT_TIMEOUT = seconds before giving up on the openssl connection -CERT_MONITOR_DOMAINS=( - "Gmer4Lfe.com" - "Gmer4Lfe.us" -) - CERT_WARN_DAYS=30 # warn when cert expires within this many days - CERT_CRIT_DAYS=7 # critical alert within this many days - CERT_TIMEOUT=10 # seconds per domain check + CERT_MONITOR_DOMAINS=( + "Gmer4Lfe.com" + "Gmer4Lfe.us" + ) + CERT_WARN_DAYS=30 # warn when cert expires within this many days + CERT_CRIT_DAYS=7 # critical alert within this many days + CERT_TIMEOUT=10 # seconds per domain check # ━━━ Backup Verify ━━━ # Verifies rsync mirror health by comparing random file checksums between servers. # Catches silent corruption or incomplete syncs that rsync itself wouldn't detect. # Leave BACKUP_VERIFY_SHARES empty to use HOST*_DAILY_SYNC_SHARES automatically. -# BACKUP_VERIFY_SAMPLE = number of random files to checksum per share -# BACKUP_VERIFY_MIN_SIZE = skip files smaller than this (small files are rarely corrupted) -BACKUP_VERIFY_SHARES=( - # leave empty to use HOST*_DAILY_SYNC_SHARES automatically -) + BACKUP_VERIFY_SHARES=( + # leave empty to use HOST*_DAILY_SYNC_SHARES automatically + ) BACKUP_VERIFY_SAMPLE=10 # random files to check per share BACKUP_VERIFY_MIN_SIZE=1M # minimum file size to include in sample # ━━━ SMART Health ━━━ -# Monitors drive SMART attributes — discovers all drives automatically via /dev/sd* and /dev/nvme*. -# Reads live SMART data — no persistent writes. -# SMART_IGNORE_DRIVES = drives to skip (boot USB, drives without meaningful SMART data) - SMART_TEMP_WARN=45 # Celsius — warn above this temperature - SMART_TEMP_CRIT=55 # Celsius — critical above this temperature -SMART_IGNORE_DRIVES=( - "sda" # boot USB — SMART not meaningful on flash drives -) +# Monitors drive SMART attributes — discovers all drives via /dev/sd* and /dev/nvme*. +# Thresholds read from /boot/config/plugins/dynamix/dynamix.cfg at runtime +# (hot/max/hotssd/maxssd) — these vars are fallback only if dynamix.cfg not found. + SMART_TEMP_WARN=45 # fallback — Celsius warn threshold + SMART_TEMP_CRIT=55 # fallback — Celsius critical threshold + SMART_IGNORE_DRIVES=( + "sda" # boot USB — SMART not meaningful on flash drives + ) # ━━━ ZFS Memory Snapshot ━━━ -# Weekly ZFS pool health and memory diagnostic report — informational only, no action taken. +# Weekly ZFS pool health and memory diagnostic report — informational only. # ZFS_REPORT_ARC_WARN_PCT = warn if ARC is using more than this % of its max # ZFS_REPORT_FREE_WARN_GB = warn if less than this GB free RAM # ZFS_REPORT_AVAIL_WARN_GB = warn if less than this GB available on ZFS pool @@ -1285,37 +1237,34 @@ SMART_IGNORE_DRIVES=( ZFS_REPORT_FREE_WARN_GB=10 ZFS_REPORT_AVAIL_WARN_GB=20 ZFS_REPORT_DOCKER_TOP=10 -ZFS_REPORT_IGNORE_POOLS=( - "disk10" - "disk9" - "disk8" - "disk6" - "disk5" -) + ZFS_REPORT_IGNORE_POOLS=( + "disk5" + "disk6" + "disk8" + "disk9" + "disk10" + ) # ━━━ Bandwidth Monitor ━━━ # Called automatically by rsync.sh after each sync — one bounded write per run. # Tracks transfer size, duration and profile per sync for weekly summary reporting. -# BANDWIDTH_LOG_RETENTION = days to keep entries before auto-purging old records -# BANDWIDTH_WARN_GB = flag in weekly summary if a single sync exceeded this size BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db" - BANDWIDTH_LOG_RETENTION=90 # days before old entries are purged - BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report + BANDWIDTH_LOG_RETENTION=90 # days before old entries purged + BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report # Stats files — written by cleanup and recovery scripts, read by coffee report # All in DATA_DIR — array always running when these are written - ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats - ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats + ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats + ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats # ━━━ Health Digest ━━━ # Aggregated system health summary — reads existing state files, no new writes. -# Three profiles control when the digest email is sent: +# Three profiles control when the digest is sent: # always — sends every run regardless of findings # smart — sends only when DIGEST_SMART_ON_* conditions are found # weekly — sends once per week on DIGEST_DAY only -# Smart profile triggers — set true to send digest when finding is detected: DIGEST_PROFILE="weekly" # always | smart | weekly - DIGEST_DAY="Sunday" # day of week for weekly profile + DIGEST_DAY="Sunday" DIGEST_SMART_ON_WATCHDOG=true # send if any watchdog strikes are active DIGEST_SMART_ON_FAILOVER=true # send if failover state is not NORMAL DIGEST_SMART_ON_CERT_WARN=true # send if any cert is under CERT_WARN_DAYS @@ -1323,9 +1272,7 @@ ZFS_REPORT_IGNORE_POOLS=( # ━━━ Emby Session Report ━━━ # Weekly Emby usage statistics via API — no persistent writes, queries fresh each run. -# Shows top content, most active users, session counts over the report period. -# URL and API key pulled from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY -# defined in Host Configuration at the top of this file — no duplication needed. +# URL and API key pulled from HOST1/HOST2_EMBY_URL and HOST1/HOST2_EMBY_API_KEY. EMBY_REPORT_DAYS=7 # days to include in the report period EMBY_REPORT_TOP_N=10 # number of top content items to show @@ -1337,12 +1284,11 @@ ZFS_REPORT_IGNORE_POOLS=( # Re-sources Master.conf each cycle — config changes take effect on next cycle. # Strike system: sustained threshold hits trigger reboot — single spikes ignored. # Reboot loop protection: shuts down instead if reboot limit hit in rolling window. -# Silent when healthy — logs only when a threshold is triggered. # ━━━ State Files ━━━ SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" # /tmp resets on reboot ✅ - SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" # survives reboots - SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" # reboot loop detection + SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" + SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" # ━━━ Strike and Reboot Loop Settings ━━━ # Strike system: a check must fail this many consecutive cycles before action is taken @@ -1351,69 +1297,60 @@ ZFS_REPORT_IGNORE_POOLS=( # How often checks run — 300s = 5 minutes # At STRIKE_LIMIT=2 and INTERVAL=300: problem must persist 10min before reboot - SYSTEM_WATCHDOG_INTERVAL=300 # seconds between watchdog cycles + SYSTEM_WATCHDOG_INTERVAL=300 # Reboot loop protection — if system keeps rebooting something is seriously wrong -# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS hours → shutdown instead of reboot -# Prevents infinite reboot loops when the underlying problem can't be fixed by rebooting - SYS_WATCHDOG_REBOOT_LIMIT=3 # max reboots before shutdown instead - SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # rolling window in hours +# After REBOOT_LIMIT reboots in REBOOT_WINDOW_HRS → shutdown instead of reboot + SYS_WATCHDOG_REBOOT_LIMIT=3 + SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 # Heartbeat — proof of life logged periodically even when everything is healthy - SYSTEM_WATCHDOG_HEARTBEAT=true # true = log heartbeat / false = completely silent - SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1 # how often to log heartbeat (hours) + SYSTEM_WATCHDOG_HEARTBEAT=true + SYSTEM_WATCHDOG_HEARTBEAT_HOURS=1 # ━━━ Thresholds ━━━ # Set at "about to become unstable" levels — not "things are a bit high" # These should be high enough that normal operation never triggers them -# rootfs (/) usage percentage — when array is down rsync writes land on rootfs -# fills rapidly and can crash the server — 95% is almost too late, act fast +# rootfs (/) usage — when array is down rsync writes land on rootfs, fills rapidly SYS_WATCHDOG_ROOTFS_PCT=95 -# /var/log usage percentage — log spam can fill rootfs, indicates something broken +# /var/log usage — log spam can fill rootfs, indicates something broken SYS_WATCHDOG_LOG_PCT=95 # Free RAM in GB — below this is critically low, OOM or swap imminent -# Your server has 128GB — 4GB free means something is consuming everything SYS_WATCHDOG_MEM_GB=4 # ZFS ARC pinned percentage — ARC not releasing after reclaim = memory stuck -# SYS_WATCHDOG_ARC_RELEASE_PCT = after reclaim attempt, if still above this → trigger SYS_WATCHDOG_ARC_PINNED_PCT=98 SYS_WATCHDOG_ARC_RELEASE_PCT=95 # Load average multiplier — threshold = MULTIPLIER × CPU core count # MULTIPLIER=3 on 16-core = load average of 48 before triggering -# Set high — transcoding causes legitimate high load spikes SYS_WATCHDOG_LOAD_MULTIPLIER=3 # Zombie process count — large numbers indicate serious process management failure -# A few zombies are normal — 50 means something is very wrong SYS_WATCHDOG_ZOMBIE_LIMIT=50 -# CPU temperature in Celsius — sustained high temp causes throttling or kernel panic -# 95°C is close to tjmax on most CPUs — triggers before thermal shutdown +# CPU temperature — sustained high temp causes throttling or kernel panic SYS_WATCHDOG_CPU_TEMP_MAX=95 # ━━━ Check Toggles ━━━ # Disable individual checks without disabling the whole watchdog -# All enabled by default except load — transcoding causes legitimate load spikes SYS_WATCHDOG_CHECK_ROOTFS=true SYS_WATCHDOG_CHECK_LOG=true SYS_WATCHDOG_CHECK_RAM=true SYS_WATCHDOG_CHECK_ARC=true SYS_WATCHDOG_CHECK_CPU_TEMP=true - SYS_WATCHDOG_CHECK_LOAD=false # disabled — load spikes during transcoding are normal + SYS_WATCHDOG_CHECK_LOAD=false # disabled — load spikes during transcoding are normal SYS_WATCHDOG_CHECK_ZOMBIES=true - SYS_WATCHDOG_CHECK_CONTAINERS=true # checks docker_watchdog persistent skip list - SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true # checks if Docker daemon is responding + SYS_WATCHDOG_CHECK_CONTAINERS=true # checks docker_watchdog persistent skip list + SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true # ━━━ Abort Toggles ━━━ # Conditions that prevent reboot even when a threshold is hit # true = abort reboot if this condition is active (conservative — avoid data loss) # false = reboot anyway (aggressive — a clean reboot beats a hard crash) -# Philosophy: aborting is safer for data, rebooting is safer for stability SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # rebooting with bad pool risks data loss SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity is better than crashing mid-check SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting move is better than crashing mid-move diff --git a/Orchestrators/critical_sync_maintenance.sh b/Orchestrators/critical_sync_maintenance.sh new file mode 100644 index 0000000..8ea6252 --- /dev/null +++ b/Orchestrators/critical_sync_maintenance.sh @@ -0,0 +1,172 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# ----------------------------- Critical Sync Maintenance -------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Orchestrator for time-sensitive syncs that run every 15 minutes. +# Keeps the mirror current between the less frequent daily and weekly windows. +# Schedule: */15 * * * * (every 15 minutes) +# +# Execution order: +# 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides) +# 2. emby-failover rsync — dirty Emby sync (watch states, library — Emby stays running) +# 3. partnership --check — read both state files, detect changes, act accordingly +# +# Why every 15 minutes: +# Auth stack changes (new users, proxy rules, certs) propagate within 15min ✅ +# Emby watch states stay in sync — mirror users see correct playback position ✅ +# Partnership state changes detected and acted on quickly ✅ +# +# Rsync gate: +# RSYNC_ENABLED=false → skips all syncs (global gate) +# CRITICAL_RSYNC_ENABLED=false → skips critical syncs (per-orchestrator) +# partnership --check still runs regardless of rsync gate +# (state check doesn't need rsync to work) +# +# Lock behavior: +# acquire_lock "strict" — if previous 15min run still going, skip this cycle +# Critical-Data taking > 15min is a problem worth knowing about +# Lock prevents pile-up ✅ +# +# Configuration in Master.conf: +# CRITICAL_RSYNC_ENABLED — enable/disable rsync section +# PARTNERSHIP_ENABLED — enable/disable partnership check +# CRITICAL_SYNC_SHARES — shares synced every 15min +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +acquire_lock "strict" + +detect_hosts +resolve_remote_ip + +START=$(date +%s) +RSYNC_OK=false +PASS=() +FAIL=() + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SYNC Critical Shares Sync ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_SYNC Critical Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + +if ! check_rsync_enabled "CRITICAL"; then + warn "Critical rsync disabled — skipping sync, running partnership check only" +else + for share in "${CRITICAL_SYNC_SHARES[@]:-}"; do + [[ -z "$share" ]] && continue + + # Parse optional profile flag: "/path/to/share|profile-name" + SHARE_PATH="${share%%|*}" + SHARE_PROFILE="${share##*|}" + SHARE_NAME=$(basename "$SHARE_PATH") + + echo "" + echo "━━━ $ICON_SYNC $SHARE_NAME ━━━" + + SHARE_START=$(date +%s) + + if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then + # No profile specified + bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" + else + bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" --profile="$SHARE_PROFILE" + fi + + RSYNC_EXIT=$? + SHARE_END=$(date +%s) + SHARE_DUR=$(format_duration $(( SHARE_END - SHARE_START ))) + + if [[ "$RSYNC_EXIT" -eq 0 ]]; then + PASS+=("$SHARE_NAME") + success "$SHARE_NAME — done in $SHARE_DUR ✅" + RSYNC_OK=true + else + FAIL+=("$SHARE_NAME") + error "$SHARE_NAME — failed after $SHARE_DUR" + fi + done +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_CLEAN Critical Maintenance Scripts ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_CLEAN Critical Maintenance — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + +if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then + log "No CRITICAL_MAINTENANCE_SCRIPTS defined — skipping" +else + for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do + [[ -z "$script_entry" ]] && continue + # Strip leading comment lines + [[ "$script_entry" == \#* ]] && continue + + SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}" + SCRIPT_ARGS="${script_entry#* }" + [[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS="" + + SCRIPT_NAME=$(basename "$SCRIPT_PATH") + echo " → $SCRIPT_NAME" + + if [[ ! -f "$SCRIPT_PATH" ]]; then + warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping" + continue + fi + + bash "$SCRIPT_PATH" $SCRIPT_ARGS + EXIT_CODE=$? + if [[ "$EXIT_CODE" -ne 0 ]]; then + warn "$SCRIPT_NAME exited with code $EXIT_CODE" + fi + done +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SHIELD Partnership Check ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_SHIELD Partnership Check ━━━" + +if [[ "${PARTNERSHIP_ENABLED:-false}" == false ]]; then + log "Partnership disabled — skipping check" +else + # Pass rsync outcome to --check so it can update last_seen_remote + if [[ "$RSYNC_OK" == true ]]; then + bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-seen + else + bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-unseen + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +END=$(date +%s) + +echo "" +echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" + +if [[ ${#PASS[@]} -gt 0 ]]; then + echo "$ICON_SUCCESS Synced: ${PASS[*]}" +fi +if [[ ${#FAIL[@]} -gt 0 ]]; then + echo "$ICON_ERROR Failed: ${FAIL[*]}" + notify "Critical sync failed on $(hostname) — ${FAIL[*]}" "Critical Sync" "warning" +fi +if [[ ${#PASS[@]} -eq 0 ]] && [[ ${#FAIL[@]} -eq 0 ]]; then + echo "$ICON_SKIP Rsync: disabled" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Orchestrators/daily_sync_maintenance.sh b/Orchestrators/daily_sync_maintenance.sh index e4f2688..7de06d9 100644 --- a/Orchestrators/daily_sync_maintenance.sh +++ b/Orchestrators/daily_sync_maintenance.sh @@ -23,7 +23,7 @@ # # Configuration in Master.conf: # DAILY_MAINTENANCE_SCRIPTS — pre/post-sync scripts (git pull, docker restart) -# MEDIA_MANAGEMENT_JOBS — media maintenance jobs run after sync +# DAILY_MAINTENANCE_SCRIPTS — media maintenance jobs run after sync # HOST1_DAILY_SYNC_SHARES — shares HOST1 pushes to HOST2 # HOST2_DAILY_SYNC_SHARES — shares HOST2 pushes to HOST1 # HOST1/2_PERSONAL_SHARES — encrypted personal shares @@ -201,14 +201,14 @@ TOTAL_DURATION=$((TOTAL_END - TOTAL_START)) # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_CLEAN Post-sync Media Jobs ━━━ -# Reads MEDIA_MANAGEMENT_JOBS from Master.conf — permissions, cleaners, arr cleanup +# Reads DAILY_MAINTENANCE_SCRIPTS from Master.conf — permissions, cleaners, arr cleanup # Runs after sync completes — correct ownership available, clean folders guaranteed # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_CLEAN Post-sync Media Jobs ━━━" -if [[ ${#MEDIA_MANAGEMENT_JOBS[@]} -gt 0 ]]; then - for script_entry in "${MEDIA_MANAGEMENT_JOBS[@]}"; do +if [[ ${#DAILY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then + for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do [[ -z "$script_entry" ]] && continue script_args=($script_entry) script_path="$SCRIPTS_ROOT/${script_args[0]}" diff --git a/Orchestrators/weekly_sync_maintenance.sh b/Orchestrators/weekly_sync_maintenance.sh index 5560e71..f2cffe7 100644 --- a/Orchestrators/weekly_sync_maintenance.sh +++ b/Orchestrators/weekly_sync_maintenance.sh @@ -8,15 +8,15 @@ # Execution order: # 1. Stop local containers — Emby + auth stack stopped locally # 2. Stop remote containers — Emby + auth stack stopped remotely via SSH -# 3. Pull updates locally — if CRITICAL_SYNC_UPDATES=true -# 4. Pull updates remotely — if CRITICAL_SYNC_UPDATES_REMOTE=true +# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true +# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true # 5. rsync Emby — full clean mirror, both instances stopped # 6. rsync Critical-Data — auth stack clean sync, databases flushed # 7. Start remote containers — correct order, delayed start respected # 8. Start local containers — correct order, delayed start respected # 9. docker_weekly_restart.sh — weekly container restarts # -# Synced shares (WEEKLY_SYNC_JOBS in Master.conf): +# Synced shares (WEEKLY_SYNC_SHARES in Master.conf): # /mnt/user/Media_Server/Emby — emby profile — full mirror, cache resets weekly # /mnt/user/appdata-Failover/Critical-Data — critical-data — auth stack clean state # @@ -29,16 +29,16 @@ # Container updates during the window: # Containers already stopped for sync — updates pull at zero extra downtime # Both servers start on identical image versions after the window completes -# Toggle: CRITICAL_SYNC_UPDATES / CRITICAL_SYNC_UPDATES_REMOTE in Master.conf +# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in Master.conf # # What triggers weekly_health_digest.sh: # NOT this script — weekly_health_digest.sh runs on its own Saturday schedule # # Configuration in Master.conf: -# WEEKLY_SYNC_JOBS — shares synced during the maintenance window +# WEEKLY_SYNC_SHARES — shares synced during the maintenance window # WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync (docker_weekly_restart) -# CRITICAL_SYNC_UPDATES — toggle container updates on/off -# CRITICAL_SYNC_UPDATES_REMOTE — toggle remote container updates on/off +# WEEKLY_SYNC_UPDATES — toggle container updates on/off +# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates on/off # ----------------------------------------------------------------------------------------------- # All configuration in Master.conf. # Supports --dry-run to walk through without stopping containers, syncing, or updating. @@ -107,7 +107,7 @@ fi echo "" echo "━━━ $ICON_GEAR Container Updates ━━━" -if [[ "$CRITICAL_SYNC_UPDATES" == true ]]; then +if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would pull updates for local containers" for c in "${MAINTENANCE_CONTAINERS[@]}"; do @@ -133,10 +133,10 @@ if [[ "$CRITICAL_SYNC_UPDATES" == true ]]; then done fi else - info "CRITICAL_SYNC_UPDATES=false — skipping local updates" + info "WEEKLY_SYNC_UPDATES=false — skipping local updates" fi -if [[ "$CRITICAL_SYNC_UPDATES_REMOTE" == true ]]; then +if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME" else @@ -158,7 +158,7 @@ if [[ "$CRITICAL_SYNC_UPDATES_REMOTE" == true ]]; then done fi else - info "CRITICAL_SYNC_UPDATES_REMOTE=false — skipping remote updates" + info "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates" fi # ----------------------------------------------------------------------------------------------- @@ -168,7 +168,7 @@ PASS=() FAIL=() TOTAL_START=$(date +%s) -SYNC_JOBS=("${WEEKLY_SYNC_JOBS[@]}") +SYNC_JOBS=("${WEEKLY_SYNC_SHARES[@]}") SHARE_COUNT=${#SYNC_JOBS[@]} echo "" @@ -280,7 +280,7 @@ WINDOW_END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━" echo "$ICON_TIME Duration: $TOTAL_DURATION" -echo "$ICON_GEAR Updates: local=$CRITICAL_SYNC_UPDATES remote=$CRITICAL_SYNC_UPDATES_REMOTE" +echo "$ICON_GEAR Updates: local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE" echo "" echo "$ICON_SYNC Sync jobs:" @@ -307,7 +307,7 @@ if [[ "$DRY_RUN" == true ]]; then echo "$ICON_WARN Status: DRY RUN — no changes made" elif [[ "$TOTAL_FAIL" -eq 0 ]]; then echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE" - notify "Weekly sync maintenance complete on $(hostname) — synced + updated (local=$CRITICAL_SYNC_UPDATES remote=$CRITICAL_SYNC_UPDATES_REMOTE)" "Weekly Maintenance" "normal" + notify "Weekly sync maintenance complete on $(hostname) — synced + updated (local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE)" "Weekly Maintenance" "normal" else echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs" notify "Weekly sync maintenance failed on $(hostname) — sync: ${#FAIL[@]} failed, jobs: ${#JOB_FAIL[@]} failed" "Weekly Maintenance" "warning" diff --git a/Partnership/README-Partnership.md b/Partnership/README-Partnership.md new file mode 100644 index 0000000..5cadc9b --- /dev/null +++ b/Partnership/README-Partnership.md @@ -0,0 +1,438 @@ +# Partnership + +Manages the relationship lifecycle between two unRAID servers. Handles onboarding a new mirror, clean separation when someone wants to leave, and transferring ownership when the arrangement changes. + +> This script exists because the partnership between two servers has a lifecycle — and that lifecycle deserves the same engineering discipline as everything else in the ecosystem. A clean exit should be as easy as a clean setup. + +--- + +## The Relationship Model + +Two servers. One owns the shared services. One mirrors them. + +``` +HOST1 (owner) — source of truth + Auth stack ← all changes made here + NPM proxy rules ← created here, mirrored to mirror + Certs ← renewed here, mirrored to mirror + Emby ← runs here, dirty-synced every 15min + Master.conf ← source of truth, git push propagates + +HOST2 (mirror) — warm copy + Auth containers ← running, serving his domain + NPM ← running owner's config + Certs ← current, mirrored every 15min + WebUIs ← click any auth container → opens owner's WebUI via Tailscale + Never makes changes directly ← overwritten on next sync +``` + +**HOST1 is always the owner. HOST2 is always the mirror.** +`PARTNERSHIP_OWNER_HOST="HOST1"` in Master.conf — flips to `"HOST2"` only after `--transfer`. +Everything derives from HOST1/HOST2 — no duplicate IP or key configuration needed. + +**The mirror never needs to think about auth management.** His Docker UI opens, he clicks an auth container, lands on the owner's WebUI automatically via Tailscale. Changes made there sync to his server in 15 minutes. He benefits from every improvement the owner makes without doing anything. + +**The owner never needs to think about the mirror's server.** Everything propagates automatically. Critical-Data syncs every 15 minutes via `critical_sync_maintenance.sh`. Certs stay current. Config stays consistent. + +--- + +## What Makes It Work + +Three things make the mirror transparent to end users: + +**1. Auth stack warm on both servers** + +NPM, LLDAP, Authelia, certs — all running on both servers simultaneously. When traffic hits the mirror's domain, auth is already ready. No cold start, no broken auth window. + +**2. WebUIs pointing to owner** + +Every auth container on the mirror has its WebUI URL configured to point at the owner's Tailscale IP. From the mirror operator's perspective: click container, get owner's UI. From the owner's perspective: one place to manage everything. + +**3. Tailscale-only communication** + +All traffic between servers goes through Tailscale. Encrypted, no open ports, no VPN configuration. The WebUI redirect works because Tailscale keeps both servers permanently connected. HOST1 and HOST2 hostnames must match their exact Tailscale device names — already enforced by the ecosystem. + +--- + +## State Files + +Partnership state is tracked in two files on `/boot/config` — survives reboots, available before the array starts, minimal flash wear. + +``` +/boot/config/partnership_HOST1.db ← HOST1 writes only +/boot/config/partnership_HOST2.db ← HOST2 writes only +``` + +Each server writes **only its own file**. State propagates via SSH — no rsync needed. `critical_sync_maintenance.sh` calls `partnership_manage.sh --check` every 15 minutes, which SSHes to read the remote file and act on any changes. + +**This is how deferred offboard works:** HOST2 offboards while HOST1 is temporarily unreachable → HOST2 writes its state file → next time HOST1 can reach HOST2 → reads HOST2's state → finalises from its side automatically. + +A per-server offline counter (`/boot/config/partnership_offline_days.db`) tracks consecutive missed 15-minute cycles. After `PARTNERSHIP_OFFLINE_THRESHOLD` days either server is unreachable, both independently auto-offboard. + +--- + +## The Lifecycle + +### Onboard — Setting Up a New Mirror + +Run `--onboard` from HOST1 (owner) to establish the relationship: + +```bash +# Always dry-run first +partnership_manage.sh --onboard --dry-run + +# Live onboard +partnership_manage.sh --onboard +``` + +**What happens:** +1. Pre-flight — both servers healthy, Tailscale connected +2. SSH to HOST2 — reconfigure each auth container's WebUI URL to HOST1's Tailscale IP +3. Verify WebUI connectivity — curl each WebUI, confirm reachable +4. Write state files — `partnership_HOST1.db=ACTIVE`, push to HOST2 +5. Reset offline counter +6. Notify both servers — partnership active + +**After onboard:** +``` +Mirror opens Docker UI +→ clicks NginxProxyManager +→ lands on owner's NPM WebUI +→ makes changes there +→ 15min later synced to his server +→ he never touched his own NPM +``` + +--- + +### Offboard — Clean Separation + +**Either server can initiate offboard.** The process differs slightly depending on who runs it. + +```bash +# Always dry-run first +partnership_manage.sh --offboard --dry-run + +# Live offboard +partnership_manage.sh --offboard +``` + +#### HOST1 (owner) initiates offboard: + +1. 10 second countdown +2. Stop any running rsync (`rsync_stop.sh --rsync-only`) +3. Final sync — Critical-Data and Emby pushed one last time +4. SSH to HOST2 — reconfigure auth WebUIs back to localhost +5. Disable `CRITICAL_RSYNC_ENABLED=false` in Master.conf +6. Write `partnership_HOST1.db=INACTIVE`, push to HOST2 +7. Wait `PARTNERSHIP_GRACE_HOURS` (6hr) — mirror has access to collect anything +8. Remove HOST2 from Tailscale tailnet +9. Notify HOST2 + +#### HOST2 (mirror) initiates offboard: + +1. Reconfigure own auth WebUIs → localhost +2. Write `partnership_HOST2.db=INACTIVE` +3. SSH to HOST1 — push state file (or write locally if HOST1 unreachable) +4. Notify HOST1 — "mirror has requested offboard" + +**HOST1 finalises on next `--check` cycle:** + +5. Reads HOST2's state file — sees INACTIVE +6. Runs final sync — HOST2 leaves with current state +7. Disables critical sync +8. Writes `partnership_HOST1.db=INACTIVE` +9. Waits grace period → removes HOST2 from Tailscale + +#### What HOST2 leaves with: + +``` +Current auth config ✓ — final sync was clean +His own DDNS + updater ✓ — always was his, unaffected +His own Git mirror ✓ — full ecosystem, always mirrored +His anime source of truth ✓ — always was his +Auth WebUIs → localhost ✓ — managing his own auth now +Full independence ✓ — just stop the sync +Certs valid ~60-90 days ✓ — cert_monitor.sh warns at 30 days +``` + +**Tailscale access and backup window both expire at the same time** (`PARTNERSHIP_GRACE_HOURS=6`). Keeping backups accessible beyond Tailscale removal is meaningless — they expire together by design. + +--- + +### Auto-Offboard — 30 Day Offline Threshold + +If either server is unreachable for `PARTNERSHIP_OFFLINE_THRESHOLD` days (default: 30), the other server auto-offboards. Works both directions: + +``` +HOST1 sees HOST2 offline 30 days: + AM_OWNER → full offboard from owner side + Tailscale removal ✅ (API call works regardless) + +HOST2 sees HOST1 offline 30 days: + AM_MIRROR → mirror offboard + Reconfigures own WebUIs → localhost ✅ + Writes HOST2.db=INACTIVE ✅ + Fully independent immediately ✅ + When HOST1 comes back → sees INACTIVE → cleans its side ✅ +``` + +This handles the "partner disappeared" scenario without manual intervention. 30 days is long enough to cover extended outages, short enough that you're not waiting forever for resolution. + +--- + +### Transfer — Flipping Ownership + +Run `--transfer` from HOST1 when ownership needs to change: + +```bash +# Always dry-run first — this is significant +partnership_manage.sh --transfer --dry-run + +# Live transfer — requires explicit confirmation string +partnership_manage.sh --transfer --confirm=i-understand-this-transfers-ownership +``` + +**The confirmation string is long and specific by design.** You cannot type it accidentally. + +**What happens:** +1. Display current and future ownership clearly +2. Confirmation string check +3. Health strike system — both servers must pass `PARTNERSHIP_TRANSFER_STRIKES` consecutive checks (max `PARTNERSHIP_TRANSFER_MAX_ATTEMPTS` attempts before giving up) +4. Final sync in current direction — new mirror leaves with current state +5. Reconfigure HOST2 WebUIs → new owner's Tailscale IP +6. Reconfigure HOST1 WebUIs → localhost (now manages directly) +7. Flip `PARTNERSHIP_OWNER_HOST` in Master.conf on both servers +8. Write updated state files +9. Notify both servers + +**After transfer:** +``` +Before: HOST1 = owner, HOST2 = mirror +After: HOST2 = owner, HOST1 = mirror + +HOST2 now: + Makes all auth changes ← source of truth + Pushes Critical-Data sync ← every 15min + Manages NPM, certs, LLDAP ← directly + +HOST1 now: + Receives sync ← warm mirror + WebUIs point to HOST2 ← transparent + Never makes changes directly +``` + +--- + +### Check — Automated State Monitor + +Called automatically by `critical_sync_maintenance.sh` every 15 minutes. Never run manually. + +```bash +# Called internally — do not run manually +partnership_manage.sh --check --remote-seen # rsync succeeded, HOST2 reachable +partnership_manage.sh --check --remote-unseen # rsync failed, HOST2 unreachable +``` + +**What it does each cycle:** +- `--remote-seen`: resets offline counter, updates `last_seen_remote` timestamp +- `--remote-unseen`: increments offline counter, checks auto-offboard threshold +- SSHes to remote, reads remote state file +- Both ACTIVE → silent, healthy ✅ +- Remote INACTIVE → owner finalises offboard, mirror cleans up +- Threshold exceeded → auto-offboard + +--- + +## Role-Based Access + +``` +HOST1 (owner): --onboard ✅ --offboard ✅ --transfer ✅ --status ✅ +HOST2 (mirror): --onboard ❌ --offboard ✅ --transfer ❌ --status ✅ +``` + +HOST2 is blocked from onboard and transfer by design — ownership is granted not taken. Either server can initiate offboard — clean exit is always available to both parties. + +--- + +## Configuration + +All configuration in `Master.conf` under `── PARTNERSHIP ──`, placed immediately after HOST CONFIGURATION. + +```bash +PARTNERSHIP_ENABLED=false +PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer + +# Auth containers reconfigured on onboard/offboard +# Format: "ContainerName|WebUIPort" +PARTNERSHIP_AUTH_WEBUIS=( + "NginxProxyManager|81" + "Lldap-Gmer4Lfe|17170" + "Authelia|9091" + "Authelia-Secondary|9092" +) + +# Paths on owner that mirror should collect during grace window +PARTNERSHIP_MIRROR_BACKUPS=( + # "/mnt/user/appdata-Failover/Jayred365-Emby" +) + +# Timing +PARTNERSHIP_GRACE_HOURS=6 # hours after offboard before Tailscale removal + # backup access also expires at this time +PARTNERSHIP_OFFLINE_THRESHOLD=30 # days unreachable before auto-offboard (both directions) +PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale on offboard + +# Tailscale API — required for PARTNERSHIP_REMOVE_TAILSCALE=true +TAILSCALE_API_KEY="" # tskey-api-... +TAILSCALE_TAILNET="" # your tailnet name + +# Transfer safety +PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership" +PARTNERSHIP_TRANSFER_STRIKES=3 +PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 + +# Onboard +PARTNERSHIP_ONBOARD_VERIFY=true +PARTNERSHIP_ONBOARD_NOTIFY=true +PARTNERSHIP_SYNC_INTERVAL=15 # minutes — informational, actual schedule in cron +``` + +--- + +## critical_sync_maintenance.sh + +The partnership check runs inside `critical_sync_maintenance.sh` — the 15-minute orchestrator that also handles Critical-Data and Emby failover syncs. + +``` +Orchestrators/critical_sync_maintenance.sh +Schedule: */15 * * * * + +Execution order: + 1. Critical-Data rsync ← auth stack sync (15min) + 2. emby-failover rsync ← dirty Emby sync (15min) + 3. partnership --check ← state check, informed by rsync outcome +``` + +The rsync outcome directly informs the partnership check — if rsync succeeded, HOST2 was reachable and the offline counter resets. If rsync failed, the counter increments. + +Controlled by `CRITICAL_RSYNC_ENABLED` in Master.conf — set to `false` automatically on offboard. + +--- + +## Initial Setup Requirements + +Before `--onboard` can run: + +**1. Tailscale connected on both servers** + +HOST1 and HOST2 hostnames must match their exact Tailscale device names — already enforced by the ecosystem since `HOST1` and `HOST2` in Master.conf are used for Tailscale IP resolution everywhere. + +**2. SSH keys configured — passwordless both directions** + +```bash +ssh -i /root/.ssh/your_key root@[HOST2-tailscale-ip] "hostname" +``` + +**3. Auth containers exist on HOST2** + +Containers must exist before onboard. They can be stopped — onboard only reconfigures WebUI URLs. Critical-Data sync brings them live config. + +**4. Critical-Data rsync profile configured** + +The `critical-data` rsync profile must be set up in Master.conf. This is what keeps the mirror current after onboard. Configured in `CRITICAL_SYNC_SHARES`. + +**5. Tailscale API key configured** (if `PARTNERSHIP_REMOVE_TAILSCALE=true`) + +``` +https://login.tailscale.com/admin/settings/keys +Scope: Devices write +``` + +--- + +## Independence — Always One Stop Away + +The partnership is designed with a clean exit built in from day one. + +**To stop the relationship — from either server:** +```bash +partnership_manage.sh --offboard +``` + +That's it. The mirror's containers keep running. His domain keeps working. His DDNS keeps pointing to him. His git repo has the full ecosystem. His certs run for another 60-90 days. + +**What the owner keeps after separation:** +``` +His hardware ← always was his +His media shares ← his source of truth shares stay his +His services ← unaffected +His domain ← unaffected +His Emby users ← continue as before +``` + +**What the owner loses:** +``` +Offsite backup ← was the mirror's hardware +Geographic redundancy ← can't script that, it was the person +``` + +The arrangement was always mutual. Both parties contributed something the scripts couldn't provide — hardware in a different location on a different power utility. That's irreplaceable. Everything else in the ecosystem can be reconfigured. + +--- + +## Troubleshooting + +**WebUI still pointing to old IP after onboard:** +- Check template was found: `partnership_manage.sh --status` +- Some containers may need a restart to pick up the new WebUI URL +- Verify with `--status` on both servers + +**Transfer health checks failing:** +- Both arrays must be fully started +- Both Docker daemons must be responding +- Tailscale must be connected on both servers +- Max attempts: `PARTNERSHIP_TRANSFER_MAX_ATTEMPTS` (default 20) before giving up + +**Mirror's domain broken after offboard:** +- Auth WebUIs reconfigured to localhost — mirror manages auth directly now +- Verify auth containers are running: `docker ps` +- Check cert expiry: run `cert_monitor.sh` + +**State mismatch between servers:** +- Run `--status` on both servers to see both state files +- If one shows ACTIVE and other INACTIVE — run `--offboard` to resolve +- If HOST2 unreachable — HOST2 will self-resolve on next `--check` when reachable + +**Auto-offboard triggered unexpectedly:** +- Check `/boot/config/partnership_offline_days.db` for counter value +- Extended outage or Tailscale connectivity issue may have incremented counter +- Re-onboard if partnership should continue: `--onboard` + +--- + +## Design Notes + +**Why HOST1/HOST2 instead of PARTNERSHIP_OWNER/MIRROR?** + +HOST1 and HOST2 are already defined in Master.conf with SSH keys, Tailscale names, and all connection details. Duplicating them as PARTNERSHIP_OWNER and PARTNERSHIP_MIRROR would require maintaining the same values twice. A single `PARTNERSHIP_OWNER_HOST` var flips ownership direction — everything else derives from the existing HOST1/HOST2 configuration. + +**Why does transfer require a confirmation string?** + +Transferring ownership touches Master.conf on both servers, reconfigures WebUIs, and flips sync direction. The confirmation string is the same philosophy as `--i-know-what-im-doing` in the arr cleanup scripts — make accidental execution impossible, not just unlikely. + +**Why do Tailscale removal and backup access expire at the same time?** + +Keeping backups accessible beyond Tailscale removal is meaningless — if the mirror can't reach the owner's server via Tailscale, he can't access the backups anyway. `PARTNERSHIP_GRACE_HOURS` controls both simultaneously. One var, consistent behavior, no misleading "30 days available" when access is gone in 6 hours. + +**Why does either server auto-offboard after 30 days offline?** + +30 consecutive days of missed sync cycles means the relationship has effectively ended regardless of intent. The auto-offboard makes the state official and cleans up both sides without requiring manual intervention from a server that may genuinely be gone. Each server acts independently — they don't need to coordinate to offboard. + +**Why does WebUI reconfiguration use the unRAID template system?** + +unRAID stores container WebUI URLs in `/boot/config/plugins/dockerMan/templates-user/`. Editing templates is the correct way to change WebUI URLs — it persists across container restarts and array reboots. Direct Docker label manipulation would be lost on the next container recreation. + +**Why is Tailscale WebUI left on the mirror's local server?** + +Tailscale WebUI shows that server's network state from its own perspective. When diagnosing connectivity issues between the servers, you need to see the mirror's network view — not the owner's. It's the one WebUI that genuinely needs to stay local. \ No newline at end of file diff --git a/Partnership/partnership_manager.sh b/Partnership/partnership_manager.sh new file mode 100644 index 0000000..3c31ce0 --- /dev/null +++ b/Partnership/partnership_manager.sh @@ -0,0 +1,943 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# ----------------------------- Partnership Manager -------------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Manages the relationship lifecycle between two unRAID servers. +# HOST1 is always the owner (source of truth). HOST2 is always the mirror. +# PARTNERSHIP_OWNER_HOST flips to "HOST2" after --transfer. +# +# Modes: +# --onboard ← owner only — set up mirror relationship +# Reconfigures HOST2 auth WebUIs → HOST1 Tailscale IP +# HOST2 clicks NPM → gets HOST1's NPM automatically +# +# --offboard ← either server — clean separation +# Either party can initiate +# HOST2 offboard: writes state, reconfigures own WebUIs → localhost +# HOST1 sees it on next --check → finalises, runs final sync +# HOST1 offboard: final sync, reconfigures HOST2 WebUIs, removes Tailscale +# Both leave with current state, clean exit ✅ +# +# --transfer ← owner only — flip ownership +# Requires confirmation string + health strike system +# Reconfigures both servers, flips PARTNERSHIP_OWNER_HOST in Master.conf +# +# --check ← called by critical_sync_maintenance.sh every 15min +# Reads both state files via SSH +# Detects offboard requests → finalises from owner side +# Updates last_seen_remote timestamp +# Increments offline counter → auto-offboards after threshold +# Silent when healthy +# +# --status ← either server — show current state +# +# State files on /boot/config (survives reboots, available before array): +# /boot/config/partnership_HOST1.db ← HOST1 writes only +# /boot/config/partnership_HOST2.db ← HOST2 writes only +# Propagated via SSH — no rsync needed +# +# All configuration in Master.conf under PARTNERSHIP section. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +# Parse flags before parse_args +MODE="" +TRANSFER_CONFIRM_INPUT="" +REMOTE_SEEN=false +REMOTE_UNSEEN=false +FILTERED_ARGS=() + +for arg in "$@"; do + case "$arg" in + --onboard) MODE="onboard" ;; + --offboard) MODE="offboard" ;; + --transfer) MODE="transfer" ;; + --check) MODE="check" ;; + --status) MODE="status" ;; + --confirm=*) TRANSFER_CONFIRM_INPUT="${arg#--confirm=}" ;; + --remote-seen) REMOTE_SEEN=true ;; + --remote-unseen) REMOTE_UNSEEN=true ;; + *) FILTERED_ARGS+=("$arg") ;; + esac +done + +parse_args "${FILTERED_ARGS[@]}" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +detect_hosts + +# Derive owner and mirror from PARTNERSHIP_OWNER_HOST +if [[ "${PARTNERSHIP_OWNER_HOST:-HOST1}" == "HOST1" ]]; then + OWNER="$HOST1" + MIRROR="$HOST2" + OWNER_SSH_KEY="$HOST1_SSH_KEY" + MIRROR_SSH_KEY="$HOST2_SSH_KEY" +else + OWNER="$HOST2" + MIRROR="$HOST1" + OWNER_SSH_KEY="$HOST2_SSH_KEY" + MIRROR_SSH_KEY="$HOST1_SSH_KEY" +fi + +AM_OWNER=false +AM_MIRROR=false +[[ "$LOCAL_SERVER_NAME" == "$OWNER" ]] && AM_OWNER=true +[[ "$LOCAL_SERVER_NAME" == "$MIRROR" ]] && AM_MIRROR=true + +# State files +LOCAL_STATE_FILE="/boot/config/partnership_${LOCAL_SERVER_NAME}.db" +REMOTE_STATE_FILE="/boot/config/partnership_${REMOTE_SERVER_NAME}.db" +OWNER_STATE_FILE="/boot/config/partnership_${OWNER}.db" +MIRROR_STATE_FILE="/boot/config/partnership_${MIRROR}.db" + +# Offline counter file +OFFLINE_COUNTER="/boot/config/partnership_offline_days.db" + +if [[ -z "$MODE" ]]; then + error "No mode specified" + echo "Usage:" + echo " partnership_manage.sh --onboard [--dry-run]" + echo " partnership_manage.sh --offboard [--dry-run]" + echo " partnership_manage.sh --transfer --confirm=... [--dry-run]" + echo " partnership_manage.sh --check --remote-seen|--remote-unseen" + echo " partnership_manage.sh --status" + exit 1 +fi + +# Role-based access control +if [[ "$AM_MIRROR" == true ]]; then + case "$MODE" in + onboard|transfer) + error "Only the owner ($OWNER) can run --$MODE" + error "Run from $OWNER or use --offboard to separate cleanly" + exit 1 + ;; + esac +fi + +# Acquire lock for all modes except check (check is called frequently, lock would pile up) +[[ "$MODE" != "check" ]] && acquire_lock "strict" + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ----------------------------------------------------------------------------------------------- +# HELPERS +# ----------------------------------------------------------------------------------------------- + +read_state_file() { + local file="$1" + local key="$2" + grep "^${key}=" "$file" 2>/dev/null | cut -d= -f2 +} + +write_state_file() { + local file="$1" + shift + # flock to prevent concurrent writes + ( + flock -x 200 + cat > "$file" << EOF +state=${1:-UNKNOWN} +owner=${OWNER} +mirror=${MIRROR} +onboarded=${2:-} +offboarded=${3:-} +triggered_by=${4:-} +reason=${5:-} +last_seen_remote=${6:-} +updated=$(date '+%Y-%m-%d %H:%M:%S') +EOF + ) 200>"${file}.lock" +} + +push_state_to_remote() { + local local_file="$1" + local remote_ip="$2" + local ssh_key="$3" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would push state file to remote" + return 0 + fi + + scp -i "$ssh_key" -o ConnectTimeout=10 \ + "$local_file" "root@${remote_ip}:${local_file}" 2>/dev/null && \ + success "State file pushed to remote ✅" || \ + warn "Could not push state file to remote — will propagate on next sync" +} + +read_remote_state() { + local remote_ip="$1" + local ssh_key="$2" + local remote_file="$3" + + ssh -i "$ssh_key" -o ConnectTimeout=10 root@"$remote_ip" \ + "cat '$remote_file' 2>/dev/null" 2>/dev/null +} + +reconfigure_webui() { + local container="$1" + local port="$2" + local target_ip="$3" + local ssh_key="$4" + local remote_ip="$5" + local label="${6:-remote}" + + info "Reconfiguring $container WebUI → ${target_ip}:${port} on $label..." + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would reconfigure $container WebUI to http://${target_ip}:${port}/" + return 0 + fi + + local template + template=$(ssh -i "$ssh_key" -o ConnectTimeout=10 root@"$remote_ip" \ + "grep -rl '' /boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \ + xargs grep -l '\"$container\"' 2>/dev/null | head -1" 2>/dev/null) + + if [[ -z "$template" ]]; then + warn "$container template not found on $label — WebUI needs manual reconfiguration" + return 1 + fi + + ssh -i "$ssh_key" -o ConnectTimeout=10 root@"$remote_ip" \ + "sed -i 's|.*|http://${target_ip}:${port}/|g' '$template'" \ + 2>/dev/null && \ + success "$container → http://${target_ip}:${port}/ ✅" || { + error "Failed to reconfigure $container WebUI" + return 1 + } +} + +reconfigure_local_webuis() { + local target_ip="$1" + info "Reconfiguring local auth WebUIs → ${target_ip}..." + + local failures=0 + for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do + [[ -z "$entry" ]] && continue + local container="${entry%%|*}" + local port="${entry##*|}" + + local template + template=$(grep -rl '' /boot/config/plugins/dockerMan/templates-user/ 2>/dev/null | \ + xargs grep -l "\"$container\"" 2>/dev/null | head -1) + + if [[ -z "$template" ]]; then + warn "$container template not found locally" + ((failures++)) + continue + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would reconfigure $container → http://${target_ip}:${port}/" + continue + fi + + sed -i "s|.*|http://${target_ip}:${port}/|g" \ + "$template" 2>/dev/null && \ + success "$container → http://${target_ip}:${port}/ ✅" || \ + { error "Failed to reconfigure $container"; ((failures++)); } + done + return $failures +} + +get_tailscale_device_id() { + local hostname="$1" + curl -sf --max-time 10 \ + -H "Authorization: Bearer $TAILSCALE_API_KEY" \ + "https://api.tailscale.com/api/v2/tailnet/${TAILSCALE_TAILNET}/devices" \ + 2>/dev/null | \ + grep -o "\"id\":\"[^\"]*\"[^}]*\"hostname\":\"${hostname}\"" | \ + grep -o '"id":"[^"]*"' | \ + grep -o '[^"]*"$' | tr -d '"' +} + +remove_tailscale_device() { + local hostname="$1" + + if [[ -z "${TAILSCALE_API_KEY:-}" ]] || [[ -z "${TAILSCALE_TAILNET:-}" ]]; then + warn "TAILSCALE_API_KEY or TAILSCALE_TAILNET not configured — skipping Tailscale removal" + return 1 + fi + + info "Looking up Tailscale device ID for $hostname..." + local device_id + device_id=$(get_tailscale_device_id "$hostname") + + if [[ -z "$device_id" ]]; then + warn "Device $hostname not found in Tailscale — may already be removed" + return 0 + fi + + info "Removing $hostname (device $device_id) from Tailscale..." + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would remove $hostname from Tailscale tailnet" + return 0 + fi + + curl -sf --max-time 10 -X DELETE \ + -H "Authorization: Bearer $TAILSCALE_API_KEY" \ + "https://api.tailscale.com/api/v2/devices/${device_id}" 2>/dev/null && \ + success "$hostname removed from Tailscale ✅" || \ + error "Failed to remove $hostname from Tailscale — remove manually" +} + +check_both_healthy() { + mountpoint -q /mnt/user 2>/dev/null || { error "Local array not healthy"; return 1; } + + local mirror_ip + mirror_ip=$(tailscale ip -4 "$MIRROR" 2>/dev/null) + [[ -z "$mirror_ip" ]] && { error "Cannot resolve $MIRROR Tailscale IP"; return 1; } + + ssh -i "$MIRROR_SSH_KEY" -o ConnectTimeout=10 root@"$mirror_ip" \ + "mountpoint -q /mnt/user && timeout 10 docker ps" >/dev/null 2>&1 || { + error "Mirror not healthy" + return 1 + } + return 0 +} + +do_final_sync() { + info "Running final Critical-Data sync..." + if [[ "$DRY_RUN" == false ]]; then + bash "$SCRIPT_DIR/../Rsync/rsync.sh" \ + "/mnt/user/appdata-Failover/Critical-Data" --log + bash "$SCRIPT_DIR/../Rsync/rsync.sh" \ + "/mnt/user/Media_Server/Emby" --profile=emby-failover --log + else + warn "DRY RUN — would run final Critical-Data and emby-failover sync" + fi + success "Final sync complete — mirror has current state ✅" +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ STATUS ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$MODE" == "status" ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY PARTNERSHIP STATUS ━━━━━" + + OWNER_IP=$(tailscale ip -4 "$OWNER" 2>/dev/null || echo "unreachable") + MIRROR_IP=$(tailscale ip -4 "$MIRROR" 2>/dev/null || echo "unreachable") + + echo " Owner: $OWNER ($OWNER_IP)" + echo " Mirror: $MIRROR ($MIRROR_IP)" + echo " Enabled: ${PARTNERSHIP_ENABLED:-false}" + echo " Running as: $LOCAL_SERVER_NAME ($( [[ "$AM_OWNER" == true ]] && echo "owner" || echo "mirror"))" + echo "" + + # Local state + if [[ -f "$LOCAL_STATE_FILE" ]]; then + LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") + LOCAL_ONBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "onboarded") + LOCAL_OFFBOARDED=$(read_state_file "$LOCAL_STATE_FILE" "offboarded") + LOCAL_LAST_SEEN=$(read_state_file "$LOCAL_STATE_FILE" "last_seen_remote") + echo " Local state: $LOCAL_STATE" + [[ -n "$LOCAL_ONBOARDED" ]] && echo " Onboarded: $LOCAL_ONBOARDED" + [[ -n "$LOCAL_OFFBOARDED" ]] && echo " Offboarded: $LOCAL_OFFBOARDED" + [[ -n "$LOCAL_LAST_SEEN" ]] && echo " Remote last seen: $LOCAL_LAST_SEEN" + else + echo " Local state: no state file found" + fi + + echo "" + + # Remote state via SSH + REMOTE_IP=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) + if [[ -n "$REMOTE_IP" ]]; then + REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE") + if [[ -n "$REMOTE_CONTENT" ]]; then + REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2) + REMOTE_ONBOARDED=$(echo "$REMOTE_CONTENT" | grep "^onboarded=" | cut -d= -f2) + REMOTE_OFFBOARDED=$(echo "$REMOTE_CONTENT" | grep "^offboarded=" | cut -d= -f2) + echo " Remote state: $REMOTE_STATE" + [[ -n "$REMOTE_ONBOARDED" ]] && echo " Onboarded: $REMOTE_ONBOARDED" + [[ -n "$REMOTE_OFFBOARDED" ]] && echo " Offboarded: $REMOTE_OFFBOARDED" + + # Check agreement + if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]]; then + echo "" + echo " ✅ Both servers agree: $LOCAL_STATE" + else + echo "" + echo " ⚠️ State mismatch — local: $LOCAL_STATE remote: $REMOTE_STATE" + echo " Run --onboard or --offboard to resolve" + fi + else + echo " Remote state: state file not found on $REMOTE_SERVER_NAME" + fi + else + echo " Remote state: $REMOTE_SERVER_NAME unreachable" + fi + + echo "" + echo " Auth WebUIs configured:" + for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do + [[ -z "$entry" ]] && continue + echo " ${entry%%|*} → port ${entry##*|}" + done + + # Offline counter + if [[ -f "$OFFLINE_COUNTER" ]]; then + OFFLINE_DAYS=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0) + [[ "$OFFLINE_DAYS" -gt 0 ]] && \ + echo "" && \ + echo " ⚠️ Remote offline counter: ${OFFLINE_DAYS}/${PARTNERSHIP_OFFLINE_THRESHOLD} days" + fi + + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ CHECK (called every 15min by critical_sync_maintenance.sh) ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$MODE" == "check" ]]; then + + # Update last_seen_remote and offline counter based on rsync outcome + if [[ "$REMOTE_SEEN" == true ]]; then + OFFLINE_COUNT=0 + echo "0" > "$OFFLINE_COUNTER" + # Update last_seen_remote in local state file + if [[ -f "$LOCAL_STATE_FILE" ]]; then + sed -i "s|^last_seen_remote=.*|last_seen_remote=$(date '+%Y-%m-%d %H:%M:%S')|" \ + "$LOCAL_STATE_FILE" 2>/dev/null + fi + elif [[ "$REMOTE_UNSEEN" == true ]]; then + OFFLINE_COUNT=$(cat "$OFFLINE_COUNTER" 2>/dev/null || echo 0) + OFFLINE_COUNT=$(( OFFLINE_COUNT + 1 )) + echo "$OFFLINE_COUNT" > "$OFFLINE_COUNTER" + + # Check auto-offboard threshold + # Convert 15min intervals to days: threshold_days * 96 intervals/day + THRESHOLD_INTERVALS=$(( ${PARTNERSHIP_OFFLINE_THRESHOLD:-30} * 96 )) + if [[ "$OFFLINE_COUNT" -ge "$THRESHOLD_INTERVALS" ]]; then + warn "Remote offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days — triggering auto-offboard" + notify "Partnership auto-offboard on $(hostname) — $REMOTE_SERVER_NAME offline for ${PARTNERSHIP_OFFLINE_THRESHOLD} days. Both servers will offboard independently." \ + "Partnership" "warning" + bash "$0" --offboard --reason="auto-offboard-timeout" + exit 0 + fi + + log "Partnership check — remote unseen (count: $OFFLINE_COUNT/$THRESHOLD_INTERVALS)" + fi + + # Read remote state file + REMOTE_IP=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) + if [[ -z "$REMOTE_IP" ]]; then + log "Partnership check — remote unreachable, skipping state check" + exit 0 + fi + + REMOTE_CONTENT=$(read_remote_state "$REMOTE_IP" "$SSH_KEY" "$REMOTE_STATE_FILE") + [[ -z "$REMOTE_CONTENT" ]] && log "Partnership check — remote state file not found" && exit 0 + + REMOTE_STATE=$(echo "$REMOTE_CONTENT" | grep "^state=" | cut -d= -f2) + LOCAL_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state" 2>/dev/null || echo "UNKNOWN") + + # Both agree — healthy + if [[ "$LOCAL_STATE" == "$REMOTE_STATE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then + log "Partnership check — ACTIVE, both servers agree ✅" + exit 0 + fi + + # Remote requested offboard + if [[ "$REMOTE_STATE" == "INACTIVE" ]] && [[ "$LOCAL_STATE" == "ACTIVE" ]]; then + info "Partnership check — $REMOTE_SERVER_NAME requested offboard" + + if [[ "$AM_OWNER" == true ]]; then + info "Owner finalising offboard request from mirror..." + # Final sync — mirror leaves with current state + do_final_sync + # Update local state + NOW=$(date '+%Y-%m-%d %H:%M:%S') + write_state_file "$LOCAL_STATE_FILE" \ + "INACTIVE" "" "$NOW" "$REMOTE_SERVER_NAME" "mirror-requested" + # Disable critical rsync + if [[ "$DRY_RUN" == false ]]; then + sed -i 's/CRITICAL_RSYNC_ENABLED=true/CRITICAL_RSYNC_ENABLED=false/' \ + "$SCRIPT_DIR/../Master.conf" 2>/dev/null + fi + # Tailscale removal after grace period + if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then + local grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-3} * 3600 )) + info "Waiting ${PARTNERSHIP_GRACE_HOURS:-3}hr grace period for mirror to see notification..." + [[ "$DRY_RUN" == false ]] && sleep "$grace_seconds" + remove_tailscale_device "$MIRROR" + fi + notify "Partnership offboard finalised on $(hostname) — $MIRROR requested separation" \ + "Partnership" "normal" + else + # Mirror sees owner is INACTIVE — clean up own side + info "Owner has offboarded — cleaning up mirror side..." + reconfigure_local_webuis "localhost" + NOW=$(date '+%Y-%m-%d %H:%M:%S') + write_state_file "$LOCAL_STATE_FILE" \ + "INACTIVE" "" "$NOW" "$OWNER" "owner-offboarded" + notify "Partnership ended on $(hostname) — $OWNER has offboarded. Auth WebUIs reconfigured to localhost." \ + "Partnership" "normal" + fi + exit 0 + fi + + # Both inactive — nothing to do + if [[ "$LOCAL_STATE" == "INACTIVE" ]] && [[ "$REMOTE_STATE" == "INACTIVE" ]]; then + log "Partnership check — INACTIVE on both servers" + exit 0 + fi + + log "Partnership check — local: $LOCAL_STATE remote: $REMOTE_STATE" + exit 0 +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ ONBOARD ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$MODE" == "onboard" ]]; then + echo "" + echo "━━━ $ICON_FAILOVER Onboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + + # Check already onboarded + if [[ -f "$LOCAL_STATE_FILE" ]]; then + CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") + if [[ "$CURRENT_STATE" == "ACTIVE" ]]; then + info "Partnership already ACTIVE — no action needed" + info "Use --status for detail or --offboard to separate" + exit 0 + fi + fi + + info "Owner: $OWNER" + info "Mirror: $MIRROR" + + # Pre-flight + echo "" + echo "━━━ $ICON_SHIELD Pre-flight ━━━" + + resolve_remote_ip + check_connectivity + check_remote_array || exit 1 + check_remote_docker || exit 1 + + OWNER_IP=$(tailscale ip -4 "$OWNER" 2>/dev/null) + MIRROR_IP=$(tailscale ip -4 "$MIRROR" 2>/dev/null) + + [[ -z "$OWNER_IP" ]] && error "Cannot resolve $OWNER Tailscale IP" && exit 1 + [[ -z "$MIRROR_IP" ]] && error "Cannot resolve $MIRROR Tailscale IP" && exit 1 + + success "Owner IP: $OWNER_IP" + success "Mirror IP: $MIRROR_IP" + + # Reconfigure mirror WebUIs → owner IP + echo "" + echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → $OWNER_IP ━━━" + + WEBUI_FAILURES=0 + for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do + [[ -z "$entry" ]] && continue + container="${entry%%|*}" + port="${entry##*|}" + reconfigure_webui "$container" "$port" "$OWNER_IP" \ + "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || ((WEBUI_FAILURES++)) + done + + # Verify WebUI connectivity + if [[ "${PARTNERSHIP_ONBOARD_VERIFY:-true}" == true ]]; then + echo "" + echo "━━━ $ICON_HEALTH Verify WebUI Connectivity ━━━" + for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do + [[ -z "$entry" ]] && continue + container="${entry%%|*}" + port="${entry##*|}" + if curl -sf --max-time 10 "http://${OWNER_IP}:${port}/" >/dev/null 2>&1; then + success "$container reachable at http://${OWNER_IP}:${port}/ ✅" + else + warn "$container not reachable at http://${OWNER_IP}:${port}/ — may not be running" + fi + done + fi + + # Write state files + echo "" + echo "━━━ $ICON_GEAR Write State ━━━" + NOW=$(date '+%Y-%m-%d %H:%M:%S') + + write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "onboard" + success "Local state: ACTIVE ✅" + + # Push to remote + push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" + + # Reset offline counter + echo "0" > "$OFFLINE_COUNTER" + + # Summary + echo "" + echo "━━━━━ $ICON_SUMMARY ONBOARD SUMMARY ━━━━━" + echo " Owner: $OWNER ($OWNER_IP)" + echo " Mirror: $MIRROR ($MIRROR_IP)" + echo " WebUI failures: $WEBUI_FAILURES" + echo " Sync interval: ${PARTNERSHIP_SYNC_INTERVAL}min" + echo "" + + if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no changes made" + elif [[ "$WEBUI_FAILURES" -eq 0 ]]; then + echo "$ICON_DONE Status: $ICON_SUCCESS DONE" + [[ "${PARTNERSHIP_ONBOARD_NOTIFY:-true}" == true ]] && \ + notify "Partnership onboard complete — $MIRROR is now mirroring $OWNER via Tailscale" \ + "Partnership" "normal" + else + echo "$ICON_WARN Status: DONE with $WEBUI_FAILURES WebUI warning(s)" + notify "Partnership onboard complete with $WEBUI_FAILURES WebUI warning(s) on $(hostname)" \ + "Partnership" "warning" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ OFFBOARD ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$MODE" == "offboard" ]]; then + echo "" + echo "━━━ $ICON_FAILOVER Offboard — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + + # Check already offboarded + if [[ -f "$LOCAL_STATE_FILE" ]]; then + CURRENT_STATE=$(read_state_file "$LOCAL_STATE_FILE" "state") + if [[ "$CURRENT_STATE" == "INACTIVE" ]]; then + info "Partnership already INACTIVE" + info "Use --status to verify both servers agree" + exit 0 + fi + fi + + REASON="${REASON:-manual}" + + # ── MIRROR-initiated offboard ──────────────────────────────────────────────────────────── + if [[ "$AM_MIRROR" == true ]]; then + warn "$MIRROR is initiating offboard" + warn "This will reconfigure your local auth WebUIs → localhost" + warn "HOST1 will finalise the offboard on its next --check cycle" + + if [[ "$DRY_RUN" == false ]]; then + echo "" + echo "You have 10 seconds to cancel (Ctrl+C)..." + sleep 10 + fi + + # Reconfigure own WebUIs → localhost + echo "" + echo "━━━ $ICON_CONTAINERS Reconfigure Local WebUIs → localhost ━━━" + reconfigure_local_webuis "localhost" + + # Write own state file + NOW=$(date '+%Y-%m-%d %H:%M:%S') + write_state_file "$LOCAL_STATE_FILE" \ + "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" + success "Local state: INACTIVE ✅" + + # Try to push state to owner immediately + OWNER_IP=$(tailscale ip -4 "$OWNER" 2>/dev/null) + if [[ -n "$OWNER_IP" ]]; then + push_state_to_remote "$LOCAL_STATE_FILE" "$OWNER_IP" "$MIRROR_SSH_KEY" + notify "Partnership offboard requested by $MIRROR — owner will finalise on next check cycle" \ + "Partnership" "normal" + else + warn "$OWNER unreachable — state written locally, owner will see it when reachable" + fi + + echo "" + echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━" + echo " Your WebUIs: reconfigured → localhost ✅" + echo " State: INACTIVE written ✅" + echo " Owner: will finalise + run final sync on next --check ✅" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 + fi + + # ── OWNER-initiated offboard ───────────────────────────────────────────────────────────── + warn "Offboarding $MIRROR from partnership" + warn "Final sync will run — mirror leaves with current state" + warn "$MIRROR auth WebUIs will be reconfigured → localhost" + + if [[ "$DRY_RUN" == false ]]; then + echo "" + echo "You have 10 seconds to cancel (Ctrl+C)..." + sleep 10 + echo "Proceeding..." + fi + + resolve_remote_ip + + # Stop any running rsync first + echo "" + echo "━━━ $ICON_STOP Stop Running Rsync ━━━" + bash "$SCRIPT_DIR/../unRAID_Essentials/rsync_stop.sh" --rsync-only 2>/dev/null || true + + # Final sync + echo "" + echo "━━━ $ICON_SYNC Final Sync ━━━" + do_final_sync + + MIRROR_IP=$(tailscale ip -4 "$MIRROR" 2>/dev/null) + MIRROR_REACHABLE=false + [[ -n "$MIRROR_IP" ]] && MIRROR_REACHABLE=true + + # Reconfigure mirror WebUIs → localhost + echo "" + echo "━━━ $ICON_CONTAINERS Reconfigure Mirror WebUIs → localhost ━━━" + + WEBUI_FAILURES=0 + if [[ "$MIRROR_REACHABLE" == true ]]; then + for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do + [[ -z "$entry" ]] && continue + container="${entry%%|*}" + port="${entry##*|}" + reconfigure_webui "$container" "$port" "localhost" \ + "$MIRROR_SSH_KEY" "$MIRROR_IP" "$MIRROR" || ((WEBUI_FAILURES++)) + done + else + warn "$MIRROR unreachable — WebUI reconfiguration skipped" + warn "$MIRROR will reconfigure its own WebUIs when it sees INACTIVE state on next --check" + ((WEBUI_FAILURES++)) + fi + + # Disable critical rsync + echo "" + echo "━━━ $ICON_GEAR Disable Critical Sync ━━━" + if [[ "$DRY_RUN" == false ]]; then + sed -i 's/CRITICAL_RSYNC_ENABLED=true/CRITICAL_RSYNC_ENABLED=false/' \ + "$SCRIPT_DIR/../Master.conf" 2>/dev/null + success "CRITICAL_RSYNC_ENABLED=false ✅" + else + warn "DRY RUN — would set CRITICAL_RSYNC_ENABLED=false" + fi + + # Write state files + echo "" + echo "━━━ $ICON_GEAR Write State ━━━" + NOW=$(date '+%Y-%m-%d %H:%M:%S') + + write_state_file "$LOCAL_STATE_FILE" \ + "INACTIVE" "" "$NOW" "$LOCAL_SERVER_NAME" "$REASON" + success "Local state: INACTIVE ✅" + + # Push to mirror + if [[ "$MIRROR_REACHABLE" == true ]]; then + push_state_to_remote "$LOCAL_STATE_FILE" "$MIRROR_IP" "$MIRROR_SSH_KEY" + fi + + # Tailscale removal + if [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]]; then + echo "" + echo "━━━ $ICON_NET Tailscale Separation ━━━" + if [[ "$MIRROR_REACHABLE" == true ]]; then + local grace_seconds=$(( ${PARTNERSHIP_GRACE_HOURS:-3} * 3600 )) + info "Waiting ${PARTNERSHIP_GRACE_HOURS:-3}hr grace period for mirror to see state..." + [[ "$DRY_RUN" == false ]] && sleep "$grace_seconds" + fi + remove_tailscale_device "$MIRROR" + fi + + # Backup notification + if [[ ${#PARTNERSHIP_MIRROR_BACKUPS[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_DISK Backup Handover ━━━" + info "Backups available for $MIRROR:" + for path in "${PARTNERSHIP_MIRROR_BACKUPS[@]}"; do + [[ -z "$path" ]] && continue + echo " $path" + done + info "Available for ${PARTNERSHIP_GRACE_HOURS:-3} hours — Tailscale access removed after this window" + notify "$MIRROR offboard complete — backups available for ${PARTNERSHIP_GRACE_HOURS:-3}hr. Tailscale access expires then. Certs valid ~60-90 days. Run tailscale up to rejoin." \ + "Partnership" "warning" + fi + + # Summary + echo "" + echo "━━━━━ $ICON_SUMMARY OFFBOARD SUMMARY ━━━━━" + echo " Owner: $OWNER" + echo " Mirror: $MIRROR" + echo " Final sync: complete ✅" + echo " WebUI failures: $WEBUI_FAILURES" + echo " Critical rsync: disabled ✅" + echo " State: INACTIVE ✅" + [[ "${PARTNERSHIP_REMOVE_TAILSCALE:-true}" == true ]] && \ + echo " Tailscale: $MIRROR removed ✅" + echo "" + echo " $MIRROR leaves with:" + echo " ✓ Current auth config (final sync)" + echo " ✓ His own DDNS and updater" + echo " ✓ His own Git mirror" + echo " ✓ Auth WebUIs → localhost" + echo " ✓ ${PARTNERSHIP_GRACE_HOURS:-3}hr to collect backups before Tailscale removed" + echo " ✓ Certs valid ~60-90 more days" + echo " ✓ Full ecosystem — just stop the sync" + + if [[ "$DRY_RUN" == true ]]; then + echo "" + echo "$ICON_WARN Status: DRY RUN — no changes made" + else + echo "" + echo "$ICON_DONE Status: $ICON_SUCCESS DONE — clean separation complete" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ TRANSFER ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$MODE" == "transfer" ]]; then + echo "" + echo "━━━ $ICON_FAILOVER Transfer Ownership — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "⚠️ WARNING — OWNERSHIP TRANSFER" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " Current owner: $OWNER" + echo " Current mirror: $MIRROR" + echo "" + echo " After transfer:" + echo " New owner: $MIRROR" + echo " New mirror: $OWNER" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + # Confirmation string + if [[ "$DRY_RUN" == false ]]; then + if [[ -z "$TRANSFER_CONFIRM_INPUT" ]]; then + echo "" + echo "To proceed type exactly:" + echo " --confirm=${PARTNERSHIP_TRANSFER_CONFIRM}" + echo "" + error "Transfer cancelled — confirmation required" + exit 1 + fi + if [[ "$TRANSFER_CONFIRM_INPUT" != "$PARTNERSHIP_TRANSFER_CONFIRM" ]]; then + error "Confirmation string does not match — transfer cancelled" + exit 1 + fi + success "Confirmation accepted" + else + warn "DRY RUN — confirmation check skipped" + fi + + # Health strike system + echo "" + echo "━━━ $ICON_SHIELD Health Verification ━━━" + info "Both servers must pass ${PARTNERSHIP_TRANSFER_STRIKES} consecutive health checks" + + STRIKES=0 + ATTEMPTS=0 + MAX_ATTEMPTS="${PARTNERSHIP_TRANSFER_MAX_ATTEMPTS:-20}" + + while [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]]; do + ((ATTEMPTS++)) + if [[ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]]; then + error "Health checks failed after $MAX_ATTEMPTS attempts — servers not stable" + error "Transfer cancelled — try again when both servers are healthy" + exit 1 + fi + + if check_both_healthy; then + ((STRIKES++)) + success "Health check passed ($STRIKES/${PARTNERSHIP_TRANSFER_STRIKES})" + [[ "$STRIKES" -lt "$PARTNERSHIP_TRANSFER_STRIKES" ]] && sleep 10 + else + warn "Health check failed — resetting (attempt $ATTEMPTS/$MAX_ATTEMPTS)" + STRIKES=0 + sleep 30 + fi + done + success "Both servers healthy — proceeding" + + NEW_OWNER="$MIRROR" + NEW_MIRROR="$OWNER" + NEW_OWNER_IP=$(tailscale ip -4 "$NEW_OWNER" 2>/dev/null) + NEW_MIRROR_IP=$(tailscale ip -4 "$NEW_MIRROR" 2>/dev/null) + + [[ -z "$NEW_OWNER_IP" ]] && error "Cannot resolve new owner Tailscale IP" && exit 1 + + # Final sync in current direction + echo "" + echo "━━━ $ICON_SYNC Pre-transfer Sync ━━━" + do_final_sync + + # Reconfigure new mirror WebUIs → new owner + echo "" + echo "━━━ $ICON_CONTAINERS Reconfigure WebUIs ━━━" + info "New mirror ($NEW_MIRROR) WebUIs → new owner ($NEW_OWNER_IP)" + + NEW_MIRROR_SSH="${HOST1_SSH_KEY}" + [[ "$NEW_MIRROR" == "$HOST2" ]] && NEW_MIRROR_SSH="${HOST2_SSH_KEY}" + NEW_OWNER_SSH="${HOST1_SSH_KEY}" + [[ "$NEW_OWNER" == "$HOST2" ]] && NEW_OWNER_SSH="${HOST2_SSH_KEY}" + + WEBUI_FAILURES=0 + for entry in "${PARTNERSHIP_AUTH_WEBUIS[@]:-}"; do + [[ -z "$entry" ]] && continue + container="${entry%%|*}" + port="${entry##*|}" + reconfigure_webui "$container" "$port" "$NEW_OWNER_IP" \ + "$NEW_MIRROR_SSH" "$NEW_MIRROR_IP" "$NEW_MIRROR" || ((WEBUI_FAILURES++)) + done + + # New owner WebUIs → localhost (now manages directly) + info "New owner ($NEW_OWNER) WebUIs → localhost" + reconfigure_local_webuis "localhost" + + # Flip PARTNERSHIP_OWNER_HOST in Master.conf on both servers + echo "" + echo "━━━ $ICON_GEAR Update Ownership ━━━" + NEW_OWNER_HOST="HOST1" + [[ "$NEW_OWNER" == "$HOST2" ]] && NEW_OWNER_HOST="HOST2" + + if [[ "$DRY_RUN" == false ]]; then + sed -i "s/PARTNERSHIP_OWNER_HOST=.*/PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_HOST\"/" \ + "$SCRIPT_DIR/../Master.conf" 2>/dev/null + success "Local Master.conf updated: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_HOST ✅" + + # Update remote + ssh -i "$NEW_MIRROR_SSH" -o ConnectTimeout=10 root@"$NEW_MIRROR_IP" \ + "sed -i 's/PARTNERSHIP_OWNER_HOST=.*/PARTNERSHIP_OWNER_HOST=\"$NEW_OWNER_HOST\"/' \ + '$SCRIPT_DIR/../Master.conf'" 2>/dev/null && \ + success "Remote Master.conf updated ✅" || \ + error "Failed to update remote Master.conf — update manually" + else + warn "DRY RUN — would set PARTNERSHIP_OWNER_HOST=$NEW_OWNER_HOST on both servers" + fi + + # Write state files + NOW=$(date '+%Y-%m-%d %H:%M:%S') + write_state_file "$LOCAL_STATE_FILE" "ACTIVE" "$NOW" "" "$LOCAL_SERVER_NAME" "transfer" + push_state_to_remote "$LOCAL_STATE_FILE" "$NEW_MIRROR_IP" "$NEW_MIRROR_SSH" + + # Summary + echo "" + echo "━━━━━ $ICON_SUMMARY TRANSFER SUMMARY ━━━━━" + echo " New owner: $NEW_OWNER ($NEW_OWNER_IP)" + echo " New mirror: $NEW_MIRROR ($NEW_MIRROR_IP)" + echo " WebUI failures: $WEBUI_FAILURES" + echo " Sync direction: $NEW_OWNER → $NEW_MIRROR" + echo " Master.conf: PARTNERSHIP_OWNER_HOST=$NEW_OWNER_HOST" + + if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no changes made" + else + echo "$ICON_DONE Status: $ICON_SUCCESS DONE — ownership transferred" + notify "Partnership ownership transferred — new owner: $NEW_OWNER new mirror: $NEW_MIRROR" \ + "Partnership" "normal" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi \ No newline at end of file diff --git a/safe_master.conf b/safe_master.conf index 1fe15fb..e6c9822 100644 --- a/safe_master.conf +++ b/safe_master.conf @@ -98,8 +98,8 @@ # SSH keys for server-to-server rsync and failover container operations. # Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys. - HOST1_SSH_KEY="/root/.ssh/your_key" - HOST2_SSH_KEY="/root/.ssh/your_key" + HOST1_SSH_KEY="/root/.ssh/your_host1_key" + HOST2_SSH_KEY="/root/.ssh/your_host2_key" # ━━━ Emby ━━━ # Defined once here — referenced by transcode_manager.sh, emby_session_report.sh, @@ -113,6 +113,75 @@ HOST2_EMBY_URL="http://localhost:8096" # same port — different server, different key HOST2_EMBY_API_KEY="your-emby-api-key" +# ============================================================================================== +# ── PARTNERSHIP ──────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Manages the relationship lifecycle between two unRAID servers. +# HOST1 is always the owner (source of truth) — HOST2 is always the mirror. +# PARTNERSHIP_OWNER_HOST flips to "HOST2" after a --transfer operation. +# All other vars derive from HOST1/HOST2 — no duplicate IP or key configuration needed. +# +# State files live on /boot/config — survives reboots, available before array starts: +# /boot/config/partnership_HOST1.db ← HOST1 writes only +# /boot/config/partnership_HOST2.db ← HOST2 writes only +# Each server SSHes to write/read the other's file directly — no rsync needed +# +# critical_sync_maintenance.sh runs --check every 15min: +# Reads both state files via SSH +# Detects offboard requests → finalises from owner side +# Increments offline counter → auto-offboards after threshold +# Silent when healthy ✅ + + PARTNERSHIP_ENABLED=false + PARTNERSHIP_OWNER_HOST="HOST1" # "HOST1" or "HOST2" — flips on --transfer + +# Auth containers whose WebUI URL gets reconfigured on onboard/offboard +# Format: "ContainerName|Port" +# These are the containers the mirror operator clicks in his Docker UI +# On onboard → pointed at owner's Tailscale IP +# On offboard → pointed back at localhost +PARTNERSHIP_AUTH_WEBUIS=( + "NginxProxyManager|81" + "Lldap|17170" + "Authelia|9091" + "Authelia-Secondary|9092" +) + +# Paths on HOST1 that HOST2 operator should collect before backup retention expires +# Notified on offboard — no auto-deletion, manual collection +PARTNERSHIP_MIRROR_BACKUPS=( + # "/mnt/user/appdata-Failover/Host2-Emby" + # add any paths HOST2 wants to collect after separation +) + +# Offboard settings + PARTNERSHIP_GRACE_HOURS=3 # hours after offboard before Tailscale removal + # backup access also expires at this time + # collect anything needed before this window closes + PARTNERSHIP_REMOVE_TAILSCALE=true # remove mirror from Tailscale on offboard + # false = skip removal (manual or testing) + +# Tailscale API — used to remove HOST2 from tailnet on offboard +# API key: https://login.tailscale.com/admin/settings/keys +# Tailnet: your tailnet name (e.g. "yourname.github" or "youremail.com") + TAILSCALE_API_KEY="" # tskey-api-... + TAILSCALE_TAILNET="" # your tailnet name + +# Transfer safety + PARTNERSHIP_TRANSFER_CONFIRM="i-understand-this-transfers-ownership" + PARTNERSHIP_TRANSFER_STRIKES=3 # consecutive health checks required + PARTNERSHIP_TRANSFER_MAX_ATTEMPTS=20 # max attempts before giving up + +# Onboard settings + PARTNERSHIP_ONBOARD_VERIFY=true # verify WebUI reachable after reconfiguration + PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on completion + PARTNERSHIP_SYNC_INTERVAL=15 # minutes — Critical-Data sync frequency + +# Auto-offboard threshold + PARTNERSHIP_OFFLINE_THRESHOLD=30 # days either server unreachable before auto-offboard + # works both directions — mirror offline 30d → owner offboards + # owner offline 30d → mirror offboards itself + # ============================================================================================== # ── LOGGING ─────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -142,7 +211,7 @@ # Falls back to GITEA_DOMAIN if local and Tailscale both fail. GITEA_CONTAINER="Gitea" # exact Docker container name GITEA_REPO_PATH="youruser/Unraid_Scripts.git" # repo path on Gitea server - GITEA_DOMAIN="" # e.g. git.gmer4lfe.com — requires NPM + DNS setup + GITEA_DOMAIN="" # e.g. git.yourdomain.com — requires NPM + DNS setup TARGET_DIR="/mnt/user/appdata/unraid_scripts" # where scripts are cloned to GITEA_SSH_KEY="/root/.ssh/your_gitea_key" # SSH key for authenticating to Gitea SSH_PORT=221 # Gitea SSH port (default 22, Gitea often uses 222/221) @@ -238,6 +307,15 @@ WEEKLY_SYNC_JOBS=( "/mnt/user/appdata-Failover/Critical-Data" # critical-data profile — auth stack ) +# ━━━ Critical Sync Maintenance ━━━ +# Shares synced every 15 minutes by critical_sync_maintenance.sh +# Format: "/path/to/share" or "/path/to/share|profile-name" +# Order matters — Critical-Data first (auth stack), then emby-failover (dirty sync) +CRITICAL_SYNC_SHARES=( + "/mnt/user/appdata-Failover/Critical-Data" # auth stack — full sync + "/mnt/user/Media_Server/Emby|emby-failover" # Emby dirty sync — stays running +) + # Container update toggles for the weekly sync window. # Containers are already stopped for the sync — updates pull at no extra downtime. # Both false → sync only, no updates. @@ -245,6 +323,14 @@ WEEKLY_SYNC_JOBS=( CRITICAL_SYNC_UPDATES=true # pull container updates locally CRITICAL_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH +# Orchestrators that use rsync — read by rsync_stop.sh for auto-detection +# Add any new orchestrator that calls rsync.sh here — no script changes needed +# rsync_stop.sh will detect and handle it automatically on stop +RSYNC_ORCHESTRATORS=( + "daily_sync_maintenance" + "weekly_sync_maintenance" +) + # ━━━ Media Management ━━━ # Job list run directly by daily_sync_maintenance.sh after the media share sync. # Runs sequentially — permissions first, then cleaners, then arr cleanup. @@ -265,6 +351,31 @@ MEDIA_MANAGEMENT_JOBS=( # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== +# ━━━ Rsync Enable/Disable ━━━ +# Two-tier toggle system — Tier 1 overrides Tier 2. +# +# Tier 1 — Global gate: +# RSYNC_ENABLED=false → ALL rsync stops everywhere, no exceptions +# Use when: remote server completely offline, major maintenance, disaster recovery +# +# Tier 2 — Per-orchestrator (only applies when Tier 1 is true): +# Fine grained control — disable specific orchestrators while keeping others +# Use when: rebuilding secondary, testing, per-window bandwidth management +# +# Real world example (HOST2 data rebuild — your current situation): +# RSYNC_ENABLED=true ← rsync works, individual scripts run fine +# DAILY_RSYNC_ENABLED=false ← skip daily HDD syncs during rebuild +# WEEKLY_RSYNC_ENABLED=true ← Emby + Critical-Data still sync (NVMe, separate BW) +# FAILOVER_RSYNC_ENABLED=true ← handback writeback still works when needed +# → Run individual: bash Rsync/rsync.sh /mnt/user/Movies (test each share manually) +# → When ready: DAILY_RSYNC_ENABLED=true + + RSYNC_ENABLED=true # Tier 1 — global gate, overrides everything below + DAILY_RSYNC_ENABLED=false # Tier 2 — daily_sync_maintenance.sh rsync section + WEEKLY_RSYNC_ENABLED=true # Tier 2 — weekly_sync_maintenance.sh rsync section + FAILOVER_RSYNC_ENABLED=true # Tier 2 — failover.sh writeback jobs on handback + CRITICAL_RSYNC_ENABLED=true # Tier 2 — critical_sync_maintenance.sh (every 15min) + # ━━━ Rsync Defaults ━━━ # Global fallback values used when no profile match is found. # Media shares in HOST*_DAILY_SYNC_SHARES always use these globals — no profile needed. @@ -300,7 +411,7 @@ MEDIA_MANAGEMENT_JOBS=( # Current profiles: # arrs_stack — arr databases — lower bandwidth, containers stopped for consistency # critical-data — auth stack — containers stopped both sides, Authelia delayed start -# gmer4lfe — server-specific appdata — no container stops needed +# host1 — server-specific appdata — no container stops needed # important-data — NextCloud + Postgres — NextCloud delayed start after Postgres # emby — weekly clean sync — both Emby stopped, full mirror, minimal excludes # called by weekly_sync_maintenance.sh only — do NOT schedule separately @@ -310,7 +421,7 @@ MEDIA_MANAGEMENT_JOBS=( declare -A PROFILE_RSYNC_OPTS=( [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" [critical-data]="-av --human-readable --bwlimit=$BW_LIMIT --delete" - [gmer4lfe]="-av --info=progress2 --bwlimit=$BW_LIMIT" + [host1]="-av --info=progress2 --bwlimit=$BW_LIMIT" [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" [emby]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" [emby-failover]="-av --human-readable --bwlimit=$BW_LIMIT --delete --inplace --no-whole-file" @@ -321,7 +432,7 @@ declare -A PROFILE_RSYNC_OPTS=( declare -A PROFILE_BW_LIMIT=( [arrs_stack]=5000 # lower — runs alongside other syncs, avoids saturating link [critical-data]=9500 # high — small dataset, get it synced fast and clean - [gmer4lfe]=8000 + [host1]=8000 [important-data]=9500 # high — database sync needs to be fast [emby]=8000 # medium — large full mirror, steady transfer [emby-failover]=9500 # high — small critical dataset, sync as fast as possible @@ -331,7 +442,7 @@ declare -A PROFILE_BW_LIMIT=( declare -A PROFILE_RETRY_COUNT=( [arrs_stack]=3 [critical-data]=3 - [gmer4lfe]=3 + [host1]=3 [important-data]=3 [emby]=3 [emby-failover]=3 @@ -342,7 +453,7 @@ declare -A PROFILE_RETRY_COUNT=( declare -A PROFILE_SLEEP=( [arrs_stack]=300 [critical-data]=300 - [gmer4lfe]=300 + [host1]=300 [important-data]=300 [emby]=300 [emby-failover]=120 # shorter — frequent dirty sync, retry faster @@ -358,7 +469,7 @@ declare -A PROFILE_SLEEP=( declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap NginxProxyManager Authelia Authelia-Secondary" - [gmer4lfe]="Organizrv2 UptimeKuma VaultWarden" + [host1]="Organizrv2 UptimeKuma VaultWarden" [important-data]="Postgres-NextCloud NextCloud" [emby]="Emby" # weekly clean sync — both Emby instances stopped, WAL checkpointed [emby-failover]="" # dirty sync — Emby stays running both sides, WAL excluded from sync @@ -371,7 +482,7 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( declare -A PROFILE_DELAYED_CONTAINERS=( [arrs_stack]="" [critical-data]="Authelia Authelia-Secondary" # wait for Mariadb + Redis to be ready - [gmer4lfe]="" + [host1]="" [important-data]="NextCloud" # wait for Postgres to accept connections [emby]="" [emby-failover]="" @@ -382,7 +493,7 @@ declare -A PROFILE_DELAYED_CONTAINERS=( declare -A PROFILE_CONTAINER_DELAY=( [arrs_stack]=5 [critical-data]=15 # Mariadb + Redis need time to accept connections - [gmer4lfe]=5 + [host1]=5 [important-data]=10 # Postgres needs time before NextCloud [emby]=5 [emby-failover]=5 @@ -394,7 +505,7 @@ declare -A PROFILE_CONTAINER_DELAY=( # SPACE-SEPARATED STRINGS — converted to array at runtime declare -A PROFILE_EXCLUDE_DIRS=( [arrs_stack]="logs *.tmp" - [gmer4lfe]="logs *.tmp" + [host1]="logs *.tmp" [important-data]="logs *.tmp" [critical-data]="logs *.tmp *.log nginx/temp nginx/cache __pycache__ notification.txt" [emby]="logs transcodes cache crash*" @@ -403,16 +514,8 @@ declare -A PROFILE_EXCLUDE_DIRS=( [emby-failover]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" ) -# Skip per-disk space check for these profiles — appdata syncs go to cache/appdata -# not to array disks, so disk space check is irrelevant and just slows things down -declare -A PROFILE_SKIP_DISK_CHECK=( - [arrs_stack]=true - [critical-data]=true - [gmer4lfe]=true - [important-data]=true - [emby]=true - [emby-failover]=true -) +# Skip per-disk space check is no longer needed — check_remote_disks() auto-detects +# XFS and ZFS filesystem types from disks.ini, no manual configuration required # ============================================================================================== # ── FAILOVER ────────────────────────────────────────────────────────────────────────────────── diff --git a/unRAID_Essentials/system_watchdog.sh b/unRAID_Essentials/system_watchdog.sh index d05810f..0479f53 100644 --- a/unRAID_Essentials/system_watchdog.sh +++ b/unRAID_Essentials/system_watchdog.sh @@ -2,7 +2,7 @@ # ----------------------------------------------------------------------------------------------- # --------------------------------- System Watchdog -------------------------------------------- # ----------------------------------------------------------------------------------------------- -# Last line of defense — reboots the system cleanly if it is about to become unstable. +# Last line of defense — reb on oots the system cleanly if it is about to become unstable. # Runs continuously as a background process — started by array_start.sh at array start. # Works alongside docker_watchdog.sh which handles container-level healing first. #