Compare commits

..
2 Commits
10 changed files with 32 additions and 25 deletions
+8 -8
View File
@@ -358,7 +358,7 @@
# Shares synced every 30 minutes — defined per host in host*.conf. # Shares synced every 30 minutes — defined per host in host*.conf.
# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES # HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES
# Format: "/path/to/share" or "/path/to/share|profile-name" # Format: "/path/to/share" or "/path/to/share|profile-name"
# Order matters — Critical-Data first (auth stack), then Emby dirty sync. # Order matters — Critical-Data first (auth stack), then maintenance scripts.
# ━━━ Intermediate Sync Maintenance ━━━ # ━━━ Intermediate Sync Maintenance ━━━
# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch, # intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch,
@@ -568,7 +568,7 @@
# emby — weekly clean sync — both Emby stopped, full mirror # emby — weekly clean sync — both Emby stopped, full mirror
# called by weekly_sync_maintenance.sh only — do NOT schedule separately # called by weekly_sync_maintenance.sh only — do NOT schedule separately
# emby-fallback — dirty sync — Emby stays running, WAL excluded # emby-fallback — dirty sync — Emby stays running, WAL excluded
# called by critical_sync_maintenance.sh every 30min # unused — replaced by play_state_sync
declare -A PROFILE_RSYNC_OPTS=( declare -A PROFILE_RSYNC_OPTS=(
[arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace"
@@ -606,7 +606,7 @@
[critical-fallback]=120 # shorter — frequent dirty sync, retry faster [critical-fallback]=120 # shorter — frequent dirty sync, retry faster
[important-data]=300 [important-data]=300
[emby]=300 [emby]=300
[emby-fallback]=120 # shorter — frequent dirty sync, retry faster [emby-fallback]=120 # shorter — retry faster (unused)
) )
# Containers stopped on BOTH LOCAL and REMOTE before rsync. # Containers stopped on BOTH LOCAL and REMOTE before rsync.
@@ -615,11 +615,11 @@
# SPACE-SEPARATED STRINGS — converted to array at runtime # SPACE-SEPARATED STRINGS — converted to array at runtime
declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
[arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat"
[critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap-Gmer4Lfe NginxProxyManager Authelia Authelia-Secondary" [critical-data]="Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary Lldap NginxProxyManager Authelia Authelia-Secondary"
[critical-fallback]="" # dirty sync — auth stays running both sides [critical-fallback]="" # dirty sync — auth stays running both sides
[important-data]="Postgres-NextCloud NextCloud" [important-data]="Postgres-NextCloud NextCloud"
[emby]="Emby" [emby]="Emby"
[emby-fallback]="" # dirty sync — Emby stays running both sides [emby-fallback]="" # unused — no containers stopped
) )
# Containers needing a delay after rsync before starting. # Containers needing a delay after rsync before starting.
@@ -654,12 +654,12 @@
[emby-fallback]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root" [emby-fallback]="logs transcodes cache metadata *.db-wal *.db-shm crash* plugins root"
) )
# Remote restart after dirty sync — restart these on remote IF they were running before sync. # Remote restart after live sync — restart these on remote IF they were running before sync.
# Same logic as stop/start — was stopped = stays stopped, was running = gets restarted. # Same logic as stop/start — was stopped = stays stopped, was running = gets restarted.
# Used by dirty sync profiles (critical-fallback, emby-fallback) so remote picks up changes. # Used by live sync profiles (critical-fallback) so remote picks up changes.
# SPACE-SEPARATED STRINGS — converted to array at runtime # SPACE-SEPARATED STRINGS — converted to array at runtime
declare -A PROFILE_REMOTE_RESTART_CONTAINERS=( declare -A PROFILE_REMOTE_RESTART_CONTAINERS=(
[critical-fallback]="NginxProxyManager Authelia Authelia-Secondary Lldap-Gmer4Lfe Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary" [critical-fallback]="NginxProxyManager Authelia Authelia-Secondary Lldap Mariadb-Authelia Mariadb-Authelia-Secondary Redis-Authelia Redis-Authelia-Secondary"
[emby-fallback]="Emby" [emby-fallback]="Emby"
) )
+1 -1
View File
@@ -193,7 +193,7 @@ FALLBACK_HOST2_COVERS_HOST1_TIER1=(
"Gmer4Lfe.com-DDNS" # ALWAYS FIRST — DNS coverage before anything else "Gmer4Lfe.com-DDNS" # ALWAYS FIRST — DNS coverage before anything else
"Emby" # media server — people are watching "Emby" # media server — people are watching
"NginxProxyManager" # reverse proxy — all external access routes through this "NginxProxyManager" # reverse proxy — all external access routes through this
"Lldap-Gmer4Lfe" # user directory — already warm, verify and keep "Lldap" # user directory — already warm, verify and keep
"Mariadb-Authelia" # auth database — already warm, verify and keep "Mariadb-Authelia" # auth database — already warm, verify and keep
"Redis-Authelia" # auth session cache — already warm, verify and keep "Redis-Authelia" # auth session cache — already warm, verify and keep
"Authelia" # SSO — already warm, serving users already "Authelia" # SSO — already warm, serving users already
+6 -6
View File
@@ -104,7 +104,7 @@ array_started.sh
# Cron — managed via Varaverk Scheduler: # Cron — managed via Varaverk Scheduler:
*/7 * * * * transcode_management.sh */7 * * * * transcode_management.sh
*/15 * * * * watchdog_orchestrator.sh ← resource → docker → system → stability */15 * * * * watchdog_orchestrator.sh ← resource → docker → system → stability
*/30 * * * * critical_sync_maintenance.sh ← auth + Emby dirty sync + partnership */30 * * * * critical_sync_maintenance.sh ← auth + play_state_sync + partnership
0 */4 * * * intermediate_sync_maintenance.sh ← arr sync + failed recovery + optional rsync 0 */4 * * * intermediate_sync_maintenance.sh ← arr sync + failed recovery + optional rsync
0 1 * * * daily_sync_maintenance.sh 0 1 * * * daily_sync_maintenance.sh
0 7 * * 0 sunday_morning_coffee_report.sh 0 7 * * 0 sunday_morning_coffee_report.sh
@@ -735,11 +735,11 @@ maintenance block before the 7am coffee report.
```bash ```bash
# ───────────────────────────────────────────────────────────────────────────── # ─────────────────────────────────────────────────────────────────────────────
# Two Emby syncs run in parallel — dirty and clean: # Two Emby sync mechanisms keep the mirror current:
# #
# emby-fallback dirty sync (every 30 minutes via critical_sync_maintenance.sh, Emby running): # play_state_sync (every 30 minutes via critical_sync_maintenance.sh):
# watch states, library deltas, user activity — continuous coverage # API-based — syncs watched state and resume points directly via Emby API
# WAL files excluded — safe to copy while Emby writes # No rsync of live files — no WAL risk, no partial-write corruption
# HOST2 always within 30 minutes of HOST1 on playback state # HOST2 always within 30 minutes of HOST1 on playback state
# #
# weekly clean sync (Sunday 2:30am, Emby stopped): # weekly clean sync (Sunday 2:30am, Emby stopped):
@@ -908,7 +908,7 @@ array_started.sh
*/15 * * * * watchdog_orchestrator.sh */15 * * * * watchdog_orchestrator.sh
# ───────────────────────────────────────────────────────────────────────────── # ─────────────────────────────────────────────────────────────────────────────
# Every 30 minutes — auth stack + Emby dirty sync + partnership check: # Every 30 minutes — auth stack + play_state_sync + partnership check:
# ───────────────────────────────────────────────────────────────────────────── # ─────────────────────────────────────────────────────────────────────────────
*/30 * * * * critical_sync_maintenance.sh */30 * * * * critical_sync_maintenance.sh
+3 -4
View File
@@ -7,10 +7,9 @@
# Schedule: */30 * * * * (every 30 minutes via User Scripts plugin) # Schedule: */30 * * * * (every 30 minutes via User Scripts plugin)
# #
# ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── # ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
# 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides) # 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides)
# 2. emby-fallback rsync — dirty Emby sync (watch states, library — Emby stays running) # 2. CRITICAL_MAINTENANCE_SCRIPTS — play_state_sync + any other per-window scripts
# 3. CRITICAL_MAINTENANCE_SCRIPTS — any scripts configured for critical window # 3. partnership --check — read both state files, detect changes, act accordingly
# 4. partnership --check — read both state files, detect changes, act accordingly
# #
# ── WHY EVERY 30 MINUTES ────────────────────────────────────────────────────────────────────── # ── WHY EVERY 30 MINUTES ──────────────────────────────────────────────────────────────────────
# Auth stack changes (new users, proxy rules, certs) propagate within 30min ✅ # Auth stack changes (new users, proxy rules, certs) propagate within 30min ✅
+1 -1
View File
@@ -19,7 +19,7 @@
# Emby builds a warm image cache on HOST2 throughout the week. # Emby builds a warm image cache on HOST2 throughout the week.
# Syncing nightly resets cache — cold loads every morning for users. # Syncing nightly resets cache — cold loads every morning for users.
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep. # Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep.
# emby-fallback dirty sync covers watch states + library every 30min between weekly syncs. # play_state_sync covers watch/resume state every 30 min between weekly syncs.
# #
# ── CONTAINER UPDATES ───────────────────────────────────────────────────────────────────────── # ── CONTAINER UPDATES ─────────────────────────────────────────────────────────────────────────
# Containers already stopped for sync — updates pull at zero extra downtime. # Containers already stopped for sync — updates pull at zero extra downtime.
+2 -2
View File
@@ -138,7 +138,7 @@ PARTNERSHIP_ONBOARD_NOTIFY=true # notify both servers on successful onboard
# Format: "ContainerName|WebUIPort" # Format: "ContainerName|WebUIPort"
HOST1_PARTNERSHIP_AUTH_WEBUIS=( HOST1_PARTNERSHIP_AUTH_WEBUIS=(
"NginxProxyManager|81" "NginxProxyManager|81"
"Lldap-Gmer4Lfe|17170" "Lldap|17170"
"Authelia|9091" "Authelia|9091"
"Authelia-Secondary|9092" "Authelia-Secondary|9092"
) )
@@ -154,7 +154,7 @@ HOST1_PARTNERSHIP_AUTH_STACK=(
"my-Authelia.xml" "my-Authelia.xml"
"my-Authelia-Secondary.xml" "my-Authelia-Secondary.xml"
"my-NginxProxyManager.xml" "my-NginxProxyManager.xml"
"my-Lldap-Gmer4Lfe.xml" "my-Lldap.xml"
) )
# XML templates pushed to mirror during onboard (arr stack). # XML templates pushed to mirror during onboard (arr stack).
+8
View File
@@ -431,6 +431,9 @@ cleanup_deployed_stack_locally() {
return 0 return 0
fi fi
local _local_short
_local_short=$(derive_short_name "${LOCAL_SERVER_NAME:-}")
log "Removing owner-deployed containers (auth/arr/services stacks) locally..." log "Removing owner-deployed containers (auth/arr/services stacks) locally..."
for xml_name in "${xml_names[@]}"; do for xml_name in "${xml_names[@]}"; do
[[ -z "$xml_name" ]] && continue [[ -z "$xml_name" ]] && continue
@@ -444,6 +447,11 @@ cleanup_deployed_stack_locally() {
cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file") cname=$(awk 'match($0,/<Name>([^<]+)<\/Name>/,a){print a[1];exit}' "$xml_file")
[[ -z "$cname" ]] && continue [[ -z "$cname" ]] && continue
if [[ -n "$_local_short" && "$cname" == *"$_local_short"* ]]; then
warn " $cname contains local server identity ($_local_short) — skipping to protect local containers"
continue
fi
if [[ "$DRY_RUN" == true ]]; then if [[ "$DRY_RUN" == true ]]; then
warn " DRY RUN — would stop + rm $cname" warn " DRY RUN — would stop + rm $cname"
warn " DRY RUN — would delete appdata for $cname" warn " DRY RUN — would delete appdata for $cname"
+1 -1
View File
@@ -519,7 +519,7 @@
# Logs each transfer to BANDWIDTH_LOG automatically (read by bandwidth_monitor --report). # Logs each transfer to BANDWIDTH_LOG automatically (read by bandwidth_monitor --report).
# #
# Profile behaviour — what containers stop: # Profile behaviour — what containers stop:
# critical-data Mariadb-Authelia, Redis-Authelia, NginxProxyManager, Lldap-Gmer4Lfe # critical-data Mariadb-Authelia, Redis-Authelia, NginxProxyManager, Lldap
# + delayed restart: Authelia, Authelia-Secondary (30s after dependencies) # + delayed restart: Authelia, Authelia-Secondary (30s after dependencies)
# important-data Postgres-NextCloud + delayed: NextCloud # important-data Postgres-NextCloud + delayed: NextCloud
# arrs_stack Sonarr, Radarr, Lidarr, Prowlarr, Bazarr, Pinchflat # arrs_stack Sonarr, Radarr, Lidarr, Prowlarr, Bazarr, Pinchflat
+1 -1
View File
@@ -68,7 +68,7 @@ break silently.
# #
Emby # the shared Emby — runs on HOST1, picked up by HOST2 on fallback Emby # the shared Emby — runs on HOST1, picked up by HOST2 on fallback
NginxProxyManager # shared NPM — runs on both, same config via sync NginxProxyManager # shared NPM — runs on both, same config via sync
Lldap-Gmer4Lfe # shared LLDAP — same user directory, both servers Lldap # shared LLDAP — same user directory, both servers
Authelia # shared auth — both servers, same config Authelia # shared auth — both servers, same config
Mariadb-Authelia # Authelia's database — same on both Mariadb-Authelia # Authelia's database — same on both
Redis-Authelia # Authelia's cache — same on both Redis-Authelia # Authelia's cache — same on both
+1 -1
View File
@@ -321,7 +321,7 @@ PROFILES["arrs_stack_CRITICAL_CONTAINER_NAMES"]=(
# Auth stack — stopped for clean database snapshot, Authelia delayed restart # Auth stack — stopped for clean database snapshot, Authelia delayed restart
PROFILES["critical-data_CRITICAL_CONTAINER_NAMES"]=( PROFILES["critical-data_CRITICAL_CONTAINER_NAMES"]=(
"Mariadb-Authelia" "Redis-Authelia" "Mariadb-Authelia" "Redis-Authelia"
"NginxProxyManager" "Lldap-Gmer4Lfe" "NginxProxyManager" "Lldap"
) )
PROFILES["critical-data_DELAYED_CONTAINERS"]=( PROFILES["critical-data_DELAYED_CONTAINERS"]=(
"Authelia" "Authelia-Secondary" "Authelia" "Authelia-Secondary"