padded partnership set up
This commit is contained in:
@@ -0,0 +1,172 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------------------- Critical Sync Maintenance --------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Orchestrator for time-sensitive syncs that run every 15 minutes.
|
||||
# Keeps the mirror current between the less frequent daily and weekly windows.
|
||||
# Schedule: */15 * * * * (every 15 minutes)
|
||||
#
|
||||
# Execution order:
|
||||
# 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides)
|
||||
# 2. emby-failover rsync — dirty Emby sync (watch states, library — Emby stays running)
|
||||
# 3. partnership --check — read both state files, detect changes, act accordingly
|
||||
#
|
||||
# Why every 15 minutes:
|
||||
# Auth stack changes (new users, proxy rules, certs) propagate within 15min ✅
|
||||
# Emby watch states stay in sync — mirror users see correct playback position ✅
|
||||
# Partnership state changes detected and acted on quickly ✅
|
||||
#
|
||||
# Rsync gate:
|
||||
# RSYNC_ENABLED=false → skips all syncs (global gate)
|
||||
# CRITICAL_RSYNC_ENABLED=false → skips critical syncs (per-orchestrator)
|
||||
# partnership --check still runs regardless of rsync gate
|
||||
# (state check doesn't need rsync to work)
|
||||
#
|
||||
# Lock behavior:
|
||||
# acquire_lock "strict" — if previous 15min run still going, skip this cycle
|
||||
# Critical-Data taking > 15min is a problem worth knowing about
|
||||
# Lock prevents pile-up ✅
|
||||
#
|
||||
# Configuration in Master.conf:
|
||||
# CRITICAL_RSYNC_ENABLED — enable/disable rsync section
|
||||
# PARTNERSHIP_ENABLED — enable/disable partnership check
|
||||
# CRITICAL_SYNC_SHARES — shares synced every 15min
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
acquire_lock "strict"
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
START=$(date +%s)
|
||||
RSYNC_OK=false
|
||||
PASS=()
|
||||
FAIL=()
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Critical Shares Sync ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Critical Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
if ! check_rsync_enabled "CRITICAL"; then
|
||||
warn "Critical rsync disabled — skipping sync, running partnership check only"
|
||||
else
|
||||
for share in "${CRITICAL_SYNC_SHARES[@]:-}"; do
|
||||
[[ -z "$share" ]] && continue
|
||||
|
||||
# Parse optional profile flag: "/path/to/share|profile-name"
|
||||
SHARE_PATH="${share%%|*}"
|
||||
SHARE_PROFILE="${share##*|}"
|
||||
SHARE_NAME=$(basename "$SHARE_PATH")
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC $SHARE_NAME ━━━"
|
||||
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then
|
||||
# No profile specified
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH"
|
||||
else
|
||||
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" --profile="$SHARE_PROFILE"
|
||||
fi
|
||||
|
||||
RSYNC_EXIT=$?
|
||||
SHARE_END=$(date +%s)
|
||||
SHARE_DUR=$(format_duration $(( SHARE_END - SHARE_START )))
|
||||
|
||||
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
|
||||
PASS+=("$SHARE_NAME")
|
||||
success "$SHARE_NAME — done in $SHARE_DUR ✅"
|
||||
RSYNC_OK=true
|
||||
else
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME — failed after $SHARE_DUR"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CLEAN Critical Maintenance Scripts ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Critical Maintenance — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
||||
log "No CRITICAL_MAINTENANCE_SCRIPTS defined — skipping"
|
||||
else
|
||||
for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
# Strip leading comment lines
|
||||
[[ "$script_entry" == \#* ]] && continue
|
||||
|
||||
SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
|
||||
SCRIPT_ARGS="${script_entry#* }"
|
||||
[[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
|
||||
|
||||
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
|
||||
echo " → $SCRIPT_NAME"
|
||||
|
||||
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
||||
warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
bash "$SCRIPT_PATH" $SCRIPT_ARGS
|
||||
EXIT_CODE=$?
|
||||
if [[ "$EXIT_CODE" -ne 0 ]]; then
|
||||
warn "$SCRIPT_NAME exited with code $EXIT_CODE"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SHIELD Partnership Check ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Partnership Check ━━━"
|
||||
|
||||
if [[ "${PARTNERSHIP_ENABLED:-false}" == false ]]; then
|
||||
log "Partnership disabled — skipping check"
|
||||
else
|
||||
# Pass rsync outcome to --check so it can update last_seen_remote
|
||||
if [[ "$RSYNC_OK" == true ]]; then
|
||||
bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-seen
|
||||
else
|
||||
bash "$SCRIPT_DIR/partnership_manage.sh" --check --remote-unseen
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
END=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
|
||||
if [[ ${#PASS[@]} -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Synced: ${PASS[*]}"
|
||||
fi
|
||||
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Failed: ${FAIL[*]}"
|
||||
notify "Critical sync failed on $(hostname) — ${FAIL[*]}" "Critical Sync" "warning"
|
||||
fi
|
||||
if [[ ${#PASS[@]} -eq 0 ]] && [[ ${#FAIL[@]} -eq 0 ]]; then
|
||||
echo "$ICON_SKIP Rsync: disabled"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -23,7 +23,7 @@
|
||||
#
|
||||
# Configuration in Master.conf:
|
||||
# DAILY_MAINTENANCE_SCRIPTS — pre/post-sync scripts (git pull, docker restart)
|
||||
# MEDIA_MANAGEMENT_JOBS — media maintenance jobs run after sync
|
||||
# DAILY_MAINTENANCE_SCRIPTS — media maintenance jobs run after sync
|
||||
# HOST1_DAILY_SYNC_SHARES — shares HOST1 pushes to HOST2
|
||||
# HOST2_DAILY_SYNC_SHARES — shares HOST2 pushes to HOST1
|
||||
# HOST1/2_PERSONAL_SHARES — encrypted personal shares
|
||||
@@ -201,14 +201,14 @@ TOTAL_DURATION=$((TOTAL_END - TOTAL_START))
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CLEAN Post-sync Media Jobs ━━━
|
||||
# Reads MEDIA_MANAGEMENT_JOBS from Master.conf — permissions, cleaners, arr cleanup
|
||||
# Reads DAILY_MAINTENANCE_SCRIPTS from Master.conf — permissions, cleaners, arr cleanup
|
||||
# Runs after sync completes — correct ownership available, clean folders guaranteed
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Post-sync Media Jobs ━━━"
|
||||
|
||||
if [[ ${#MEDIA_MANAGEMENT_JOBS[@]} -gt 0 ]]; then
|
||||
for script_entry in "${MEDIA_MANAGEMENT_JOBS[@]}"; do
|
||||
if [[ ${#DAILY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
|
||||
@@ -8,15 +8,15 @@
|
||||
# Execution order:
|
||||
# 1. Stop local containers — Emby + auth stack stopped locally
|
||||
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
|
||||
# 3. Pull updates locally — if CRITICAL_SYNC_UPDATES=true
|
||||
# 4. Pull updates remotely — if CRITICAL_SYNC_UPDATES_REMOTE=true
|
||||
# 3. Pull updates locally — if WEEKLY_SYNC_UPDATES=true
|
||||
# 4. Pull updates remotely — if WEEKLY_SYNC_UPDATES_REMOTE=true
|
||||
# 5. rsync Emby — full clean mirror, both instances stopped
|
||||
# 6. rsync Critical-Data — auth stack clean sync, databases flushed
|
||||
# 7. Start remote containers — correct order, delayed start respected
|
||||
# 8. Start local containers — correct order, delayed start respected
|
||||
# 9. docker_weekly_restart.sh — weekly container restarts
|
||||
#
|
||||
# Synced shares (WEEKLY_SYNC_JOBS in Master.conf):
|
||||
# Synced shares (WEEKLY_SYNC_SHARES in Master.conf):
|
||||
# /mnt/user/Media_Server/Emby — emby profile — full mirror, cache resets weekly
|
||||
# /mnt/user/appdata-Failover/Critical-Data — critical-data — auth stack clean state
|
||||
#
|
||||
@@ -29,16 +29,16 @@
|
||||
# Container updates during the window:
|
||||
# Containers already stopped for sync — updates pull at zero extra downtime
|
||||
# Both servers start on identical image versions after the window completes
|
||||
# Toggle: CRITICAL_SYNC_UPDATES / CRITICAL_SYNC_UPDATES_REMOTE in Master.conf
|
||||
# Toggle: WEEKLY_SYNC_UPDATES / WEEKLY_SYNC_UPDATES_REMOTE in Master.conf
|
||||
#
|
||||
# What triggers weekly_health_digest.sh:
|
||||
# NOT this script — weekly_health_digest.sh runs on its own Saturday schedule
|
||||
#
|
||||
# Configuration in Master.conf:
|
||||
# WEEKLY_SYNC_JOBS — shares synced during the maintenance window
|
||||
# WEEKLY_SYNC_SHARES — shares synced during the maintenance window
|
||||
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync (docker_weekly_restart)
|
||||
# CRITICAL_SYNC_UPDATES — toggle container updates on/off
|
||||
# CRITICAL_SYNC_UPDATES_REMOTE — toggle remote container updates on/off
|
||||
# WEEKLY_SYNC_UPDATES — toggle container updates on/off
|
||||
# WEEKLY_SYNC_UPDATES_REMOTE — toggle remote container updates on/off
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# All configuration in Master.conf.
|
||||
# Supports --dry-run to walk through without stopping containers, syncing, or updating.
|
||||
@@ -107,7 +107,7 @@ fi
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Container Updates ━━━"
|
||||
|
||||
if [[ "$CRITICAL_SYNC_UPDATES" == true ]]; then
|
||||
if [[ "$WEEKLY_SYNC_UPDATES" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull updates for local containers"
|
||||
for c in "${MAINTENANCE_CONTAINERS[@]}"; do
|
||||
@@ -133,10 +133,10 @@ if [[ "$CRITICAL_SYNC_UPDATES" == true ]]; then
|
||||
done
|
||||
fi
|
||||
else
|
||||
info "CRITICAL_SYNC_UPDATES=false — skipping local updates"
|
||||
info "WEEKLY_SYNC_UPDATES=false — skipping local updates"
|
||||
fi
|
||||
|
||||
if [[ "$CRITICAL_SYNC_UPDATES_REMOTE" == true ]]; then
|
||||
if [[ "$WEEKLY_SYNC_UPDATES_REMOTE" == true ]]; then
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would pull updates on $REMOTE_SERVER_NAME"
|
||||
else
|
||||
@@ -158,7 +158,7 @@ if [[ "$CRITICAL_SYNC_UPDATES_REMOTE" == true ]]; then
|
||||
done
|
||||
fi
|
||||
else
|
||||
info "CRITICAL_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
||||
info "WEEKLY_SYNC_UPDATES_REMOTE=false — skipping remote updates"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
@@ -168,7 +168,7 @@ PASS=()
|
||||
FAIL=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
SYNC_JOBS=("${WEEKLY_SYNC_JOBS[@]}")
|
||||
SYNC_JOBS=("${WEEKLY_SYNC_SHARES[@]}")
|
||||
SHARE_COUNT=${#SYNC_JOBS[@]}
|
||||
|
||||
echo ""
|
||||
@@ -280,7 +280,7 @@ WINDOW_END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $TOTAL_DURATION"
|
||||
echo "$ICON_GEAR Updates: local=$CRITICAL_SYNC_UPDATES remote=$CRITICAL_SYNC_UPDATES_REMOTE"
|
||||
echo "$ICON_GEAR Updates: local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Sync jobs:"
|
||||
@@ -307,7 +307,7 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE"
|
||||
notify "Weekly sync maintenance complete on $(hostname) — synced + updated (local=$CRITICAL_SYNC_UPDATES remote=$CRITICAL_SYNC_UPDATES_REMOTE)" "Weekly Maintenance" "normal"
|
||||
notify "Weekly sync maintenance complete on $(hostname) — synced + updated (local=$WEEKLY_SYNC_UPDATES remote=$WEEKLY_SYNC_UPDATES_REMOTE)" "Weekly Maintenance" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $TOTAL_FAIL failure(s) — check logs"
|
||||
notify "Weekly sync maintenance failed on $(hostname) — sync: ${#FAIL[@]} failed, jobs: ${#JOB_FAIL[@]} failed" "Weekly Maintenance" "warning"
|
||||
|
||||
Reference in New Issue
Block a user