Gmer4LfeandClaude Sonnet 4.6
0ae31b5fa6
common.sh:
- Add resolve_tailscale_ip() helper — tries `tailscale ip -4` first, falls back to
parsing `tailscale status` output; handles hosts where MagicDNS short-name resolution
is not active
- Add PARTNERSHIP_OWN_CONTAINERS alias in detect_hosts()
- Add aliasing for 4 Emby provisioning vars (PARTNERSHIP_PROVISION_EMBY_ADMIN,
PARTNERSHIP_EMBY_ADMIN_USER, PARTNERSHIP_EMBY_ADMIN_PASS, PARTNERSHIP_EMBY_PORT)
Partnership/partnership_manager.sh:
- Replace 9 bare `tailscale ip -4` calls with resolve_tailscale_ip()
- Add read_remote_conf_var() and read_remote_conf_array() — SSH to mirror, source its
own load_config.sh + detect_hosts(), return aliased variable; solves sparse-checkout
problem where HOST1 cannot read master_host2.conf directly
- Add derive_short_name() — strips unraid- prefix, capitalises first char
- Add cleanup_partner_containers() — removes partner containers via FolderView3 folder
if enabled, else falls back to FALLBACK_*_COVERS_*_TIER* arrays
- Add cleanup_owner_containers_on_mirror() — SSH to mirror, stops and removes containers
matching *-${OWNER_SHORT} naming convention
- Add start_own_stack() and start_mirror_own_stack() — restart own containers locally
or on mirror via SSH using PARTNERSHIP_OWN_CONTAINERS
- Add provision_emby_admin() — reads mirror credentials via read_remote_conf_var, checks
for username collision, creates user + sets password + grants admin policy via Emby API
- Add revoke_emby_admin() — looks up mirror username on local Emby, deletes via REST API
- Wire offboard paths (both mirror-initiated and owner-initiated) to call container
cleanup and stack restart; update --check finalisation paths accordingly
- Fix write_state_file in --onboard not gated on DRY_RUN (was writing ACTIVE state on
dry runs)
master_host1.conf:
- Add HOST1_PARTNERSHIP_OWN_CONTAINERS array
- Add partnership Emby provisioning config (toggle + port + per-host credentials)
master_host2.conf:
- Add HOST2_PARTNERSHIP_OWN_CONTAINERS array
- Add HOST2_PARTNERSHIP_EMBY_ADMIN_USER and HOST2_PARTNERSHIP_EMBY_ADMIN_PASS
Tailscale fix applied to:
- Initial_run/ssh_setup.sh (2 callsites)
- unRAID_Essentials/rsync_stop.sh (1 callsite)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
232 lines
11 KiB
Bash
232 lines
11 KiB
Bash
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Critical Sync Maintenance ======================================
|
|
# ==============================================================================================
|
|
# Orchestrator for time-sensitive syncs that run every 15 minutes.
|
|
# Keeps the mirror current between the less frequent daily and weekly windows.
|
|
# Schedule: */15 * * * * (every 15 minutes via User Scripts plugin)
|
|
#
|
|
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
|
# 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides)
|
|
# 2. emby-fallback rsync — dirty Emby sync (watch states, library — Emby stays running)
|
|
# 3. CRITICAL_MAINTENANCE_SCRIPTS — any scripts configured for critical window
|
|
# 4. partnership --check — read both state files, detect changes, act accordingly
|
|
#
|
|
# ── WHY EVERY 15 MINUTES ──────────────────────────────────────────────────────────────────────
|
|
# Auth stack changes (new users, proxy rules, certs) propagate within 15min ✅
|
|
# Emby watch states stay in sync — mirror users see correct playback position ✅
|
|
# Partnership state changes detected and acted on quickly ✅
|
|
# Lock prevents: daily rsync doing Critical-Data mid-critical window ✅
|
|
#
|
|
# ── RSYNC GATE ────────────────────────────────────────────────────────────────────────────────
|
|
# RSYNC_ENABLED=false → skips all syncs (global gate)
|
|
# CRITICAL_RSYNC_ENABLED=false → skips critical syncs only (per-orchestrator gate)
|
|
# partnership --check always runs regardless — state check doesn't need rsync
|
|
#
|
|
# ── LOCK BEHAVIOUR ────────────────────────────────────────────────────────────────────────────
|
|
# acquire_lock "strict" — if previous 15min run still going, skip this cycle entirely
|
|
# Critical-Data taking > 15min is a problem worth knowing about
|
|
# Strict mode prevents pile-up without waiting — log and move on ✅
|
|
#
|
|
# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
|
|
# Runs 96 times per day — clean runs must produce zero output ✅
|
|
# Only failures and notable events produce visible output
|
|
#
|
|
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
|
# CRITICAL_RSYNC_ENABLED — enable/disable rsync section
|
|
# CRITICAL_SYNC_SHARES — shares synced every 15min (HOST*_CRITICAL_SYNC_SHARES)
|
|
# CRITICAL_MAINTENANCE_SCRIPTS — scripts run in critical window (optional)
|
|
# PARTNERSHIP_ENABLED — enable/disable partnership check
|
|
#
|
|
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
|
# critical_sync_maintenance.sh — normal run
|
|
# critical_sync_maintenance.sh --dry-run — preview syncs without transferring
|
|
# critical_sync_maintenance.sh --log — verbose per-share output
|
|
# critical_sync_maintenance.sh --status — show configuration and exit
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
validate_unraid_cmd \
|
|
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
|
"" "" \
|
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
|
|
|
acquire_lock "strict"
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
|
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
|
echo "$ICON_SYNC Critical enabled: ${CRITICAL_RSYNC_ENABLED:-false}"
|
|
echo "$ICON_SHIELD Partnership: ${PARTNERSHIP_ENABLED:-false}"
|
|
echo ""
|
|
echo "━━━ Critical Sync Shares ━━━"
|
|
if [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
|
warn " No CRITICAL_SYNC_SHARES configured"
|
|
else
|
|
for share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
|
[[ -z "$share" ]] && continue
|
|
SHARE_PATH="${share%%|*}"
|
|
SHARE_PROFILE="${share##*|}"
|
|
SHARE_NAME=$(basename "$SHARE_PATH")
|
|
[[ "$SHARE_PATH" == "$SHARE_PROFILE" ]] && \
|
|
echo " $ICON_SYNC $SHARE_NAME — no profile" || \
|
|
echo " $ICON_SYNC $SHARE_NAME — profile: $SHARE_PROFILE"
|
|
done
|
|
fi
|
|
echo ""
|
|
echo "━━━ Critical Maintenance Scripts ━━━"
|
|
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
|
echo " None configured"
|
|
else
|
|
for entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$entry" || "$entry" == \#* ]] && continue
|
|
echo " $ICON_GEAR $(basename "${entry%% *}")"
|
|
done
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Critical Shares Sync ━━━
|
|
# ==============================================================================================
|
|
START=$(date +%s)
|
|
RSYNC_OK=false
|
|
PASS=()
|
|
FAIL=()
|
|
|
|
if ! check_rsync_enabled "CRITICAL"; then
|
|
log "Critical rsync disabled — skipping sync, running partnership check only"
|
|
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
|
warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID"
|
|
warn "Check HOST*_CRITICAL_SYNC_SHARES in master_host*.conf"
|
|
else
|
|
log "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')"
|
|
|
|
# Build dry-run flag to pass through
|
|
RSYNC_DRY=""
|
|
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
|
|
|
for share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
|
[[ -z "$share" ]] && continue
|
|
|
|
# Parse optional profile flag: "/path/to/share|profile-name"
|
|
SHARE_PATH="${share%%|*}"
|
|
SHARE_PROFILE="${share##*|}"
|
|
SHARE_NAME=$(basename "$SHARE_PATH")
|
|
|
|
SHARE_START=$(date +%s)
|
|
|
|
if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then
|
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" $RSYNC_DRY
|
|
else
|
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" \
|
|
--profile="$SHARE_PROFILE" $RSYNC_DRY
|
|
fi
|
|
|
|
RSYNC_EXIT=$?
|
|
SHARE_DUR=$(format_duration $(( $(date +%s) - SHARE_START )))
|
|
|
|
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
|
|
PASS+=("$SHARE_NAME")
|
|
log "$SHARE_NAME — done in $SHARE_DUR ✅"
|
|
RSYNC_OK=true
|
|
else
|
|
FAIL+=("$SHARE_NAME")
|
|
error "$SHARE_NAME — failed after $SHARE_DUR (exit $RSYNC_EXIT)"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Critical Maintenance Scripts ━━━
|
|
# ==============================================================================================
|
|
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
|
for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$script_entry" || "$script_entry" == \#* ]] && continue
|
|
|
|
SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
|
|
SCRIPT_ARGS="${script_entry#* }"
|
|
[[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
|
|
[[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run"
|
|
|
|
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
|
|
|
|
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
|
warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
|
|
continue
|
|
fi
|
|
|
|
log "Running: $SCRIPT_NAME"
|
|
bash "$SCRIPT_PATH" $SCRIPT_ARGS
|
|
EXIT_CODE=$?
|
|
[[ "$EXIT_CODE" -ne 0 ]] && \
|
|
warn "$SCRIPT_NAME exited with code $EXIT_CODE"
|
|
done
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Partnership Check ━━━
|
|
# ==============================================================================================
|
|
if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
|
|
PARTNER_DRY=""
|
|
[[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run"
|
|
|
|
if [[ "$RSYNC_OK" == true ]]; then
|
|
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
|
--check --remote-seen $PARTNER_DRY
|
|
else
|
|
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
|
--check --remote-unseen $PARTNER_DRY
|
|
fi
|
|
else
|
|
log "Partnership disabled — skipping check"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
END=$(date +%s)
|
|
DURATION=$(format_duration $(( END - START )))
|
|
|
|
# Silent when healthy — only show summary if there were failures or notable events
|
|
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $DURATION"
|
|
[[ ${#PASS[@]} -gt 0 ]] && log "Synced: ${PASS[*]}"
|
|
echo "$ICON_ERROR Failed: ${FAIL[*]}"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
|
"Critical Sync" "warning"
|
|
exit 1
|
|
else
|
|
log "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)"
|
|
fi
|
|
|
|
exit 0 |