Gmer4Lfe
95151c2278
Move all watchdog scripts to a dedicated Watchdogs/ folder: Docker_Essentials/docker_watchdog.sh → Watchdogs/ unRAID_Essentials/system_watchdog.sh → Watchdogs/ unRAID_Essentials/resource_watchdog.sh → Watchdogs/ Orchestrators/watchdog_orchestrator.sh → Watchdogs/ Tools/watchdog_skip_list_manager.sh → Watchdogs/ Rename host config files: master_host1.conf → host1.conf master_host2.conf → host2.conf Update all references across the ecosystem: master.conf: WATCHDOG_ORCHESTRATOR_SCRIPTS paths → Watchdogs/ load_config.sh: host*.conf glob + all comments git_pull_execute.sh: sparse checkout glob + all comments Partnership/ssh_setup.sh: HOST_CONF path construction user_script_plug-in.sh: all script paths + per-host conf path common.sh, README.md, README-User_Script_Plug-in.md: comment refs All Partnership, Fallback, Monitors, Transcodes, Tools scripts: comment refs
232 lines
11 KiB
Bash
232 lines
11 KiB
Bash
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Critical Sync Maintenance ======================================
|
|
# ==============================================================================================
|
|
# Orchestrator for time-sensitive syncs that run every 15 minutes.
|
|
# Keeps the mirror current between the less frequent daily and weekly windows.
|
|
# Schedule: */15 * * * * (every 15 minutes via User Scripts plugin)
|
|
#
|
|
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
|
# 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides)
|
|
# 2. emby-fallback rsync — dirty Emby sync (watch states, library — Emby stays running)
|
|
# 3. CRITICAL_MAINTENANCE_SCRIPTS — any scripts configured for critical window
|
|
# 4. partnership --check — read both state files, detect changes, act accordingly
|
|
#
|
|
# ── WHY EVERY 15 MINUTES ──────────────────────────────────────────────────────────────────────
|
|
# Auth stack changes (new users, proxy rules, certs) propagate within 15min ✅
|
|
# Emby watch states stay in sync — mirror users see correct playback position ✅
|
|
# Partnership state changes detected and acted on quickly ✅
|
|
# Lock prevents: daily rsync doing Critical-Data mid-critical window ✅
|
|
#
|
|
# ── RSYNC GATE ────────────────────────────────────────────────────────────────────────────────
|
|
# RSYNC_ENABLED=false → skips all syncs (global gate)
|
|
# CRITICAL_RSYNC_ENABLED=false → skips critical syncs only (per-orchestrator gate)
|
|
# partnership --check always runs regardless — state check doesn't need rsync
|
|
#
|
|
# ── LOCK BEHAVIOUR ────────────────────────────────────────────────────────────────────────────
|
|
# acquire_lock "strict" — if previous 15min run still going, skip this cycle entirely
|
|
# Critical-Data taking > 15min is a problem worth knowing about
|
|
# Strict mode prevents pile-up without waiting — log and move on ✅
|
|
#
|
|
# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
|
|
# Runs 96 times per day — clean runs must produce zero output ✅
|
|
# Only failures and notable events produce visible output
|
|
#
|
|
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
|
# CRITICAL_RSYNC_ENABLED — enable/disable rsync section
|
|
# CRITICAL_SYNC_SHARES — shares synced every 15min (HOST*_CRITICAL_SYNC_SHARES)
|
|
# CRITICAL_MAINTENANCE_SCRIPTS — scripts run in critical window (optional)
|
|
# PARTNERSHIP_ENABLED — enable/disable partnership check
|
|
#
|
|
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
|
# critical_sync_maintenance.sh — normal run
|
|
# critical_sync_maintenance.sh --dry-run — preview syncs without transferring
|
|
# critical_sync_maintenance.sh --log — verbose per-share output
|
|
# critical_sync_maintenance.sh --status — show configuration and exit
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
validate_unraid_cmd \
|
|
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
|
"" "" \
|
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
|
|
|
acquire_lock "strict"
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
|
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
|
echo "$ICON_SYNC Critical enabled: ${CRITICAL_RSYNC_ENABLED:-false}"
|
|
echo "$ICON_SHIELD Partnership: ${PARTNERSHIP_ENABLED:-false}"
|
|
echo ""
|
|
echo "━━━ Critical Sync Shares ━━━"
|
|
if [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
|
warn " No CRITICAL_SYNC_SHARES configured"
|
|
else
|
|
for share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
|
[[ -z "$share" ]] && continue
|
|
SHARE_PATH="${share%%|*}"
|
|
SHARE_PROFILE="${share##*|}"
|
|
SHARE_NAME=$(basename "$SHARE_PATH")
|
|
[[ "$SHARE_PATH" == "$SHARE_PROFILE" ]] && \
|
|
echo " $ICON_SYNC $SHARE_NAME — no profile" || \
|
|
echo " $ICON_SYNC $SHARE_NAME — profile: $SHARE_PROFILE"
|
|
done
|
|
fi
|
|
echo ""
|
|
echo "━━━ Critical Maintenance Scripts ━━━"
|
|
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
|
echo " None configured"
|
|
else
|
|
for entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$entry" || "$entry" == \#* ]] && continue
|
|
echo " $ICON_GEAR $(basename "${entry%% *}")"
|
|
done
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Critical Shares Sync ━━━
|
|
# ==============================================================================================
|
|
START=$(date +%s)
|
|
RSYNC_OK=false
|
|
PASS=()
|
|
FAIL=()
|
|
|
|
if ! check_rsync_enabled "CRITICAL"; then
|
|
echo "Critical rsync disabled — skipping sync, running partnership check only"
|
|
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
|
warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID"
|
|
warn "Check HOST*_CRITICAL_SYNC_SHARES in host*.conf"
|
|
else
|
|
echo "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')"
|
|
|
|
# Build dry-run flag to pass through
|
|
RSYNC_DRY=""
|
|
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
|
|
|
for share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
|
[[ -z "$share" ]] && continue
|
|
|
|
# Parse optional profile flag: "/path/to/share|profile-name"
|
|
SHARE_PATH="${share%%|*}"
|
|
SHARE_PROFILE="${share##*|}"
|
|
SHARE_NAME=$(basename "$SHARE_PATH")
|
|
|
|
SHARE_START=$(date +%s)
|
|
|
|
if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then
|
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" $RSYNC_DRY
|
|
else
|
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" \
|
|
--profile="$SHARE_PROFILE" $RSYNC_DRY
|
|
fi
|
|
|
|
RSYNC_EXIT=$?
|
|
SHARE_DUR=$(format_duration $(( $(date +%s) - SHARE_START )))
|
|
|
|
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
|
|
PASS+=("$SHARE_NAME")
|
|
log "$SHARE_NAME — done in $SHARE_DUR ✅"
|
|
RSYNC_OK=true
|
|
else
|
|
FAIL+=("$SHARE_NAME")
|
|
error "$SHARE_NAME — failed after $SHARE_DUR (exit $RSYNC_EXIT)"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Critical Maintenance Scripts ━━━
|
|
# ==============================================================================================
|
|
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
|
for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$script_entry" || "$script_entry" == \#* ]] && continue
|
|
|
|
SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
|
|
SCRIPT_ARGS="${script_entry#* }"
|
|
[[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
|
|
[[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run"
|
|
|
|
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
|
|
|
|
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
|
warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
|
|
continue
|
|
fi
|
|
|
|
log "Running: $SCRIPT_NAME"
|
|
bash "$SCRIPT_PATH" $SCRIPT_ARGS
|
|
EXIT_CODE=$?
|
|
[[ "$EXIT_CODE" -ne 0 ]] && \
|
|
warn "$SCRIPT_NAME exited with code $EXIT_CODE"
|
|
done
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Partnership Check ━━━
|
|
# ==============================================================================================
|
|
if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
|
|
PARTNER_DRY=""
|
|
[[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run"
|
|
|
|
if [[ "$RSYNC_OK" == true ]]; then
|
|
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
|
--check --remote-seen $PARTNER_DRY
|
|
else
|
|
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
|
--check --remote-unseen $PARTNER_DRY
|
|
fi
|
|
else
|
|
echo "Partnership disabled — skipping check"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
END=$(date +%s)
|
|
DURATION=$(format_duration $(( END - START )))
|
|
|
|
# Silent when healthy — only show summary if there were failures or notable events
|
|
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $DURATION"
|
|
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
|
echo "$ICON_ERROR Failed: ${FAIL[*]}"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
|
"Critical Sync" "warning"
|
|
exit 1
|
|
else
|
|
echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)"
|
|
fi
|
|
|
|
exit 0 |