Gmer4Lfe
2a062e5140
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
253 lines
11 KiB
Bash
Executable File
253 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Critical Sync Maintenance ======================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Orchestrator for time-sensitive syncs running every 30 minutes. Keeps the
|
|
# mirror current between the less frequent daily and weekly windows.
|
|
# Schedule: */30 * * * * (every 30 minutes via User Scripts plugin)
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# 1. Critical-Data rsync — auth stack, NPM config, certs (containers stopped both sides)
|
|
# 2. CRITICAL_MAINTENANCE_SCRIPTS — play_state_sync + any other per-window scripts
|
|
# 3. partnership --check — read both state files, detect changes, act accordingly
|
|
#
|
|
# RSYNC GATE
|
|
# RSYNC_ENABLED=false → skips all syncs (global gate)
|
|
# CRITICAL_RSYNC_ENABLED=false → skips critical syncs only (per-orchestrator gate)
|
|
# partnership --check always runs regardless — state check doesn't need rsync.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Silent When Healthy
|
|
# Runs 48 times per day — clean runs must produce zero output. Only failures
|
|
# and notable events produce visible output.
|
|
#
|
|
# Auth-First Window
|
|
# Auth stack changes (new users, proxy rules, certs) propagate within 30min.
|
|
# Emby watch states stay in sync — mirror users see correct playback position.
|
|
# Partnership state changes detected and acted on quickly.
|
|
#
|
|
# Strict Lock, Never Queue
|
|
# acquire_lock "strict" — if the previous 30-min run is still going, skip
|
|
# this cycle entirely. Critical-Data taking > 30min is a problem worth
|
|
# knowing about. Strict mode prevents pile-up without waiting.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root check — rsync and container stop/start require root
|
|
# acquire_lock "strict" — no pile-up; skip cycle if prior run still active
|
|
# detect_hosts() — MY_ID and REMOTE_ID for routing and logs
|
|
# resolve_remote_ip — confirms remote reachability before any transfer
|
|
# RSYNC_ENABLED gate — global kill switch respected before any rsync call
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# CRITICAL_RSYNC_ENABLED — enable/disable rsync section
|
|
# CRITICAL_SYNC_SHARES — shares synced every 30min (HOST*_CRITICAL_SYNC_SHARES)
|
|
# CRITICAL_MAINTENANCE_SCRIPTS — scripts run in critical window (optional)
|
|
# PARTNERSHIP_ENABLED — enable/disable partnership check
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# critical_sync_maintenance.sh
|
|
# Normal run.
|
|
#
|
|
# critical_sync_maintenance.sh --dry-run
|
|
# Preview syncs without transferring.
|
|
#
|
|
# critical_sync_maintenance.sh --log
|
|
# Verbose per-share output.
|
|
#
|
|
# critical_sync_maintenance.sh --status
|
|
# Show configuration and exit.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
acquire_lock "strict"
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
|
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
|
echo "$ICON_SYNC Critical enabled: ${CRITICAL_RSYNC_ENABLED:-false}"
|
|
echo "$ICON_SHIELD Partnership: ${PARTNERSHIP_ENABLED:-false}"
|
|
echo ""
|
|
echo "━━━ Critical Sync Shares ━━━"
|
|
if [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
|
warn " No CRITICAL_SYNC_SHARES configured"
|
|
else
|
|
for share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
|
[[ -z "$share" ]] && continue
|
|
SHARE_PATH="${share%%|*}"
|
|
SHARE_PROFILE="${share##*|}"
|
|
SHARE_NAME=$(basename "$SHARE_PATH")
|
|
[[ "$SHARE_PATH" == "$SHARE_PROFILE" ]] && \
|
|
echo " $ICON_SYNC $SHARE_NAME — no profile" || \
|
|
echo " $ICON_SYNC $SHARE_NAME — profile: $SHARE_PROFILE"
|
|
done
|
|
fi
|
|
echo ""
|
|
echo "━━━ Critical Maintenance Scripts ━━━"
|
|
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
|
echo " None configured"
|
|
else
|
|
for entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$entry" || "$entry" == \#* ]] && continue
|
|
echo " $ICON_GEAR $(basename "${entry%% *}")"
|
|
done
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Critical Shares Sync ━━━
|
|
# ==============================================================================================
|
|
START=$(date +%s)
|
|
RSYNC_OK=false
|
|
PASS=()
|
|
FAIL=()
|
|
|
|
log "$ICON_SYNC Critical shares (${#CRITICAL_SYNC_SHARES[@]}): $(for s in "${CRITICAL_SYNC_SHARES[@]}"; do printf '%s ' "$(basename "${s%%|*}")"; done)"
|
|
[[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]] && \
|
|
log "$ICON_GEAR Maintenance scripts: $(for s in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do printf '%s ' "$(basename "${s%% *}")"; done)"
|
|
|
|
if ! check_rsync_enabled "CRITICAL"; then
|
|
echo "Critical rsync disabled — skipping sync, running partnership check only"
|
|
elif [[ ${#CRITICAL_SYNC_SHARES[@]} -eq 0 ]]; then
|
|
warn "CRITICAL_RSYNC_ENABLED=true but CRITICAL_SYNC_SHARES is empty for $MY_ID"
|
|
warn "Check HOST*_CRITICAL_SYNC_SHARES in host*.conf"
|
|
else
|
|
echo "Critical sync — $MY_ID → $REMOTE_ID — $(date '+%H:%M:%S')"
|
|
|
|
# Build dry-run flag to pass through
|
|
RSYNC_DRY=""
|
|
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
|
|
|
for share in "${CRITICAL_SYNC_SHARES[@]}"; do
|
|
[[ -z "$share" ]] && continue
|
|
|
|
# Parse optional profile flag: "/path/to/share|profile-name"
|
|
SHARE_PATH="${share%%|*}"
|
|
SHARE_PROFILE="${share##*|}"
|
|
SHARE_NAME=$(basename "$SHARE_PATH")
|
|
|
|
SHARE_START=$(date +%s)
|
|
|
|
if [[ "$SHARE_PATH" == "$SHARE_PROFILE" ]]; then
|
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" $RSYNC_DRY
|
|
else
|
|
bash "$SCRIPT_DIR/../Rsync/rsync.sh" "$SHARE_PATH" \
|
|
--profile="$SHARE_PROFILE" $RSYNC_DRY
|
|
fi
|
|
|
|
RSYNC_EXIT=$?
|
|
SHARE_DUR=$(format_duration $(( $(date +%s) - SHARE_START )))
|
|
|
|
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
|
|
PASS+=("$SHARE_NAME")
|
|
echo "$SHARE_NAME — done in $SHARE_DUR ✅"
|
|
RSYNC_OK=true
|
|
else
|
|
FAIL+=("$SHARE_NAME")
|
|
error "$SHARE_NAME — failed after $SHARE_DUR (exit $RSYNC_EXIT)"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Critical Maintenance Scripts ━━━
|
|
# ==============================================================================================
|
|
JOB_PASS=()
|
|
JOB_FAIL=()
|
|
for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$script_entry" || "$script_entry" == \#* ]] && continue
|
|
run_orch_child "$script_entry"
|
|
done
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Partnership Check ━━━
|
|
# ==============================================================================================
|
|
if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
|
|
PARTNER_DRY=""
|
|
[[ "$DRY_RUN" == true ]] && PARTNER_DRY="--dry-run"
|
|
|
|
if [[ "$RSYNC_OK" == true ]]; then
|
|
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
|
--check --remote-seen $PARTNER_DRY
|
|
else
|
|
bash "$SCRIPT_DIR/../Partnership/partnership_manager.sh" \
|
|
--check --remote-unseen $PARTNER_DRY
|
|
fi
|
|
else
|
|
echo "Partnership disabled — skipping check"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
END=$(date +%s)
|
|
DURATION=$(format_duration $(( END - START )))
|
|
|
|
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
|
|
|
# Minimal one-liner when healthy — 30-min cadence, keep it quiet. Full detail on failure.
|
|
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $DURATION"
|
|
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
|
[[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed shares: ${FAIL[*]}"
|
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]} ${JOB_FAIL[*]}" \
|
|
"Critical Sync" "warning"
|
|
exit 1
|
|
else
|
|
echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s), ${#JOB_PASS[@]} job(s)"
|
|
fi
|
|
|
|
exit 0 |