Gmer4Lfe
2a062e5140
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
343 lines
14 KiB
Bash
Executable File
343 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# =========================== Intermediate Sync Maintenance ====================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# 4-hour orchestrator — arr library reconciliation, artwork fetching, and
|
|
# optional rsync. Schedule: 0 */4 * * * (every 4 hours)
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# 1. conf_sync.sh --pull-only — refresh partner conf cache in RAM (/tmp/.cache/vv/d/)
|
|
# 2. arr_sync.sh — sync Lidarr/Sonarr/Radarr libraries across all nodes
|
|
# 3. Rsync window (optional) — INTERMEDIATE_SYNC_SHARES, if any configured
|
|
# 4. INTERMEDIATE_MAINTENANCE_SCRIPTS — artwork fetch and any future 4-hour jobs
|
|
#
|
|
# DRIVE TEMP HANDLING (rsync.sh exit codes):
|
|
# exit 1 = temp WARN → skip this share, continue to next
|
|
# exit 2 = temp CRITICAL → abort ALL remaining syncs in this window
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Closes the Library Gap
|
|
# Arr libraries need to converge more frequently than once a day. If a remote
|
|
# node adds something at 2am, the next daily window is 23 hours away — remote
|
|
# arrs search for content they don't know is already owned. Running every 4
|
|
# hours closes that gap.
|
|
#
|
|
# Idempotent Artwork
|
|
# lidarr_missing_art.sh skips existing files and runs fast after initial fill.
|
|
# Pairing it here means artwork catches up within 4 hours of a new album landing.
|
|
#
|
|
# Optional Rsync Layer
|
|
# INTERMEDIATE_SYNC_SHARES is empty by default — the rsync step is skipped
|
|
# entirely when nothing is configured. Add shares only if a subset of data
|
|
# needs mid-day propagation. Full media share sync stays in the daily window.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root check — scripts called here require root
|
|
# acquire_lock — prevents concurrent intermediate windows
|
|
# detect_hosts() — aliases correct per-host share lists
|
|
# check_connectivity — verified before any rsync (skipped if no shares)
|
|
# check_remote_rootfs — aborts rsync if remote rootfs nearly full
|
|
# Non-fatal jobs — a failed arr_sync warns but does not block rsync or artwork fetch
|
|
# Minimal on success — runs 6x/day; full breakdown only on failure or --log
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_INTERMEDIATE_SYNC_SHARES — shares synced mid-day (empty = rsync skipped)
|
|
#
|
|
# master.conf
|
|
#
|
|
# INTERMEDIATE_RSYNC_ENABLED — enable/disable rsync section (default: true)
|
|
# INTERMEDIATE_MAINTENANCE_SCRIPTS — jobs run after rsync
|
|
# ARR_SYNC_ENABLED — toggle inside arr_sync.sh
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# intermediate_sync_maintenance.sh
|
|
# Normal run.
|
|
#
|
|
# intermediate_sync_maintenance.sh --dry-run
|
|
# Preview without changes.
|
|
#
|
|
# intermediate_sync_maintenance.sh --log
|
|
# Verbose per-job output.
|
|
#
|
|
# intermediate_sync_maintenance.sh --status
|
|
# Show configured shares/jobs and exit.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
acquire_lock
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
|
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
|
echo "$ICON_SYNC Interm. enabled: ${INTERMEDIATE_RSYNC_ENABLED:-true}"
|
|
echo "$ICON_GEAR Arr sync: ${ARR_SYNC_ENABLED:-true}"
|
|
echo ""
|
|
echo "━━━ Intermediate Sync Shares ━━━"
|
|
if [[ ${#INTERMEDIATE_SYNC_SHARES[@]} -eq 0 ]]; then
|
|
echo " None configured — add to INTERMEDIATE_SYNC_SHARES in master.conf to enable"
|
|
else
|
|
for share in "${INTERMEDIATE_SYNC_SHARES[@]}"; do
|
|
[[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)"
|
|
done
|
|
fi
|
|
echo ""
|
|
echo "━━━ Intermediate Maintenance Scripts ━━━"
|
|
if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
|
echo " None configured"
|
|
else
|
|
for entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -n "$entry" ]] && echo " $ICON_GEAR ${entry##*/}"
|
|
done
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
WINDOW_START=$(date +%s)
|
|
JOB_PASS=()
|
|
JOB_FAIL=()
|
|
PASS=()
|
|
FAIL=()
|
|
SHARE_TIMES=()
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Conf Pull ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Conf Pull ━━━"
|
|
|
|
CONF_SYNC_SCRIPT="$ECOSYSTEM_ROOT/System_Essentials/conf_sync.sh"
|
|
if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
|
|
warn "conf_sync.sh not found — skipping partner conf refresh"
|
|
else
|
|
_conf_args=("--pull-only")
|
|
[[ "$DRY_RUN" == true ]] && _conf_args+=("--dry-run")
|
|
if bash "$CONF_SYNC_SCRIPT" "${_conf_args[@]}"; then
|
|
echo "Partner conf cache refreshed ✅"
|
|
JOB_PASS+=("conf_sync.sh --pull-only")
|
|
else
|
|
warn "Partner conf pull failed — cache may be stale"
|
|
JOB_FAIL+=("conf_sync.sh --pull-only")
|
|
fi
|
|
unset _conf_args
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Arr Sync ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
|
|
|
ARR_SYNC_SCRIPT="$ECOSYSTEM_ROOT/Arrs_Stack/arr_sync.sh"
|
|
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
|
echo "ARR_SYNC_ENABLED=false — skipping"
|
|
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
|
warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
|
|
JOB_FAIL+=("arr_sync.sh")
|
|
else
|
|
_arr_sync_args=()
|
|
[[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
|
|
if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
|
|
echo "Arr sync complete ✅"
|
|
JOB_PASS+=("arr_sync.sh")
|
|
else
|
|
warn "Arr sync completed with errors — continuing"
|
|
JOB_FAIL+=("arr_sync.sh")
|
|
fi
|
|
unset _arr_sync_args
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Rsync (optional) ━━━
|
|
# ==============================================================================================
|
|
SHARE_COUNT=${#INTERMEDIATE_SYNC_SHARES[@]}
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Mid-day Share Sync — $SHARE_COUNT share(s) ━━━"
|
|
|
|
TOTAL_START=$(date +%s)
|
|
SHARE_INDEX=0
|
|
ABORT_ALL_SYNCS=false
|
|
|
|
if [[ "$SHARE_COUNT" -eq 0 ]]; then
|
|
echo "No INTERMEDIATE_SYNC_SHARES configured — skipping"
|
|
echo "Add shares to INTERMEDIATE_SYNC_SHARES in master.conf to enable mid-day sync"
|
|
elif ! check_rsync_enabled "INTERMEDIATE"; then
|
|
warn "Intermediate rsync disabled — skipping all $SHARE_COUNT share sync(s)"
|
|
else
|
|
check_connectivity
|
|
check_remote_rootfs
|
|
|
|
RSYNC_DRY=""
|
|
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
|
|
|
for SHARE in "${INTERMEDIATE_SYNC_SHARES[@]}"; do
|
|
(( SHARE_INDEX++ ))
|
|
SHARE_NAME=$(basename "$SHARE")
|
|
SHARE_START=$(date +%s)
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"
|
|
|
|
if [[ "$ABORT_ALL_SYNCS" == true ]]; then
|
|
warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
|
|
FAIL+=("$SHARE_NAME:temp-critical")
|
|
continue
|
|
fi
|
|
|
|
bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
|
|
RSYNC_EXIT=$?
|
|
|
|
SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")
|
|
|
|
case "$RSYNC_EXIT" in
|
|
0)
|
|
PASS+=("$SHARE_NAME")
|
|
echo "$SHARE_NAME — done ✅"
|
|
;;
|
|
1)
|
|
FAIL+=("$SHARE_NAME:temp-warn")
|
|
warn "$SHARE_NAME skipped — drive temps too high"
|
|
;;
|
|
2)
|
|
FAIL+=("$SHARE_NAME:temp-critical")
|
|
ABORT_ALL_SYNCS=true
|
|
error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
|
|
notify "Intermediate sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
|
|
"Intermediate Sync" "warning"
|
|
;;
|
|
*)
|
|
FAIL+=("$SHARE_NAME")
|
|
error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
|
|
;;
|
|
esac
|
|
done
|
|
fi
|
|
|
|
TOTAL_END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Maintenance Jobs ━━━
|
|
# ==============================================================================================
|
|
if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Maintenance Jobs ━━━"
|
|
for script_entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$script_entry" ]] && continue
|
|
echo ""
|
|
run_orch_child "$script_entry"
|
|
done
|
|
fi
|
|
|
|
WINDOW_END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
|
|
|
# Full breakdown on failure or --log; minimal one-liner otherwise (4-hour cadence — keep it quiet).
|
|
SHOW_FULL=false
|
|
[[ "$TOTAL_FAIL" -gt 0 || "$ENABLE_LOGGING" == true ]] && SHOW_FULL=true
|
|
|
|
if [[ "$SHOW_FULL" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
|
echo ""
|
|
|
|
if [[ "$SHARE_COUNT" -gt 0 ]]; then
|
|
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
|
for entry in "${SHARE_TIMES[@]}"; do
|
|
sname="${entry%%:*}"
|
|
sdur="${entry##*:}"
|
|
if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
|
|
echo " $ICON_ERROR $sname — $(format_duration "$sdur")"
|
|
else
|
|
echo " $ICON_DONE $sname — $(format_duration "$sdur")"
|
|
fi
|
|
done
|
|
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
|
echo ""
|
|
fi
|
|
|
|
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|
echo "$ICON_GEAR Jobs:"
|
|
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
|
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
|
echo ""
|
|
fi
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no changes made"
|
|
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
|
if [[ "$SHOW_FULL" == true ]]; then
|
|
echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
|
|
else
|
|
echo "$ICON_DONE Intermediate sync — ${#JOB_PASS[@]} job(s), ${#PASS[@]}/$SHARE_COUNT share(s) ($(format_duration $(( WINDOW_END - WINDOW_START ))))"
|
|
fi
|
|
else
|
|
warn "Status: $TOTAL_FAIL failure(s)"
|
|
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
|
"Intermediate Sync" "warning"
|
|
fi
|
|
[[ "$SHOW_FULL" == true ]] && echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
|
exit 0
|