Gmer4Lfe
d5cf3db2ec
A gated-off section left nothing failed, so the weekly could run for hours and report "all complete" beside "0 shares synced"; skipped is now derived from what was expected rather than self-reported, and the verdict degrades to PARTIAL instead of flattering.
365 lines
15 KiB
Bash
Executable File
365 lines
15 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# =========================== Intermediate Sync Maintenance ====================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# 4-hour orchestrator — arr library reconciliation, artwork fetching, and
|
|
# optional rsync. Schedule: 0 */4 * * * (every 4 hours)
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# 1. conf_sync.sh — refresh partner conf cache in RAM (/tmp/varaverk/conf/),
|
|
# both directions: pull theirs, push ours
|
|
# 2. arr_sync.sh — sync Lidarr/Sonarr/Radarr libraries across all nodes
|
|
# 3. Rsync window (optional) — INTERMEDIATE_SYNC_SHARES, if any configured
|
|
# 4. INTERMEDIATE_MAINTENANCE_SCRIPTS — artwork fetch and any future 4-hour jobs
|
|
#
|
|
# DRIVE TEMP HANDLING (rsync.sh exit codes):
|
|
# exit 1 = temp WARN → skip this share, continue to next
|
|
# exit 2 = temp CRITICAL → abort ALL remaining syncs in this window
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Closes the Library Gap
|
|
# Arr libraries need to converge more frequently than once a day. If a remote
|
|
# node adds something at 2am, the next daily window is 23 hours away — remote
|
|
# arrs search for content they don't know is already owned. Running every 4
|
|
# hours closes that gap.
|
|
#
|
|
# Optional Rsync Layer
|
|
# INTERMEDIATE_SYNC_SHARES is empty by default — the rsync step is skipped
|
|
# entirely when nothing is configured. Add shares only if a subset of data
|
|
# needs mid-day propagation. Full media share sync stays in the daily window.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root Enforcement
|
|
# Every script called from here requires root.
|
|
#
|
|
# Lock Acquisition
|
|
# acquire_lock prevents concurrent intermediate windows.
|
|
#
|
|
# Host Detection
|
|
# detect_hosts() aliases the correct per-host share lists.
|
|
#
|
|
# Empty Job List Guard
|
|
# Exits with an error and a notification if INTERMEDIATE_MAINTENANCE_SCRIPTS is empty,
|
|
# rather than running six no-op windows a day that all report success.
|
|
#
|
|
# Connectivity Check
|
|
# check_connectivity is verified before any rsync, and skipped entirely when no shares
|
|
# are configured — there is nothing to reach a partner for.
|
|
#
|
|
# Remote Rootfs Check
|
|
# check_remote_rootfs aborts rsync if the remote rootfs is nearly full.
|
|
#
|
|
# Drive Temperature Escalation
|
|
# rsync.sh's exit code is honoured per share: exit 1 skips that share, exit 2 aborts
|
|
# every remaining sync in the window and notifies.
|
|
#
|
|
# Non-Fatal Jobs
|
|
# A failing arr_sync warns but does not block rsync or the artwork fetch that follow it.
|
|
#
|
|
# Minimal on Success
|
|
# Runs six times a day, so the full breakdown only prints on failure or with --log.
|
|
# A quiet run is the normal outcome and should not fill the log.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_INTERMEDIATE_SYNC_SHARES — shares synced mid-day (empty = rsync skipped)
|
|
#
|
|
# master.conf
|
|
#
|
|
# INTERMEDIATE_RSYNC_ENABLED — enable/disable rsync section (default: true)
|
|
# INTERMEDIATE_MAINTENANCE_SCRIPTS — jobs run after rsync
|
|
# ARR_SYNC_ENABLED — toggle inside arr_sync.sh
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# intermediate_sync_maintenance.sh
|
|
# Normal run.
|
|
#
|
|
# intermediate_sync_maintenance.sh --dry-run
|
|
# Preview without changes.
|
|
#
|
|
# intermediate_sync_maintenance.sh --log
|
|
# Verbose per-job output.
|
|
#
|
|
# intermediate_sync_maintenance.sh --status
|
|
# Show configured shares/jobs and exit.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
acquire_lock
|
|
|
|
detect_hosts
|
|
|
|
# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
|
|
# from a healthy run. Fail loudly instead of silently doing no work.
|
|
if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
|
error "INTERMEDIATE_MAINTENANCE_SCRIPTS is empty — no intermediate maintenance scripts will run"
|
|
error "Check INTERMEDIATE_MAINTENANCE_SCRIPTS in master.conf"
|
|
notify "intermediate maintenance scripts skipped on $(hostname) ($MY_ID) — INTERMEDIATE_MAINTENANCE_SCRIPTS is empty" \
|
|
"$(basename "$0" .sh)" "warning"
|
|
exit 1
|
|
fi
|
|
resolve_remote_ip
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
|
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
|
echo "$ICON_SYNC Interm. enabled: ${INTERMEDIATE_RSYNC_ENABLED:-true}"
|
|
echo "$ICON_GEAR Arr sync: ${ARR_SYNC_ENABLED:-true}"
|
|
echo ""
|
|
echo "━━━ Intermediate Sync Shares ━━━"
|
|
if [[ ${#INTERMEDIATE_SYNC_SHARES[@]} -eq 0 ]]; then
|
|
echo " None configured — add to INTERMEDIATE_SYNC_SHARES in master.conf to enable"
|
|
else
|
|
for share in "${INTERMEDIATE_SYNC_SHARES[@]}"; do
|
|
[[ -n "$share" ]] && echo " $ICON_SYNC $(basename "$share") ($share)"
|
|
done
|
|
fi
|
|
echo ""
|
|
echo "━━━ Intermediate Maintenance Scripts ━━━"
|
|
if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -eq 0 ]]; then
|
|
echo " None configured"
|
|
else
|
|
for entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -n "$entry" ]] && echo " $ICON_GEAR ${entry##*/}"
|
|
done
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
WINDOW_START=$(date +%s)
|
|
JOB_PASS=()
|
|
JOB_FAIL=()
|
|
PASS=()
|
|
FAIL=()
|
|
SHARE_TIMES=()
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Conf Sync ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Conf Sync ━━━"
|
|
|
|
# Full sync, not --pull-only. The push half was written as an event-driven fast path for the
|
|
# conf-save hook, but no such hook was ever built — so outside array start nothing pushed this
|
|
# host's conf to its partners at all, and a partner's copy of our conf stayed at whatever it was
|
|
# when we last rebooted. Pull alone kept our view of them fresh while their view of us decayed.
|
|
CONF_SYNC_SCRIPT="$ECOSYSTEM_ROOT/System_Essentials/conf_sync.sh"
|
|
if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
|
|
warn "conf_sync.sh not found — skipping partner conf refresh"
|
|
else
|
|
_conf_args=()
|
|
[[ "$DRY_RUN" == true ]] && _conf_args+=("--dry-run")
|
|
if bash "$CONF_SYNC_SCRIPT" "${_conf_args[@]}"; then
|
|
echo "Partner conf cache refreshed ✅"
|
|
JOB_PASS+=("conf_sync.sh")
|
|
else
|
|
warn "Partner conf sync failed — cache may be stale"
|
|
JOB_FAIL+=("conf_sync.sh")
|
|
fi
|
|
unset _conf_args
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Arr Sync ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
|
|
|
ARR_SYNC_SCRIPT="$ECOSYSTEM_ROOT/Arrs_Stack/arr_sync.sh"
|
|
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
|
echo "ARR_SYNC_ENABLED=false — skipping"
|
|
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
|
warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
|
|
JOB_FAIL+=("arr_sync.sh")
|
|
else
|
|
_arr_sync_args=()
|
|
[[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
|
|
if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
|
|
echo "Arr sync complete ✅"
|
|
JOB_PASS+=("arr_sync.sh")
|
|
else
|
|
warn "Arr sync completed with errors — continuing"
|
|
JOB_FAIL+=("arr_sync.sh")
|
|
fi
|
|
unset _arr_sync_args
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Rsync (optional) ━━━
|
|
# ==============================================================================================
|
|
SHARE_COUNT=${#INTERMEDIATE_SYNC_SHARES[@]}
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Mid-day Share Sync — $SHARE_COUNT share(s) ━━━"
|
|
|
|
TOTAL_START=$(date +%s)
|
|
SHARE_INDEX=0
|
|
ABORT_ALL_SYNCS=false
|
|
|
|
if [[ "$SHARE_COUNT" -eq 0 ]]; then
|
|
echo "No INTERMEDIATE_SYNC_SHARES configured — skipping"
|
|
echo "Add shares to INTERMEDIATE_SYNC_SHARES in master.conf to enable mid-day sync"
|
|
elif ! check_rsync_enabled "INTERMEDIATE"; then
|
|
warn "Intermediate rsync disabled — skipping all $SHARE_COUNT share sync(s)"
|
|
else
|
|
check_connectivity
|
|
check_remote_rootfs
|
|
|
|
RSYNC_DRY=""
|
|
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
|
|
|
for SHARE in "${INTERMEDIATE_SYNC_SHARES[@]}"; do
|
|
(( SHARE_INDEX++ ))
|
|
SHARE_NAME=$(basename "$SHARE")
|
|
SHARE_START=$(date +%s)
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"
|
|
|
|
if [[ "$ABORT_ALL_SYNCS" == true ]]; then
|
|
warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
|
|
FAIL+=("$SHARE_NAME:temp-critical")
|
|
continue
|
|
fi
|
|
|
|
bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
|
|
RSYNC_EXIT=$?
|
|
|
|
SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")
|
|
|
|
case "$RSYNC_EXIT" in
|
|
0)
|
|
PASS+=("$SHARE_NAME")
|
|
echo "$SHARE_NAME — done ✅"
|
|
;;
|
|
1)
|
|
FAIL+=("$SHARE_NAME:temp-warn")
|
|
warn "$SHARE_NAME skipped — drive temps too high"
|
|
;;
|
|
2)
|
|
FAIL+=("$SHARE_NAME:temp-critical")
|
|
ABORT_ALL_SYNCS=true
|
|
error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
|
|
notify "Intermediate sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
|
|
"Intermediate Sync" "warning"
|
|
;;
|
|
*)
|
|
FAIL+=("$SHARE_NAME")
|
|
error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
|
|
;;
|
|
esac
|
|
done
|
|
fi
|
|
|
|
TOTAL_END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Maintenance Jobs ━━━
|
|
# ==============================================================================================
|
|
if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Maintenance Jobs ━━━"
|
|
for script_entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$script_entry" ]] && continue
|
|
echo ""
|
|
run_orch_child "$script_entry"
|
|
done
|
|
fi
|
|
|
|
WINDOW_END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
|
|
|
# Full breakdown on failure or --log; minimal one-liner otherwise (4-hour cadence — keep it quiet).
|
|
SHOW_FULL=false
|
|
[[ "$TOTAL_FAIL" -gt 0 || "$ENABLE_LOGGING" == true ]] && SHOW_FULL=true
|
|
|
|
if [[ "$SHOW_FULL" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
|
echo ""
|
|
|
|
if [[ "$SHARE_COUNT" -gt 0 ]]; then
|
|
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
|
for entry in "${SHARE_TIMES[@]}"; do
|
|
sname="${entry%%:*}"
|
|
sdur="${entry##*:}"
|
|
if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
|
|
echo " $ICON_ERROR $sname — $(format_duration "$sdur")"
|
|
else
|
|
echo " $ICON_DONE $sname — $(format_duration "$sdur")"
|
|
fi
|
|
done
|
|
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
|
echo ""
|
|
fi
|
|
|
|
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|
echo "$ICON_GEAR Jobs:"
|
|
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
|
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
|
echo ""
|
|
fi
|
|
fi
|
|
|
|
# Standard ending, quiet mode — 4-hour cadence, so an OK cycle is one parseable line and
|
|
# anything skipped or failed expands to the full block on its own.
|
|
_mode=quiet; [[ "$ENABLE_LOGGING" == true ]] && _mode=full
|
|
orchestrator_summary "INTERMEDIATE SYNC" "$WINDOW_START" "Intermediate Sync" "$_mode"
|
|
exit $?
|