Gmer4Lfe
2a062e5140
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
360 lines
14 KiB
Bash
Executable File
360 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Daily Sync Maintenance =========================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Daily maintenance window orchestrator — runs the full daily sequence in the
|
|
# correct order. Schedule: 0 1 * * * (1am daily via User Scripts plugin)
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Pre-sync:
|
|
# git_pull_execute.sh — pull latest scripts first, always
|
|
#
|
|
# Arr Sync (arr_sync.sh):
|
|
# Syncs Lidarr/Sonarr/Radarr libraries across all nodes bidirectionally.
|
|
# All nodes agree on tracked library before any files are transferred.
|
|
# Remote nodes that don't have an arr running are skipped gracefully.
|
|
#
|
|
# Rsync window (DAILY_SYNC_SHARES per host):
|
|
# HOST*_DAILY_SYNC_SHARES — media shares spread to all nodes (no --delete)
|
|
# HOST*_PERSONAL_SHARES — encrypted personal shares
|
|
#
|
|
# Post-sync maintenance (DAILY_MAINTENANCE_SCRIPTS):
|
|
# media_shares_permissions.sh — fix ownership before arr cleanup
|
|
# media_cleaner.sh anime/media — remove junk from media and anime shares
|
|
# lidarr/sonarr/radarr_cleanup.sh — remove orphaned files (local arr = truth)
|
|
# docker_daily_restart.sh — restart containers needing daily restart
|
|
#
|
|
# DRIVE TEMP HANDLING (rsync.sh exit codes):
|
|
# exit 1 = temp WARN → skip this share, continue to next
|
|
# exit 2 = temp CRITICAL → abort ALL remaining syncs in this window
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Order Is Load-Bearing
|
|
# git pull first — maintenance runs on latest code, not yesterday's. Arr sync
|
|
# before rsync — all nodes track the same library before files are spread,
|
|
# preventing remote arrs from searching for content already owned. Rsync before
|
|
# cleanup — cleanup sees fully spread state. Permissions before arr cleanup —
|
|
# arrs need correct ownership to delete/rename. Docker restart last — containers
|
|
# already processed by cleanup.
|
|
#
|
|
# Host-Aware Routing
|
|
# Bidirectional — same script runs on both servers, correct direction automatic.
|
|
# detect_hosts() aliases DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars.
|
|
# No manual HOST1/HOST2 comparisons needed.
|
|
#
|
|
# Silent When Healthy
|
|
# Runs daily at 1am — clean runs produce minimal output. Each job logs silently
|
|
# on success; failures surface to warn()/error(). Notify only on failure.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root check — rsync and docker operations require root
|
|
# acquire_lock — prevents concurrent daily windows
|
|
# detect_hosts() — aliases correct per-host share lists
|
|
# check_connectivity — verified before any rsync
|
|
# check_remote_rootfs — aborts rsync if remote rootfs nearly full
|
|
# Non-fatal jobs — a failed job logs and continues; remaining jobs still run
|
|
# notify on failure — successful daily run produces no notification
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# host*.conf
|
|
#
|
|
# HOST*_DAILY_SYNC_SHARES — shares pushed to mirror each day
|
|
# HOST*_PERSONAL_SHARES — encrypted personal shares
|
|
#
|
|
# master.conf
|
|
#
|
|
# DAILY_MAINTENANCE_SCRIPTS — maintenance jobs (permissions, cleanup, restart)
|
|
# DAILY_RSYNC_ENABLED — enable/disable rsync section
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# daily_sync_maintenance.sh
|
|
# Normal run.
|
|
#
|
|
# daily_sync_maintenance.sh --dry-run
|
|
# Preview without syncing or changing.
|
|
#
|
|
# daily_sync_maintenance.sh --log
|
|
# Verbose per-share/per-job output.
|
|
#
|
|
# daily_sync_maintenance.sh --status
|
|
# Show configured shares and jobs.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
if ! command -v docker &>/dev/null; then
|
|
error "Docker command not found"
|
|
exit 1
|
|
fi
|
|
|
|
detect_hosts
|
|
resolve_remote_ip
|
|
|
|
acquire_lock
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY DAILY SYNC STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
|
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
|
echo "$ICON_SYNC Daily enabled: ${DAILY_RSYNC_ENABLED:-false}"
|
|
echo ""
|
|
echo "━━━ Daily Sync Shares ━━━"
|
|
for share in "${DAILY_SYNC_SHARES[@]}"; do
|
|
[[ -n "$share" ]] && echo " $ICON_SYNC $share"
|
|
done
|
|
for share in "${PERSONAL_SHARES[@]}"; do
|
|
[[ -n "$share" ]] && echo " $ICON_SYNC $share (personal)"
|
|
done
|
|
echo ""
|
|
echo "━━━ Daily Maintenance Scripts ━━━"
|
|
for entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$entry" ]] && continue
|
|
echo " $ICON_GEAR ${entry##*/}"
|
|
done
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ── BUILD SHARE LIST via detect_hosts aliases ─────────────────────────────────────────────────
|
|
# detect_hosts() sets DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars
|
|
# No manual HOST1/HOST2 comparison needed — aliased automatically per server
|
|
# ==============================================================================================
|
|
ALL_SHARES=()
|
|
for share in "${DAILY_SYNC_SHARES[@]}"; do
|
|
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
|
done
|
|
for share in "${PERSONAL_SHARES[@]}"; do
|
|
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
|
done
|
|
SHARE_COUNT=${#ALL_SHARES[@]}
|
|
|
|
# ── Split maintenance scripts: git pull runs pre-sync, rest run post-sync ─────────────────────
|
|
PRE_SYNC_SCRIPTS=()
|
|
POST_SYNC_SCRIPTS=()
|
|
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$script_entry" ]] && continue
|
|
script_name=$(basename "${script_entry%% *}")
|
|
if [[ "$script_name" == "git_pull_execute.sh" ]]; then
|
|
PRE_SYNC_SCRIPTS+=("$script_entry")
|
|
else
|
|
POST_SYNC_SCRIPTS+=("$script_entry")
|
|
fi
|
|
done
|
|
|
|
WINDOW_START=$(date +%s)
|
|
JOB_PASS=()
|
|
JOB_FAIL=()
|
|
PASS=()
|
|
FAIL=()
|
|
SHARE_TIMES=()
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Daily Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Pre-sync — git pull ━━━
|
|
# ==============================================================================================
|
|
if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_GIT Pre-sync ━━━"
|
|
for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
|
|
run_orch_child "$script_entry"
|
|
done
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Arr Sync — library reconciliation before file spreading ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
|
|
|
ARR_SYNC_SCRIPT="$ECOSYSTEM_ROOT/Arrs_Stack/arr_sync.sh"
|
|
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
|
echo "ARR_SYNC_ENABLED=false — skipping"
|
|
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
|
warn "arr_sync.sh not found at $ARR_SYNC_SCRIPT — skipping"
|
|
JOB_FAIL+=("arr_sync.sh")
|
|
else
|
|
_arr_sync_args=()
|
|
[[ "$DRY_RUN" == true ]] && _arr_sync_args+=("--dry-run")
|
|
if bash "$ARR_SYNC_SCRIPT" "${_arr_sync_args[@]}"; then
|
|
echo "Arr sync complete ✅"
|
|
JOB_PASS+=("arr_sync.sh")
|
|
else
|
|
warn "Arr sync completed with errors — continuing to rsync"
|
|
JOB_FAIL+=("arr_sync.sh")
|
|
fi
|
|
unset _arr_sync_args
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Media Share Sync ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Media Share Sync — $SHARE_COUNT share(s) ━━━"
|
|
|
|
TOTAL_START=$(date +%s)
|
|
SHARE_INDEX=0
|
|
ABORT_ALL_SYNCS=false
|
|
|
|
if ! check_rsync_enabled "DAILY"; then
|
|
warn "Daily rsync disabled — skipping all $SHARE_COUNT share syncs"
|
|
warn "Proceeding to maintenance jobs..."
|
|
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
|
|
warn "No shares configured for $MY_ID — check HOST*_DAILY_SYNC_SHARES in host*.conf"
|
|
else
|
|
# Pre-flight — connectivity then remote rootfs
|
|
check_connectivity
|
|
check_remote_rootfs
|
|
|
|
RSYNC_DRY=""
|
|
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
|
|
|
for SHARE in "${ALL_SHARES[@]}"; do
|
|
(( SHARE_INDEX++ ))
|
|
SHARE_NAME=$(basename "$SHARE")
|
|
SHARE_START=$(date +%s)
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"
|
|
|
|
if [[ "$ABORT_ALL_SYNCS" == true ]]; then
|
|
warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
|
|
FAIL+=("$SHARE_NAME:temp-critical")
|
|
continue
|
|
fi
|
|
|
|
bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
|
|
RSYNC_EXIT=$?
|
|
|
|
SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")
|
|
|
|
case "$RSYNC_EXIT" in
|
|
0)
|
|
PASS+=("$SHARE_NAME")
|
|
echo "$SHARE_NAME — done ✅"
|
|
;;
|
|
1)
|
|
FAIL+=("$SHARE_NAME:temp-warn")
|
|
warn "$SHARE_NAME skipped — drive temps too high"
|
|
;;
|
|
2)
|
|
FAIL+=("$SHARE_NAME:temp-critical")
|
|
ABORT_ALL_SYNCS=true
|
|
error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
|
|
notify "Daily sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
|
|
"Daily Sync" "warning"
|
|
;;
|
|
*)
|
|
FAIL+=("$SHARE_NAME")
|
|
error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
|
|
;;
|
|
esac
|
|
|
|
done
|
|
fi
|
|
|
|
TOTAL_END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Post-sync Maintenance Jobs ━━━
|
|
# ==============================================================================================
|
|
if [[ ${#POST_SYNC_SCRIPTS[@]} -gt 0 ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━"
|
|
for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
|
|
echo ""
|
|
run_orch_child "$script_entry"
|
|
done
|
|
fi
|
|
|
|
WINDOW_END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
|
echo ""
|
|
|
|
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
|
for entry in "${SHARE_TIMES[@]}"; do
|
|
sname="${entry%%:*}"
|
|
sdur="${entry##*:}"
|
|
if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
|
|
echo " $ICON_ERROR $sname — $(format_duration "$sdur")"
|
|
else
|
|
echo " $ICON_DONE $sname — $(format_duration "$sdur")"
|
|
fi
|
|
done
|
|
[[ "$SHARE_COUNT" -eq 0 || "${DAILY_RSYNC_ENABLED:-false}" == "false" ]] && \
|
|
echo " (rsync disabled)"
|
|
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
|
echo ""
|
|
|
|
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|
echo "$ICON_GEAR Jobs:"
|
|
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
|
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
|
echo ""
|
|
fi
|
|
|
|
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
|
|
|
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
|
warn "Status: $TOTAL_FAIL failure(s)"
|
|
notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
|
"Daily Maintenance" "warning"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 1
|
|
else
|
|
echo "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi |