massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
@@ -1,81 +1,139 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Daily Sync Maintenance ------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ============================= Daily Sync Maintenance =========================================
|
||||
# ==============================================================================================
|
||||
# Daily orchestrator — runs the full daily maintenance window in the correct order.
|
||||
# Schedule: 0 1 * * * (1am daily)
|
||||
# Schedule: 0 1 * * * (1am daily via User Scripts plugin)
|
||||
#
|
||||
# Execution order:
|
||||
# 1. git_pull_execute.sh — pull latest scripts first, always
|
||||
# 2. Media share sync — HOST*_DAILY_SYNC_SHARES pushed to remote
|
||||
# 3. HOST*_PERSONAL_SHARES — personal encrypted shares after media
|
||||
# 4. media_shares_permissions.sh — fix ownership before arr cleanup
|
||||
# 5. media_cleaner.sh anime — remove junk from anime shares
|
||||
# 6. media_cleaner.sh media — remove junk from media shares
|
||||
# 7. lidarr_cleanup.sh — remove orphaned music files
|
||||
# 8. sonarr_cleanup.sh — remove orphaned TV files
|
||||
# 9. radarr_cleanup.sh — remove orphaned movie files
|
||||
# 10. docker_daily_restart.sh — restart containers that need daily restart
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# Pre-sync:
|
||||
# git_pull_execute.sh — pull latest scripts first, always
|
||||
#
|
||||
# What triggers weekly_health_digest.sh:
|
||||
# NOT this script — weekly_health_digest.sh runs on its own schedule (Saturday)
|
||||
# This script writes no stats — it just syncs and maintains
|
||||
# Rsync window (DAILY_SYNC_SHARES per host):
|
||||
# HOST*_DAILY_SYNC_SHARES — media shares pushed to mirror
|
||||
# HOST*_PERSONAL_SHARES — encrypted personal shares
|
||||
#
|
||||
# Configuration in Master.conf:
|
||||
# DAILY_MAINTENANCE_SCRIPTS — pre/post-sync scripts (git pull, docker restart)
|
||||
# DAILY_MAINTENANCE_SCRIPTS — media maintenance jobs run after sync
|
||||
# HOST1_DAILY_SYNC_SHARES — shares HOST1 pushes to HOST2
|
||||
# HOST2_DAILY_SYNC_SHARES — shares HOST2 pushes to HOST1
|
||||
# HOST1/2_PERSONAL_SHARES — encrypted personal shares
|
||||
# Post-sync maintenance (DAILY_MAINTENANCE_SCRIPTS):
|
||||
# media_shares_permissions.sh — fix ownership before arr cleanup
|
||||
# media_cleaner.sh anime — remove junk from anime shares
|
||||
# media_cleaner.sh media — remove junk from media shares
|
||||
# lidarr_cleanup.sh — remove orphaned music files
|
||||
# sonarr_cleanup.sh — remove orphaned TV files
|
||||
# radarr_cleanup.sh — remove orphaned movie files
|
||||
# docker_daily_restart.sh — restart containers needing daily restart
|
||||
#
|
||||
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
|
||||
# git pull first — maintenance runs on latest code, not yesterday's
|
||||
# Rsync before cleanup — cleanup sees the post-sync state
|
||||
# Permissions before arr cleanup — arrs need correct ownership to delete/rename
|
||||
# Arr cleanup after permissions — clean ownership = successful orphan deletion
|
||||
# Docker restart last — containers already processed by cleanup
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# Bidirectional — same script runs on both servers, correct direction automatic.
|
||||
# Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# detect_hosts() aliases DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars.
|
||||
# No manual HOST1/HOST2 comparisons — MY_ID routes correctly on any server.
|
||||
#
|
||||
# ── DRIVE TEMP HANDLING ───────────────────────────────────────────────────────────────────────
|
||||
# rsync.sh returns exit codes for temperature issues:
|
||||
# exit 1 = temp WARN — skip this share, continue to next
|
||||
# exit 2 = temp CRITICAL — abort ALL remaining syncs in this window
|
||||
# All other failures — skip share, continue to next
|
||||
#
|
||||
# ── SILENT WHEN HEALTHY ───────────────────────────────────────────────────────────────────────
|
||||
# Runs daily at 1am — clean run should produce minimal output.
|
||||
# Each job reports log() on success (silent), warn()/error() on failure (visible).
|
||||
# Summary always shown — gives window timing and share/job counts.
|
||||
# Notify only on failure — successful daily maintenance doesn't need notification.
|
||||
#
|
||||
# ── CONFIGURATION (master.conf + master_host*.conf) ───────────────────────────────────────────
|
||||
# HOST*_DAILY_SYNC_SHARES — shares pushed to mirror each day
|
||||
# HOST*_PERSONAL_SHARES — encrypted personal shares
|
||||
# DAILY_MAINTENANCE_SCRIPTS — maintenance jobs (permissions, cleanup, restart)
|
||||
# DAILY_RSYNC_ENABLED — enable/disable rsync section
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# daily_sync_maintenance.sh — normal run
|
||||
# daily_sync_maintenance.sh --dry-run — preview without syncing or changing
|
||||
# daily_sync_maintenance.sh --log — verbose per-share/per-job output
|
||||
# daily_sync_maintenance.sh --status — show configured shares and jobs
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
acquire_lock
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY SYNC STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_NET Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_SYNC Rsync enabled: ${RSYNC_ENABLED:-true}"
|
||||
echo "$ICON_SYNC Daily enabled: ${DAILY_RSYNC_ENABLED:-false}"
|
||||
echo ""
|
||||
echo "━━━ Daily Sync Shares ━━━"
|
||||
for share in "${DAILY_SYNC_SHARES[@]:-}"; do
|
||||
[[ -n "$share" ]] && echo " $ICON_SYNC $share"
|
||||
done
|
||||
for share in "${PERSONAL_SHARES[@]:-}"; do
|
||||
[[ -n "$share" ]] && echo " $ICON_SYNC $share (personal)"
|
||||
done
|
||||
echo ""
|
||||
echo "━━━ Daily Maintenance Scripts ━━━"
|
||||
for entry in "${DAILY_MAINTENANCE_SCRIPTS[@]:-}"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
echo " $ICON_GEAR $(basename "${entry%% *}") ${entry#* }"
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
WINDOW_START=$(date +%s)
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GIT Pre-sync Jobs ━━━
|
||||
# git_pull_execute.sh runs first — pulls latest scripts before anything else runs
|
||||
# Identified by script name — all other DAILY_MAINTENANCE_SCRIPTS run after sync
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Pre-sync Jobs ━━━"
|
||||
# ==============================================================================================
|
||||
# ── BUILD SHARE LIST via detect_hosts aliases ─────────────────────────────────────────────────
|
||||
# detect_hosts() sets DAILY_SYNC_SHARES and PERSONAL_SHARES from HOST*_ vars
|
||||
# No manual HOST1/HOST2 comparison needed — aliased automatically per server
|
||||
# ==============================================================================================
|
||||
ALL_SHARES=()
|
||||
for share in "${DAILY_SYNC_SHARES[@]:-}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
for share in "${PERSONAL_SHARES[@]:-}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
SHARE_COUNT=${#ALL_SHARES[@]}
|
||||
|
||||
# ── Split maintenance scripts: git pull runs pre-sync, rest run post-sync ─────────────────────
|
||||
PRE_SYNC_SCRIPTS=()
|
||||
POST_SYNC_SCRIPTS=()
|
||||
|
||||
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]:-}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_name=$(basename "${script_entry%% *}")
|
||||
if [[ "$script_name" == "git_pull_execute.sh" ]]; then
|
||||
@@ -85,236 +143,181 @@ for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
fi
|
||||
done
|
||||
|
||||
for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
# Helper — run a maintenance script, track pass/fail
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name"
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
continue
|
||||
return 1
|
||||
fi
|
||||
|
||||
if bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name")
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
# shellcheck disable=SC2086
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
||||
log "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name")
|
||||
error "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Build share list — host-specific truth shares + personal shares
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
WINDOW_START=$(date +%s)
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
PASS=()
|
||||
FAIL=()
|
||||
SHARE_TIMES=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
ALL_SHARES=()
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Daily Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
for share in "${HOST1_DAILY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
for share in "${HOST1_PERSONAL_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
elif [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then
|
||||
for share in "${HOST2_DAILY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
for share in "${HOST2_PERSONAL_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-sync — git pull ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GIT Pre-sync ━━━"
|
||||
for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
|
||||
run_job "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
SHARE_COUNT=${#ALL_SHARES[@]}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Media Share Sync ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Media Share Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_SUMMARY Shares: $SHARE_COUNT"
|
||||
# ==============================================================================================
|
||||
# ━━━ Media Share Sync ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Media Share Sync — $SHARE_COUNT share(s) ━━━"
|
||||
|
||||
TOTAL_START=$(date +%s)
|
||||
SHARE_INDEX=0
|
||||
ABORT_ALL_SYNCS=false
|
||||
|
||||
# Tier 1 + Tier 2 rsync gate check
|
||||
if ! check_rsync_enabled "DAILY"; then
|
||||
warn "Rsync disabled — skipping all $SHARE_COUNT share syncs"
|
||||
warn "Proceeding to media management jobs..."
|
||||
warn "Daily rsync disabled — skipping all $SHARE_COUNT share syncs"
|
||||
warn "Proceeding to maintenance jobs..."
|
||||
elif [[ "$SHARE_COUNT" -eq 0 ]]; then
|
||||
warn "No shares configured for $MY_ID — check HOST*_DAILY_SYNC_SHARES in master_host*.conf"
|
||||
else
|
||||
# Pre-flight — connectivity then remote rootfs
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
|
||||
RSYNC_DRY=""
|
||||
[[ "$DRY_RUN" == true ]] && RSYNC_DRY="--dry-run"
|
||||
|
||||
for SHARE in "${ALL_SHARES[@]}"; do
|
||||
SHARE_INDEX=$((SHARE_INDEX + 1))
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
echo "━━━ $ICON_SYNC Share $SHARE_INDEX of $SHARE_COUNT: $SHARE_NAME ━━━"
|
||||
|
||||
if [[ "$ABORT_ALL_SYNCS" == true ]]; then
|
||||
warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
|
||||
FAIL+=("$SHARE_NAME:temp-critical")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
bash "$RSYNC_SCRIPT" "$SHARE"
|
||||
RSYNC_EXIT=$?
|
||||
|
||||
SHARE_END=$(date +%s)
|
||||
SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))")
|
||||
|
||||
if [[ "$RSYNC_EXIT" -eq 0 ]]; then
|
||||
PASS+=("$SHARE_NAME")
|
||||
echo "$ICON_DONE $SHARE_NAME complete"
|
||||
elif [[ "$RSYNC_EXIT" -eq 1 ]]; then
|
||||
# Temp warning — skip this profile, continue to next
|
||||
FAIL+=("$SHARE_NAME:temp-warn")
|
||||
warn "$SHARE_NAME skipped — drive temps too high"
|
||||
elif [[ "$RSYNC_EXIT" -eq 2 ]]; then
|
||||
# Temp critical — abort all remaining syncs
|
||||
FAIL+=("$SHARE_NAME:temp-critical")
|
||||
ABORT_ALL_SYNCS=true
|
||||
error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
|
||||
notify "Daily sync aborted on $(hostname) — drive temps CRITICAL during $SHARE_NAME sync" "Daily Sync" "warning"
|
||||
else
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME failed — continuing to next share"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
fi # end check_rsync_enabled "DAILY"
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
TOTAL_DURATION=$((TOTAL_END - TOTAL_START))
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CLEAN Post-sync Media Jobs ━━━
|
||||
# Reads DAILY_MAINTENANCE_SCRIPTS from Master.conf — permissions, cleaners, arr cleanup
|
||||
# Runs after sync completes — correct ownership available, clean folders guaranteed
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Post-sync Media Jobs ━━━"
|
||||
|
||||
if [[ ${#DAILY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
(( SHARE_INDEX++ ))
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name ${extra_args[*]}"
|
||||
echo "━━━ $ICON_SYNC Share $SHARE_INDEX/$SHARE_COUNT: $SHARE_NAME ━━━"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
if [[ "$ABORT_ALL_SYNCS" == true ]]; then
|
||||
warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)"
|
||||
FAIL+=("$SHARE_NAME:temp-critical")
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
if bash "$script_path" "${extra_args[@]}" --dry-run; then
|
||||
success "$script_name — done (dry run)"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
else
|
||||
if bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
fi
|
||||
bash "$RSYNC_SCRIPT" "$SHARE" $RSYNC_DRY
|
||||
RSYNC_EXIT=$?
|
||||
|
||||
SHARE_TIMES+=("$SHARE_NAME:$(( $(date +%s) - SHARE_START ))")
|
||||
|
||||
case "$RSYNC_EXIT" in
|
||||
0)
|
||||
PASS+=("$SHARE_NAME")
|
||||
log "$SHARE_NAME — done ✅"
|
||||
;;
|
||||
1)
|
||||
FAIL+=("$SHARE_NAME:temp-warn")
|
||||
warn "$SHARE_NAME skipped — drive temps too high"
|
||||
;;
|
||||
2)
|
||||
FAIL+=("$SHARE_NAME:temp-critical")
|
||||
ABORT_ALL_SYNCS=true
|
||||
error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs"
|
||||
notify "Daily sync aborted on $(hostname) ($MY_ID) — drive temps CRITICAL during $SHARE_NAME" \
|
||||
"Daily Sync" "warning"
|
||||
;;
|
||||
*)
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME failed (exit $RSYNC_EXIT) — continuing to next share"
|
||||
;;
|
||||
esac
|
||||
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Post-sync System Jobs ━━━
|
||||
# Reads remaining DAILY_MAINTENANCE_SCRIPTS — docker restart etc.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
|
||||
|
||||
for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
TOTAL_END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Post-sync Maintenance Jobs ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#POST_SYNC_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
continue
|
||||
fi
|
||||
|
||||
if bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name")
|
||||
fi
|
||||
done
|
||||
echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━"
|
||||
for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
|
||||
echo ""
|
||||
run_job "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
WINDOW_END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Window: $(date -d @$WINDOW_START '+%Y-%m-%d %H:%M:%S') → $(date -d @$WINDOW_END '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Media shares:"
|
||||
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
||||
for entry in "${SHARE_TIMES[@]}"; do
|
||||
SHARE_NAME="${entry%%:*}"
|
||||
DURATION="${entry##*:}"
|
||||
if printf '%s\n' "${FAIL[@]}" | grep -qx "$SHARE_NAME"; then
|
||||
echo " $ICON_ERROR $SHARE_NAME — $(format_duration $DURATION)"
|
||||
sname="${entry%%:*}"
|
||||
sdur="${entry##*:}"
|
||||
if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
|
||||
echo " $ICON_ERROR $sname — $(format_duration "$sdur")"
|
||||
else
|
||||
echo " $ICON_DONE $SHARE_NAME — $(format_duration $DURATION)"
|
||||
echo " $ICON_DONE $sname — $(format_duration "$sdur")"
|
||||
fi
|
||||
done
|
||||
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
||||
[[ "$SHARE_COUNT" -eq 0 || "${DAILY_RSYNC_ENABLED:-false}" == "false" ]] && \
|
||||
echo " (rsync disabled)"
|
||||
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
||||
echo ""
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo "$ICON_GEAR Jobs (media + system):"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo "$ICON_GEAR Jobs:"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
echo "$ICON_WARN Status: $TOTAL_FAIL failure(s) — check logs"
|
||||
notify "Daily sync maintenance completed with failures on $(hostname) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" "Daily Maintenance" "warning"
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Daily maintenance completed with failures on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Daily Maintenance" "warning"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 1
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE"
|
||||
notify "Daily sync maintenance complete on $(hostname) — ${#PASS[@]} shares synced, ${#JOB_PASS[@]} jobs run in $(format_duration $(( WINDOW_END - WINDOW_START )))" "Daily Maintenance" "normal"
|
||||
log "$ICON_DONE Status: all complete — ${#PASS[@]} share(s) synced, ${#JOB_PASS[@]} job(s) run"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
Reference in New Issue
Block a user