created loop for both watchdogs and consolidated orch lists
This commit is contained in:
@@ -0,0 +1,284 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Daily Sync Maintenance ------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Daily orchestrator — runs the full daily maintenance window in the correct order.
|
||||
#
|
||||
# What it does:
|
||||
# 1. Iterates DAILY_MAINTENANCE_SCRIPTS — runs git pull first, then additional jobs
|
||||
# 2. Syncs all media shares in the correct direction for the local server
|
||||
# 3. Additional scripts in DAILY_MAINTENANCE_SCRIPTS run after the sync completes
|
||||
#
|
||||
# Media share sync:
|
||||
# Each server pushes only the shares it owns (source of truth) — direction is automatic.
|
||||
# HOST1 pushes: Movies, Tv_Shows, Music, Books etc. → HOST2
|
||||
# HOST2 pushes: Anime_Shows, Anime_Movies → HOST1
|
||||
# Personal encrypted shares synced after media shares.
|
||||
# detect_hosts() determines which server is running — no script changes needed.
|
||||
# Share lists configured in Master.conf ORCHESTRATORS section.
|
||||
#
|
||||
# Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time.
|
||||
# All job lists configured in Master.conf — no script changes needed to add or remove jobs.
|
||||
# Schedule: 0 1 * * * (1am daily — configured in User Scripts plugin)
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
acquire_lock
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SHIELD Pre-flight Checks ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━"
|
||||
|
||||
check_connectivity
|
||||
check_remote_rootfs
|
||||
|
||||
WINDOW_START=$(date +%s)
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GIT Pre-sync Jobs ━━━
|
||||
# git_pull_execute.sh runs first — pulls latest scripts before anything else runs
|
||||
# Identified by script name — all other DAILY_MAINTENANCE_SCRIPTS run after sync
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Pre-sync Jobs ━━━"
|
||||
|
||||
PRE_SYNC_SCRIPTS=()
|
||||
POST_SYNC_SCRIPTS=()
|
||||
|
||||
for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_name=$(basename "${script_entry%% *}")
|
||||
if [[ "$script_name" == "git_pull_execute.sh" ]]; then
|
||||
PRE_SYNC_SCRIPTS+=("$script_entry")
|
||||
else
|
||||
POST_SYNC_SCRIPTS+=("$script_entry")
|
||||
fi
|
||||
done
|
||||
|
||||
for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
continue
|
||||
fi
|
||||
|
||||
if bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name")
|
||||
fi
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Build share list — host-specific truth shares + personal shares
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
PASS=()
|
||||
FAIL=()
|
||||
SHARE_TIMES=()
|
||||
TOTAL_START=$(date +%s)
|
||||
|
||||
ALL_SHARES=()
|
||||
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
for share in "${HOST1_DAILY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
for share in "${HOST1_PERSONAL_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
elif [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then
|
||||
for share in "${HOST2_DAILY_SYNC_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
for share in "${HOST2_PERSONAL_SHARES[@]}"; do
|
||||
[[ -n "$share" ]] && ALL_SHARES+=("$share")
|
||||
done
|
||||
fi
|
||||
|
||||
SHARE_COUNT=${#ALL_SHARES[@]}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SYNC Media Share Sync ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Media Share Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_SUMMARY Shares: $SHARE_COUNT"
|
||||
echo ""
|
||||
|
||||
SHARE_INDEX=0
|
||||
|
||||
for SHARE in "${ALL_SHARES[@]}"; do
|
||||
SHARE_INDEX=$((SHARE_INDEX + 1))
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
SHARE_START=$(date +%s)
|
||||
|
||||
echo "━━━ $ICON_SYNC Share $SHARE_INDEX of $SHARE_COUNT: $SHARE_NAME ━━━"
|
||||
|
||||
if bash "$RSYNC_SCRIPT" "$SHARE"; then
|
||||
SHARE_END=$(date +%s)
|
||||
SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))")
|
||||
PASS+=("$SHARE_NAME")
|
||||
echo "$ICON_DONE $SHARE_NAME complete"
|
||||
else
|
||||
SHARE_END=$(date +%s)
|
||||
SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))")
|
||||
FAIL+=("$SHARE_NAME")
|
||||
error "$SHARE_NAME failed — continuing to next share"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
TOTAL_END=$(date +%s)
|
||||
TOTAL_DURATION=$((TOTAL_END - TOTAL_START))
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CLEAN Post-sync Media Jobs ━━━
|
||||
# Reads MEDIA_MANAGEMENT_JOBS from Master.conf — permissions, cleaners, arr cleanup
|
||||
# Runs after sync completes — correct ownership available, clean folders guaranteed
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Post-sync Media Jobs ━━━"
|
||||
|
||||
if [[ ${#MEDIA_MANAGEMENT_JOBS[@]} -gt 0 ]]; then
|
||||
for script_entry in "${MEDIA_MANAGEMENT_JOBS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name ${extra_args[*]}"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
if bash "$script_path" "${extra_args[@]}" --dry-run; then
|
||||
success "$script_name — done (dry run)"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
else
|
||||
if bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Post-sync System Jobs ━━━
|
||||
# Reads remaining DAILY_MAINTENANCE_SCRIPTS — docker restart etc.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Post-sync Jobs ━━━"
|
||||
|
||||
for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
script_args=($script_entry)
|
||||
script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
|
||||
echo ""
|
||||
info "$ICON_START Running: $script_name"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
continue
|
||||
fi
|
||||
|
||||
if bash "$script_path" "${extra_args[@]}"; then
|
||||
success "$script_name — done"
|
||||
JOB_PASS+=("$script_name")
|
||||
else
|
||||
error "$script_name — failed"
|
||||
JOB_FAIL+=("$script_name")
|
||||
fi
|
||||
done
|
||||
|
||||
WINDOW_END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY SYNC MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Window: $(date -d @$WINDOW_START '+%Y-%m-%d %H:%M:%S') → $(date -d @$WINDOW_END '+%H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||
echo ""
|
||||
|
||||
echo "$ICON_SYNC Media shares:"
|
||||
for entry in "${SHARE_TIMES[@]}"; do
|
||||
SHARE_NAME="${entry%%:*}"
|
||||
DURATION="${entry##*:}"
|
||||
if printf '%s\n' "${FAIL[@]}" | grep -qx "$SHARE_NAME"; then
|
||||
echo " $ICON_ERROR $SHARE_NAME — $(format_duration $DURATION)"
|
||||
else
|
||||
echo " $ICON_DONE $SHARE_NAME — $(format_duration $DURATION)"
|
||||
fi
|
||||
done
|
||||
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
||||
echo ""
|
||||
|
||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo "$ICON_GEAR Jobs (media + system):"
|
||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
echo "$ICON_WARN Status: $TOTAL_FAIL failure(s) — check logs"
|
||||
notify "Daily sync maintenance completed with failures on $(hostname) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" "Daily Maintenance" "warning"
|
||||
exit 1
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE"
|
||||
notify "Daily sync maintenance complete on $(hostname) — ${#PASS[@]} shares synced, ${#JOB_PASS[@]} jobs run in $(format_duration $(( WINDOW_END - WINDOW_START )))" "Daily Maintenance" "normal"
|
||||
exit 0
|
||||
fi
|
||||
Reference in New Issue
Block a user