#!/bin/bash # ----------------------------------------------------------------------------------------------- # --------------------------------- Daily Sync Maintenance ------------------------------------ # ----------------------------------------------------------------------------------------------- # Daily orchestrator — runs the full daily maintenance window in the correct order. # Schedule: 0 1 * * * (1am daily) # # Execution order: # 1. git_pull_execute.sh — pull latest scripts first, always # 2. Media share sync — HOST*_DAILY_SYNC_SHARES pushed to remote # 3. HOST*_PERSONAL_SHARES — personal encrypted shares after media # 4. media_shares_permissions.sh — fix ownership before arr cleanup # 5. media_cleaner.sh anime — remove junk from anime shares # 6. media_cleaner.sh media — remove junk from media shares # 7. lidarr_cleanup.sh — remove orphaned music files # 8. sonarr_cleanup.sh — remove orphaned TV files # 9. radarr_cleanup.sh — remove orphaned movie files # 10. docker_daily_restart.sh — restart containers that need daily restart # # What triggers weekly_health_digest.sh: # NOT this script — weekly_health_digest.sh runs on its own schedule (Saturday) # This script writes no stats — it just syncs and maintains # # Configuration in Master.conf: # DAILY_MAINTENANCE_SCRIPTS — pre/post-sync scripts (git pull, docker restart) # DAILY_MAINTENANCE_SCRIPTS — media maintenance jobs run after sync # HOST1_DAILY_SYNC_SHARES — shares HOST1 pushes to HOST2 # HOST2_DAILY_SYNC_SHARES — shares HOST2 pushes to HOST1 # HOST1/2_PERSONAL_SHARES — encrypted personal shares # # Bidirectional — same script runs on both servers, correct direction automatic. # Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time. # ----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" SCRIPTS_ROOT="$SCRIPT_DIR/.." # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_GEAR Setup ━━━" detect_hosts resolve_remote_ip acquire_lock # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SHIELD Pre-flight Checks ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_SHIELD Pre-flight Checks ━━━" check_connectivity check_remote_rootfs WINDOW_START=$(date +%s) JOB_PASS=() JOB_FAIL=() # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GIT Pre-sync Jobs ━━━ # git_pull_execute.sh runs first — pulls latest scripts before anything else runs # Identified by script name — all other DAILY_MAINTENANCE_SCRIPTS run after sync # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_SYNC Pre-sync Jobs ━━━" PRE_SYNC_SCRIPTS=() POST_SYNC_SCRIPTS=() for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do [[ -z "$script_entry" ]] && continue script_name=$(basename "${script_entry%% *}") if [[ "$script_name" == "git_pull_execute.sh" ]]; then PRE_SYNC_SCRIPTS+=("$script_entry") else POST_SYNC_SCRIPTS+=("$script_entry") fi done for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do [[ -z "$script_entry" ]] && continue script_args=($script_entry) script_path="$SCRIPTS_ROOT/${script_args[0]}" script_name=$(basename "${script_args[0]}") extra_args=("${script_args[@]:1}") echo "" info "$ICON_START Running: $script_name" if [[ ! -f "$script_path" ]]; then error "$script_name — not found at $script_path" JOB_FAIL+=("$script_name") continue fi if bash "$script_path" "${extra_args[@]}"; then success "$script_name — done" JOB_PASS+=("$script_name") else error "$script_name — failed" JOB_FAIL+=("$script_name") fi done # ----------------------------------------------------------------------------------------------- # Build share list — host-specific truth shares + personal shares # ----------------------------------------------------------------------------------------------- PASS=() FAIL=() SHARE_TIMES=() TOTAL_START=$(date +%s) ALL_SHARES=() if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then for share in "${HOST1_DAILY_SYNC_SHARES[@]}"; do [[ -n "$share" ]] && ALL_SHARES+=("$share") done for share in "${HOST1_PERSONAL_SHARES[@]}"; do [[ -n "$share" ]] && ALL_SHARES+=("$share") done elif [[ "$LOCAL_SERVER_NAME" == "$HOST2" ]]; then for share in "${HOST2_DAILY_SYNC_SHARES[@]}"; do [[ -n "$share" ]] && ALL_SHARES+=("$share") done for share in "${HOST2_PERSONAL_SHARES[@]}"; do [[ -n "$share" ]] && ALL_SHARES+=("$share") done fi SHARE_COUNT=${#ALL_SHARES[@]} # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SYNC Media Share Sync ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_SYNC Media Share Sync — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_SUMMARY Shares: $SHARE_COUNT" echo "" SHARE_INDEX=0 ABORT_ALL_SYNCS=false # Tier 1 + Tier 2 rsync gate check if ! check_rsync_enabled "DAILY"; then warn "Rsync disabled — skipping all $SHARE_COUNT share syncs" warn "Proceeding to media management jobs..." else for SHARE in "${ALL_SHARES[@]}"; do SHARE_INDEX=$((SHARE_INDEX + 1)) SHARE_NAME=$(basename "$SHARE") SHARE_START=$(date +%s) echo "━━━ $ICON_SYNC Share $SHARE_INDEX of $SHARE_COUNT: $SHARE_NAME ━━━" if [[ "$ABORT_ALL_SYNCS" == true ]]; then warn "$SHARE_NAME — skipped (drive temps CRITICAL earlier in window)" FAIL+=("$SHARE_NAME:temp-critical") echo "" continue fi bash "$RSYNC_SCRIPT" "$SHARE" RSYNC_EXIT=$? SHARE_END=$(date +%s) SHARE_TIMES+=("$SHARE_NAME:$((SHARE_END - SHARE_START))") if [[ "$RSYNC_EXIT" -eq 0 ]]; then PASS+=("$SHARE_NAME") echo "$ICON_DONE $SHARE_NAME complete" elif [[ "$RSYNC_EXIT" -eq 1 ]]; then # Temp warning — skip this profile, continue to next FAIL+=("$SHARE_NAME:temp-warn") warn "$SHARE_NAME skipped — drive temps too high" elif [[ "$RSYNC_EXIT" -eq 2 ]]; then # Temp critical — abort all remaining syncs FAIL+=("$SHARE_NAME:temp-critical") ABORT_ALL_SYNCS=true error "$SHARE_NAME aborted — drive temps CRITICAL, stopping all remaining syncs" notify "Daily sync aborted on $(hostname) — drive temps CRITICAL during $SHARE_NAME sync" "Daily Sync" "warning" else FAIL+=("$SHARE_NAME") error "$SHARE_NAME failed — continuing to next share" fi echo "" done fi # end check_rsync_enabled "DAILY" TOTAL_END=$(date +%s) TOTAL_DURATION=$((TOTAL_END - TOTAL_START)) # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_CLEAN Post-sync Media Jobs ━━━ # Reads DAILY_MAINTENANCE_SCRIPTS from Master.conf — permissions, cleaners, arr cleanup # Runs after sync completes — correct ownership available, clean folders guaranteed # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_CLEAN Post-sync Media Jobs ━━━" if [[ ${#DAILY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do [[ -z "$script_entry" ]] && continue script_args=($script_entry) script_path="$SCRIPTS_ROOT/${script_args[0]}" script_name=$(basename "${script_args[0]}") extra_args=("${script_args[@]:1}") echo "" info "$ICON_START Running: $script_name ${extra_args[*]}" if [[ ! -f "$script_path" ]]; then error "$script_name — not found at $script_path" JOB_FAIL+=("$script_name ${extra_args[*]}") continue fi if [[ "$DRY_RUN" == true ]]; then if bash "$script_path" "${extra_args[@]}" --dry-run; then success "$script_name — done (dry run)" JOB_PASS+=("$script_name ${extra_args[*]}") else error "$script_name — failed" JOB_FAIL+=("$script_name ${extra_args[*]}") fi else if bash "$script_path" "${extra_args[@]}"; then success "$script_name — done" JOB_PASS+=("$script_name ${extra_args[*]}") else error "$script_name — failed" JOB_FAIL+=("$script_name ${extra_args[*]}") fi fi done fi # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Post-sync System Jobs ━━━ # Reads remaining DAILY_MAINTENANCE_SCRIPTS — docker restart etc. # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_GEAR Post-sync Jobs ━━━" for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do [[ -z "$script_entry" ]] && continue script_args=($script_entry) script_path="$SCRIPTS_ROOT/${script_args[0]}" script_name=$(basename "${script_args[0]}") extra_args=("${script_args[@]:1}") echo "" info "$ICON_START Running: $script_name" if [[ ! -f "$script_path" ]]; then error "$script_name — not found at $script_path" JOB_FAIL+=("$script_name") continue fi if bash "$script_path" "${extra_args[@]}"; then success "$script_name — done" JOB_PASS+=("$script_name") else error "$script_name — failed" JOB_FAIL+=("$script_name") fi done WINDOW_END=$(date +%s) # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Summary ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━━━ $ICON_SUMMARY DAILY SYNC MAINTENANCE SUMMARY ━━━━━" echo "$ICON_TIME Window: $(date -d @$WINDOW_START '+%Y-%m-%d %H:%M:%S') → $(date -d @$WINDOW_END '+%H:%M:%S')" echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))" echo "" echo "$ICON_SYNC Media shares:" for entry in "${SHARE_TIMES[@]}"; do SHARE_NAME="${entry%%:*}" DURATION="${entry##*:}" if printf '%s\n' "${FAIL[@]}" | grep -qx "$SHARE_NAME"; then echo " $ICON_ERROR $SHARE_NAME — $(format_duration $DURATION)" else echo " $ICON_DONE $SHARE_NAME — $(format_duration $DURATION)" fi done echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT" echo "" if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then echo "$ICON_GEAR Jobs (media + system):" for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done echo "" fi TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} )) if [[ "$TOTAL_FAIL" -gt 0 ]]; then echo "$ICON_WARN Status: $TOTAL_FAIL failure(s) — check logs" notify "Daily sync maintenance completed with failures on $(hostname) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" "Daily Maintenance" "warning" exit 1 else echo "$ICON_DONE Status: $ICON_SUCCESS ALL COMPLETE" notify "Daily sync maintenance complete on $(hostname) — ${#PASS[@]} shares synced, ${#JOB_PASS[@]} jobs run in $(format_duration $(( WINDOW_END - WINDOW_START )))" "Daily Maintenance" "normal" exit 0 fi