diff --git a/Master.conf b/Master.conf index 676e44c..68502fa 100644 --- a/Master.conf +++ b/Master.conf @@ -16,7 +16,7 @@ # # Section Description # ─────────────────────────────────────────────────────────────────────────────────────────── -# HOST CONFIGURATION Server hostnames, SSH keys, and Emby connection details +# HOST CONFIGURATION Server hostnames, SSH keys, Emby connection details, DATA_DIR # LOGGING Enable or disable verbose logging # NOTIFICATIONS unRAID native and Discord webhook settings # GIT / REPO Gitea repository and SSH settings @@ -90,6 +90,12 @@ HOST1="unRAID-Gmer4Lfe" HOST2="unRAID-Jayred365" +# Data directory — persistent script state and statistics files. +# Array share — survives reboots, no flash drive wear. +# Created automatically if it doesn't exist. +# Only truly critical files (failover state, watchdog reboot log) stay on /boot/config. + DATA_DIR="/mnt/user/appdata/unraid_scripts/data" + # SSH keys for server-to-server rsync and failover container operations. # Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys. HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" @@ -735,7 +741,8 @@ WATCHDOG_SCAN_IGNORE=( # Skip list auto-clears when container recovers healthy WATCHDOG_CONTAINER_RESTART_LIMIT=3 # max watchdog restarts allowed in window WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours - WATCHDOG_CONTAINER_RESTART_LOG="/boot/config/container_restart_history.db" + WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db" + # rolling restart history for loop detection # Notification batching — one clean summary per cycle instead of one ping per event # true = batch all events into one notification at end of cycle @@ -918,7 +925,7 @@ LIDARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.lrc") LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run # protects against API returning partial data on a bad day -LIDARR_TRACKED_COUNT_FILE="/boot/config/lidarr_tracked.count" +LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count" # persists last known tracked count for percentage comparison # ── Sonarr ──────────────────────────────────────────────────────────────────────────────────── @@ -946,6 +953,7 @@ declare -A HOST2_SONARR_PATH_MAP=( ) SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion +SONARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov") SONARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") # NEVER deleted — cover art, metadata, subtitles @@ -976,6 +984,7 @@ declare -A HOST2_RADARR_PATH_MAP=( ) RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion +RADARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov") RADARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa") # NEVER deleted — cover art, metadata, subtitles @@ -1081,7 +1090,7 @@ HOST2_RADARR_RECOVERY=true # Anime_Movies import recovery # Daily statistics log — read by weekly_health_digest.sh for transcode summary # Tracks peak usage, flip count, session ratio, files cleaned per day # Bounded to TRANSCODE_LOG_RETENTION days — auto-purges old entries on each write - TRANSCODE_DAILY_LOG="/boot/config/transcode_daily.db" + TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db" TRANSCODE_LOG_RETENTION=90 # days before old entries are purged # ━━━ Transcode Server Array ━━━ @@ -1164,10 +1173,15 @@ ZFS_REPORT_IGNORE_POOLS=( # Tracks transfer size, duration and profile per sync for weekly summary reporting. # BANDWIDTH_LOG_RETENTION = days to keep entries before auto-purging old records # BANDWIDTH_WARN_GB = flag in weekly summary if a single sync exceeded this size - BANDWIDTH_LOG="/boot/config/bandwidth_history.db" + BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db" BANDWIDTH_LOG_RETENTION=90 # days before old entries are purged BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report +# Stats files — written by cleanup and recovery scripts, read by coffee report +# All in DATA_DIR — array always running when these are written + ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats + ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats + # ━━━ Health Digest ━━━ # Aggregated system health summary — reads existing state files, no new writes. # Three profiles control when the digest email is sent: diff --git a/Media/arrs_failed_stalled_recovery.sh b/Media/arrs_failed_stalled_recovery.sh index e7b95c8..31af498 100644 --- a/Media/arrs_failed_stalled_recovery.sh +++ b/Media/arrs_failed_stalled_recovery.sh @@ -1,10 +1,18 @@ #!/bin/bash # ----------------------------------------------------------------------------------------------- -# --------------------------------- Arrs Failed Stalled Recovery ---------------------------------------- +# --------------------------------- Arrs Failed Stalled Recovery -------------------------------- # ----------------------------------------------------------------------------------------------- # Automatically detects and recovers from failed imports and stalled downloads # across Sonarr, Radarr, and Lidarr. Blocklists the bad release and triggers # a new search — hands free recovery while you sleep. +# Schedule: 0 */6 * * * (every 6 hours) +# +# What it checks (per-arr toggles in Master.conf): +# HOST1: Sonarr (Tv_Shows) — /api/v3/ +# HOST1: Radarr (Movies) — /api/v3/ +# HOST1: Lidarr (Music) — /api/v1/ ← HOST1 only, exits cleanly on HOST2 +# HOST2: Sonarr (Anime_Shows) — /api/v3/ +# HOST2: Radarr (Anime_Movies) — /api/v3/ # # Targets four problem types from the queue API: # importFailed — downloaded successfully but arr couldn't import the file @@ -12,6 +20,7 @@ # error status — serious failure not covered by importFailed/importPending # stalled — download stuck with no connections or no progress # +# Items newer than ARR_IMPORT_RECOVERY_AGE (6hr) are skipped — gives arr time to retry. # Never touches items with state "downloading" or "imported" — safe to run anytime. # # Action per problem item: @@ -19,6 +28,12 @@ # 2. Remove from queue — cleans up the failed item # 3. Trigger new search — finds a different release automatically # +# Configuration in Master.conf: +# ARR_IMPORT_RECOVERY_AGE — hours before item is eligible +# HOST1/2_SONARR_RECOVERY — enable/disable per arr +# HOST1/2_RADARR_RECOVERY — enable/disable per arr +# HOST1_LIDARR_RECOVERY — enable/disable Lidarr (HOST1 only) +# # Age threshold (ARR_IMPORT_RECOVERY_AGE): # Items newer than threshold are skipped — gives the arr time to retry on its own # Items older than threshold have not self-resolved — safe to intervene @@ -357,4 +372,12 @@ elif [[ "$TOTAL_ACTIONED" -gt 0 ]]; then else echo "$ICON_DONE Status: $ICON_SUCCESS DONE — nothing to recover" fi + +# Write stats for sunday_morning_coffee_report.sh +if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_RECOVERY_STATS:-}" ]]; then + DATE=$(date '+%Y-%m-%d') + TIME=$(date '+%H:%M') + echo "${DATE}|${TIME}|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}" \ + >> "$ARR_RECOVERY_STATS" 2>/dev/null || true +fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Media/lidarr_cleanup.sh b/Media/lidarr_cleanup.sh index ea97474..08bdbda 100644 --- a/Media/lidarr_cleanup.sh +++ b/Media/lidarr_cleanup.sh @@ -31,6 +31,14 @@ # Required when deletion would exceed LIDARR_MAX_DELETE_GB # Long and annoying by design — cannot be added accidentally # +# --skip-strike-list flag: +# Bypasses the LIDARR_ORPHAN_AGE age check — deletes recent files too +# Combined with --i-know-what-im-doing activates NUCLEAR MODE: +# Age check bypassed, size threshold bypassed, deletes on first pass +# Use when Soularr/other tool has filled the gaps — clean one-pass wipe +# ⚠️ The script author takes NO responsibility for data loss with both flags active +# The user accepts full responsibility — this is 100% intentional by design +# # Lidarr runs on HOST1 only — music library is HOST1's source of truth. # If run on HOST2 this script exits cleanly with no action. # All configuration in Master.conf under Arr Cleanup section. @@ -42,12 +50,16 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" -# Check for --i-know-what-im-doing flag before parse_args +# Check for --i-know-what-im-doing and --skip-strike-list flags before parse_args +# These flags are filtered out before parse_args sees them to avoid unknown flag errors I_KNOW=false +SKIP_STRIKES=false FILTERED_ARGS=() for arg in "$@"; do if [[ "$arg" == "--i-know-what-im-doing" ]]; then I_KNOW=true + elif [[ "$arg" == "--skip-strike-list" ]]; then + SKIP_STRIKES=true else FILTERED_ARGS+=("$arg") fi @@ -55,6 +67,32 @@ done parse_args "${FILTERED_ARGS[@]}" +# Nuclear mode — both override flags active +# Strike system AND size threshold bypassed — deletes on first pass +# Script author takes no responsibility for data loss when both flags are used. +# This combination is 100% intentional and the user accepts full responsibility. +if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "⚠️ WARNING — NUCLEAR MODE ACTIVE" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " Flags: --i-know-what-im-doing --skip-strike-list" + echo " Strike system: BYPASSED — deletes on first pass" + echo " Size threshold: BYPASSED — no GB limit" + echo " Data recovery: NOT POSSIBLE after deletion" + echo "" + echo " The script author takes no responsibility for data" + echo " loss when both flags are used together. This is a" + echo " 100% intentional action by the user." + echo "" + echo " Review the dry run output before proceeding." + echo " You have 10 seconds to cancel (Ctrl+C)..." + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + sleep 10 + echo " Proceeding..." + echo "" +fi + # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ # ----------------------------------------------------------------------------------------------- @@ -117,6 +155,9 @@ if [[ ! -d "$LIDARR_MUSIC_ROOT" ]]; then exit 1 fi +# Large library scans take time — override default lock warn age +[[ -n "${LIDARR_LOCK_WARN_AGE:-}" ]] && LOCK_WARN_AGE="$LIDARR_LOCK_WARN_AGE" + acquire_lock "wait" # ----------------------------------------------------------------------------------------------- @@ -156,6 +197,7 @@ esac success "All safety checks passed" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" [[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing flag active" +[[ "$SKIP_STRIKES" == true ]] && warn "OVERRIDE — --skip-strike-list flag active — strike system bypassed" # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Status ━━━ @@ -349,7 +391,7 @@ while IFS= read -r filepath; do FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) FILE_AGE=$(( NOW - FILE_MTIME )) - if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]]; then + if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then log "RECENT (skipping): $filepath" ((RECENT_COUNT++)) continue @@ -379,6 +421,7 @@ if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then error "Deletion would exceed ${LIDARR_MAX_DELETE_GB}GB threshold — $TOTAL_HUMAN would be deleted" error "Review the ORPHAN lines above carefully before proceeding" error "If this is expected, rerun with: --i-know-what-im-doing" + error "To also bypass age check and delete on first pass: add --skip-strike-list" notify "Lidarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} deletion requires --i-know-what-im-doing" "Lidarr Cleanup" "warning" exit 1 else @@ -398,7 +441,7 @@ if [[ "$DRY_RUN" == false ]]; then FILE_AGE=$(( NOW - FILE_MTIME )) if is_music_file "$filepath"; then - [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && continue + [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]] && continue fi rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath" @@ -448,4 +491,11 @@ else echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "normal" fi + +# Write stats for sunday_morning_coffee_report.sh +if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then + DATE=$(date '+%Y-%m-%d') + echo "${DATE}|lidarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \ + >> "$ARR_CLEANUP_STATS" 2>/dev/null || true +fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Media/radarr_cleanup.sh b/Media/radarr_cleanup.sh index 87ada7c..2390ac1 100644 --- a/Media/radarr_cleanup.sh +++ b/Media/radarr_cleanup.sh @@ -30,7 +30,44 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" -parse_args "$@" +# Check for --i-know-what-im-doing and --skip-strike-list flags before parse_args +I_KNOW=false +SKIP_STRIKES=false +FILTERED_ARGS=() +for arg in "$@"; do + if [[ "$arg" == "--i-know-what-im-doing" ]]; then + I_KNOW=true + elif [[ "$arg" == "--skip-strike-list" ]]; then + SKIP_STRIKES=true + else + FILTERED_ARGS+=("$arg") + fi +done + +parse_args "${FILTERED_ARGS[@]}" + +# Nuclear mode disclaimer +if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "⚠️ WARNING — NUCLEAR MODE ACTIVE" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " Flags: --i-know-what-im-doing --skip-strike-list" + echo " Strike system: BYPASSED — deletes on first pass" + echo " Size threshold: BYPASSED — no GB limit" + echo " Data recovery: NOT POSSIBLE after deletion" + echo "" + echo " The script author takes no responsibility for data" + echo " loss when both flags are used together. This is a" + echo " 100% intentional action by the user." + echo "" + echo " Review the dry run output before proceeding." + echo " You have 10 seconds to cancel (Ctrl+C)..." + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + sleep 10 + echo " Proceeding..." + echo "" +fi # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ @@ -243,7 +280,7 @@ while IFS= read -r filepath; do FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) FILE_AGE=$(( NOW - FILE_MTIME )) - if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]]; then + if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then log "RECENT (skipping): $filepath" ((RECENT_COUNT++)) continue @@ -305,6 +342,24 @@ format_bytes() { ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES) JUNK_HUMAN=$(format_bytes $JUNK_BYTES) TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) +TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES )) +MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", ${RADARR_MAX_DELETE_GB:-1} * 1073741824}") + +# Size threshold check +if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then + TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}") + if [[ "$I_KNOW" != true ]]; then + echo "" + error "Deletion would exceed ${RADARR_MAX_DELETE_GB:-1}GB threshold — $TOTAL_HUMAN would be deleted" + error "Review the ORPHAN lines above carefully before proceeding" + error "If this is expected, rerun with: --i-know-what-im-doing" + error "To also bypass age check and delete on first pass: add --skip-strike-list" + notify "Radarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} deletion requires --i-know-what-im-doing" "Radarr Cleanup" "warning" + exit 1 + else + warn "OVERRIDE — deletion is ${TOTAL_HUMAN} — proceeding because --i-know-what-im-doing" + fi +fi # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Summary ━━━ @@ -327,4 +382,11 @@ else echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed" notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Radarr Cleanup" "normal" fi + +# Write stats for sunday_morning_coffee_report.sh +if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then + DATE=$(date '+%Y-%m-%d') + echo "${DATE}|radarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \ + >> "$ARR_CLEANUP_STATS" 2>/dev/null || true +fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Media/sonarr_cleanup.sh b/Media/sonarr_cleanup.sh index 36718d9..90b4bcd 100644 --- a/Media/sonarr_cleanup.sh +++ b/Media/sonarr_cleanup.sh @@ -30,7 +30,44 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../Master.conf" source "$SCRIPT_DIR/../common.sh" -parse_args "$@" +# Check for --i-know-what-im-doing and --skip-strike-list flags before parse_args +I_KNOW=false +SKIP_STRIKES=false +FILTERED_ARGS=() +for arg in "$@"; do + if [[ "$arg" == "--i-know-what-im-doing" ]]; then + I_KNOW=true + elif [[ "$arg" == "--skip-strike-list" ]]; then + SKIP_STRIKES=true + else + FILTERED_ARGS+=("$arg") + fi +done + +parse_args "${FILTERED_ARGS[@]}" + +# Nuclear mode disclaimer +if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then + echo "" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "⚠️ WARNING — NUCLEAR MODE ACTIVE" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo " Flags: --i-know-what-im-doing --skip-strike-list" + echo " Strike system: BYPASSED — deletes on first pass" + echo " Size threshold: BYPASSED — no GB limit" + echo " Data recovery: NOT POSSIBLE after deletion" + echo "" + echo " The script author takes no responsibility for data" + echo " loss when both flags are used together. This is a" + echo " 100% intentional action by the user." + echo "" + echo " Review the dry run output before proceeding." + echo " You have 10 seconds to cancel (Ctrl+C)..." + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + sleep 10 + echo " Proceeding..." + echo "" +fi # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ @@ -243,7 +280,7 @@ while IFS= read -r filepath; do FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0) FILE_AGE=$(( NOW - FILE_MTIME )) - if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]]; then + if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then log "RECENT (skipping): $filepath" ((RECENT_COUNT++)) continue @@ -305,6 +342,24 @@ format_bytes() { ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES) JUNK_HUMAN=$(format_bytes $JUNK_BYTES) TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT )) +TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES )) +MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", ${SONARR_MAX_DELETE_GB:-1} * 1073741824}") + +# Size threshold check +if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then + TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}") + if [[ "$I_KNOW" != true ]]; then + echo "" + error "Deletion would exceed ${SONARR_MAX_DELETE_GB:-1}GB threshold — $TOTAL_HUMAN would be deleted" + error "Review the ORPHAN lines above carefully before proceeding" + error "If this is expected, rerun with: --i-know-what-im-doing" + error "To also bypass age check and delete on first pass: add --skip-strike-list" + notify "Sonarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} deletion requires --i-know-what-im-doing" "Sonarr Cleanup" "warning" + exit 1 + else + warn "OVERRIDE — deletion is ${TOTAL_HUMAN} — proceeding because --i-know-what-im-doing" + fi +fi # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Summary ━━━ @@ -327,4 +382,11 @@ else echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed" notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Sonarr Cleanup" "normal" fi + +# Write stats for sunday_morning_coffee_report.sh +if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then + DATE=$(date '+%Y-%m-%d') + echo "${DATE}|sonarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \ + >> "$ARR_CLEANUP_STATS" 2>/dev/null || true +fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Orchestrators/array_start.sh b/Orchestrators/array_start.sh index 93e7814..812ebdc 100644 --- a/Orchestrators/array_start.sh +++ b/Orchestrators/array_start.sh @@ -2,21 +2,25 @@ # ----------------------------------------------------------------------------------------------- # --------------------------------- Array Start Orchestrator ----------------------------------- # ----------------------------------------------------------------------------------------------- -# Launches all scripts configured in ARRAY_START_SCRIPTS when the unRAID array comes online. -# Set this script to run at "Startup of Array" in the User Scripts plugin. +# Single entry point for "At Startup of Array" in User Scripts plugin. +# Launches everything configured in ARRAY_START_SCRIPTS in Master.conf. # -# Each script is launched as a background process: -# One-shot scripts (ramdisk_setup, syslog_filter etc.) run and exit naturally -# Continuous scripts (system_watchdog, docker_watchdog, failover) run until array stops +# What it launches (configured in Master.conf ARRAY_START_SCRIPTS): +# Transcodes/ramdisk_setup.sh — creates tmpfs + symlink before Emby starts (one-shot) +# unRAID_Essentials/docker_syslog_filter.sh — suppress veth log noise (one-shot) +# unRAID_Essentials/php_fpm_max_children.sh — WebGUI performance tuning (one-shot) +# Docker_Essentials/docker_network_connect.sh — connect containers to extra networks (one-shot) +# unRAID_Essentials/system_watchdog.sh — system health monitor (continuous loop) +# Docker_Essentials/docker_watchdog.sh — container health monitor (continuous loop) +# Failover/failover.sh — mutual failover monitor (continuous loop) # -# Scripts are launched in the order defined in ARRAY_START_SCRIPTS in Master.conf. -# Order matters — ramdisk before Emby, network before watchdogs, watchdogs before failover. -# -# To add or remove a script: edit ARRAY_START_SCRIPTS in Master.conf. -# No changes to this script needed. -# -# Logs: each script logs its own output independently. +# One-shot scripts run and exit naturally — array_start.sh confirms completion. +# Continuous scripts run until array stops or SIGTERM received. # This orchestrator exits after launching all scripts — unRAID sees it complete normally. +# +# Add or remove scripts: edit ARRAY_START_SCRIPTS in Master.conf. +# Order matters — ramdisk first, network before watchdogs, watchdogs before failover. +# No changes to this script ever needed. # ----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Orchestrators/daily_sync_maintenance.sh b/Orchestrators/daily_sync_maintenance.sh index 82b91ab..a593291 100644 --- a/Orchestrators/daily_sync_maintenance.sh +++ b/Orchestrators/daily_sync_maintenance.sh @@ -3,23 +3,29 @@ # --------------------------------- Daily Sync Maintenance ------------------------------------ # ----------------------------------------------------------------------------------------------- # Daily orchestrator — runs the full daily maintenance window in the correct order. +# Schedule: 0 1 * * * (1am daily) # -# What it does: -# 1. Iterates DAILY_MAINTENANCE_SCRIPTS — runs git pull first, then additional jobs -# 2. Syncs all media shares in the correct direction for the local server -# 3. Additional scripts in DAILY_MAINTENANCE_SCRIPTS run after the sync completes +# Execution order: +# 1. git_pull_execute.sh — pull latest scripts first, always +# 2. Media share sync — HOST*_DAILY_SYNC_SHARES pushed to remote +# 3. HOST*_PERSONAL_SHARES — personal encrypted shares after media +# 4. media_shares_permissions.sh — fix ownership before arr cleanup +# 5. media_cleaner.sh anime — remove junk from anime shares +# 6. media_cleaner.sh media — remove junk from media shares +# 7. lidarr_cleanup.sh — remove orphaned music files +# 8. sonarr_cleanup.sh — remove orphaned TV files +# 9. radarr_cleanup.sh — remove orphaned movie files +# 10. docker_daily_restart.sh — restart containers that need daily restart # -# Media share sync: -# Each server pushes only the shares it owns (source of truth) — direction is automatic. -# HOST1 pushes: Movies, Tv_Shows, Music, Books etc. → HOST2 -# HOST2 pushes: Anime_Shows, Anime_Movies → HOST1 -# Personal encrypted shares synced after media shares. -# detect_hosts() determines which server is running — no script changes needed. -# Share lists configured in Master.conf ORCHESTRATORS section. +# Configuration in Master.conf: +# DAILY_MAINTENANCE_SCRIPTS — pre/post-sync scripts (git pull, docker restart) +# MEDIA_MANAGEMENT_JOBS — media maintenance jobs run after sync +# HOST1_DAILY_SYNC_SHARES — shares HOST1 pushes to HOST2 +# HOST2_DAILY_SYNC_SHARES — shares HOST2 pushes to HOST1 +# HOST1/2_PERSONAL_SHARES — encrypted personal shares # +# Bidirectional — same script runs on both servers, correct direction automatic. # Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time. -# All job lists configured in Master.conf — no script changes needed to add or remove jobs. -# Schedule: 0 1 * * * (1am daily — configured in User Scripts plugin) # ----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/Orchestrators/sunday_morning_coffee_report.sh b/Orchestrators/sunday_morning_coffee_report.sh new file mode 100644 index 0000000..8b8cad3 --- /dev/null +++ b/Orchestrators/sunday_morning_coffee_report.sh @@ -0,0 +1,633 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# ----------------------------- Sunday Morning Coffee Report ----------------------------------- +# ----------------------------------------------------------------------------------------------- +# Weekly system overview — everything that happened this week in one clean read. +# Designed to be read over coffee Sunday morning while the system is fully caught up +# from the 2:30am maintenance window. +# Schedule: 0 7 * * 0 (7am Sunday — after weekly_sync_maintenance.sh finishes at ~3am) +# Maintenance window completes → 4 hours of fresh data → report ready ☕ +# +# Sections: +# 🖥️ System — uptime, memory, boot drive, cache drive, reboots +# 📀 Array — disk count, parity status, ZFS health, drive temps +# 🔀 Failover — state, last change, Tailscale connectivity +# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split +# 🎵 Media Activity — arr cleanup stats, arr recovery stats, library health +# 🌐 Rsync — weekly transfer totals, per-share breakdown +# 🛡️ Watchdog — container strikes, restarts, system strikes, skip list +# 🔐 Security — SSL cert expiry per domain +# 📊 Emby — weekly stream count, top users, top content +# ⚙️ Health — SMART summary, docker container count, Gitea sync status +# ⚠️ Issues — anything requiring attention collected above +# +# Data sources (reads only — no writes except the notification): +# DATA_DIR stats files — arr cleanup, recovery, transcode, bandwidth history +# /boot/config — failover state, watchdog reboot log +# /tmp — watchdog strike state files +# /proc, /sys — system memory, uptime +# /var/local/emhttp/ — unRAID array info +# Emby API — session history +# Arr APIs — current queue depth +# tailscale — network status +# openssl — live SSL cert check +# smartctl — drive health +# +# All configuration in Master.conf. +# Supports --dry-run to preview report without sending notification. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ Setup ━━━ +# ----------------------------------------------------------------------------------------------- +acquire_lock + +detect_hosts + +REPORT_DATE=$(date '+%A, %B %-d, %Y') +WEEK_START=$(date -d "7 days ago" '+%Y-%m-%d') +TODAY=$(date '+%Y-%m-%d') +NOW=$(date +%s) + +REPORT=() # all report lines +ISSUES=() # items needing attention +FINDINGS=() # notable but not critical + +section() { + REPORT+=("") + REPORT+=("$1") + REPORT+=("$(printf '%.0s─' {1..50})") +} + +line() { + REPORT+=(" $1") +} + +issue() { + ISSUES+=("$1") + REPORT+=(" ⚠️ $1") +} + +finding() { + FINDINGS+=("$1") + REPORT+=(" ℹ️ $1") +} + +format_bytes() { + local bytes=$1 + if (( bytes > 1073741824 )); then + awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" + elif (( bytes > 1048576 )); then + awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" + else + echo "${bytes}B" + fi +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ 🖥️ System ━━━ +# ----------------------------------------------------------------------------------------------- +section "🖥️ SYSTEM" + +# Uptime +UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime) +UPTIME_DAYS=$(( UPTIME_SECONDS / 86400 )) +UPTIME_HOURS=$(( (UPTIME_SECONDS % 86400) / 3600 )) +BOOT_TIME=$(date -d "@$(( NOW - UPTIME_SECONDS ))" '+%A %-d %b at %-I:%M%p') +line "Uptime: ${UPTIME_DAYS}d ${UPTIME_HOURS}hr (up since $BOOT_TIME)" + +# Reboot history this week +REBOOT_COUNT=0 +if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then + WEEK_EPOCH=$(date -d "$WEEK_START" +%s) + REBOOT_COUNT=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \ + "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l) +fi +if [[ "$REBOOT_COUNT" -gt 0 ]]; then + issue "Reboots this week: $REBOOT_COUNT (system watchdog triggered)" +else + line "Reboots this week: 0 ✅" +fi + +# Memory +MEM_TOTAL_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo) +MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) +MEM_USED_KB=$(( MEM_TOTAL_KB - MEM_AVAIL_KB )) +MEM_TOTAL_GB=$(awk "BEGIN {printf \"%.0f\", $MEM_TOTAL_KB / 1048576}") +MEM_USED_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_USED_KB / 1048576}") +MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}") + +ARC_SIZE=0 +if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then + ARC_BYTES=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) + ARC_SIZE=$(awk "BEGIN {printf \"%.1f\", $ARC_BYTES / 1073741824}") +fi +line "Memory: ${MEM_USED_GB}GB used / ${MEM_TOTAL_GB}GB total (ARC: ${ARC_SIZE}GB)" + +# Boot drive +BOOT_PCT=$(df /boot --output=pcent 2>/dev/null | tail -1 | tr -d ' %') +BOOT_USED=$(df /boot -h --output=used 2>/dev/null | tail -1 | tr -d ' ') +BOOT_SIZE=$(df /boot -h --output=size 2>/dev/null | tail -1 | tr -d ' ') +if [[ "${BOOT_PCT:-0}" -ge 80 ]]; then + issue "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE}) — getting full" +else + line "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE})" +fi + +# Cache drive +CACHE_PCT=$(df /mnt/cache --output=pcent 2>/dev/null | tail -1 | tr -d ' %') +CACHE_AVAIL=$(df /mnt/cache -h --output=avail 2>/dev/null | tail -1 | tr -d ' ') +CACHE_SIZE=$(df /mnt/cache -h --output=size 2>/dev/null | tail -1 | tr -d ' ') +if [[ "${CACHE_PCT:-0}" -ge 85 ]]; then + issue "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})" +else + line "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ 📀 Array ━━━ +# ----------------------------------------------------------------------------------------------- +section "📀 ARRAY" + +# unRAID array info +if [[ -f /var/local/emhttp/disks.ini ]]; then + DISK_COUNT=$(grep -c "^\[disk" /var/local/emhttp/disks.ini 2>/dev/null || echo "?") + line "Array disks: $DISK_COUNT" +fi + +# Parity +if [[ -f /var/local/emhttp/parity-date.txt ]]; then + PARITY_INFO=$(cat /var/local/emhttp/parity-date.txt 2>/dev/null) + if echo "$PARITY_INFO" | grep -q "progress"; then + finding "Parity check in progress" + else + PARITY_DATE=$(echo "$PARITY_INFO" | grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -1) + PARITY_ERRORS=$(echo "$PARITY_INFO" | grep -oE 'errors=[0-9]+' | cut -d= -f2 || echo 0) + if [[ "${PARITY_ERRORS:-0}" -gt 0 ]]; then + issue "Parity: $PARITY_ERRORS errors on last check ($PARITY_DATE)" + else + line "Parity: OK (last check: ${PARITY_DATE:-unknown}, 0 errors)" + fi + fi +fi + +# ZFS pool health +if command -v zpool >/dev/null 2>&1; then + POOLS=$(zpool list -H -o name,health 2>/dev/null) + while IFS=$'\t' read -r pool health; do + [[ -z "$pool" ]] && continue + # Skip single-disk unRAID array pools + if [[ "$health" == "ONLINE" ]]; then + line "ZFS $pool: ONLINE ✅" + else + issue "ZFS $pool: $health — check immediately" + fi + done <<< "$POOLS" +fi + +# Drive temperatures from SMART +if command -v smartctl >/dev/null 2>&1; then + TEMP_WARN=false + TEMP_SUMMARY="" + for disk in /dev/sd? /dev/nvme?; do + [[ ! -e "$disk" ]] && continue + DISK_NAME=$(basename "$disk") + # Skip ignored drives + SKIP=false + for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do + [[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break + done + [[ "$SKIP" == true ]] && continue + TEMP=$(smartctl -A "$disk" 2>/dev/null | \ + awk '/Temperature_Celsius|Temperature:/ {print $NF; exit}' | tr -d '°C') + [[ -z "$TEMP" ]] && continue + TEMP_INT=$(printf "%.0f" "$TEMP" 2>/dev/null || echo 0) + if [[ "$TEMP_INT" -ge "${SMART_TEMP_CRIT:-55}" ]]; then + issue "Drive $DISK_NAME: ${TEMP_INT}°C — CRITICAL" + TEMP_WARN=true + elif [[ "$TEMP_INT" -ge "${SMART_TEMP_WARN:-45}" ]]; then + finding "Drive $DISK_NAME: ${TEMP_INT}°C — warm" + TEMP_WARN=true + fi + [[ -n "$TEMP_SUMMARY" ]] && TEMP_SUMMARY+=", " + TEMP_SUMMARY+="${DISK_NAME}:${TEMP_INT}°C" + done + if [[ "$TEMP_WARN" == false ]]; then + line "Drive temps: all normal" + fi + [[ -n "$TEMP_SUMMARY" ]] && line "Temps: $TEMP_SUMMARY" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ 🔀 Failover ━━━ +# ----------------------------------------------------------------------------------------------- +section "🔀 FAILOVER" + +if [[ -f "$FAILOVER_STATE_FILE" ]]; then + FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + FAILOVER_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2) + if [[ "$FAILOVER_STATE" == "NORMAL" ]]; then + line "State: NORMAL ✅" + else + issue "Failover state: $FAILOVER_STATE — not normal" + fi + line "Last change: ${FAILOVER_CHANGE:-unknown}" +else + issue "Failover state file not found" +fi + +# Tailscale connectivity +if command -v tailscale >/dev/null 2>&1; then + TS_STATUS=$(tailscale status 2>/dev/null) + REMOTE_VISIBLE=$(echo "$TS_STATUS" | grep -c "$HOST2" 2>/dev/null || echo 0) + if [[ "$REMOTE_VISIBLE" -gt 0 ]]; then + REMOTE_IP=$(tailscale ip -4 "$HOST2" 2>/dev/null || echo "unknown") + line "Tailscale: $HOST2 visible at $REMOTE_IP ✅" + else + issue "Tailscale: $HOST2 not visible — check connectivity" + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ 🎬 Transcodes ━━━ +# ----------------------------------------------------------------------------------------------- +section "🎬 TRANSCODES" + +if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then + RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ') + RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}") + RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail | tail -1 | tr -d ' ') + RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}") + SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null | xargs basename 2>/dev/null || echo "unknown") + line "Ramdisk now: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET" +else + issue "Ramdisk not mounted" +fi + +if [[ -f "$TRANSCODE_DAILY_LOG" ]]; then + WEEK_PEAK=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff {if ($2 > max) max=$2} END {printf "%.2f", max+0}' \ + "$TRANSCODE_DAILY_LOG") + WEEK_FLIPS=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff {sum+=$3} END {print sum+0}' "$TRANSCODE_DAILY_LOG") + WEEK_RAM=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff {sum+=$4} END {print sum+0}' "$TRANSCODE_DAILY_LOG") + WEEK_SSD=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff {sum+=$5} END {print sum+0}' "$TRANSCODE_DAILY_LOG") + WEEK_FILES=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff {sum+=$6} END {print sum+0}' "$TRANSCODE_DAILY_LOG") + + line "Week peak: ${WEEK_PEAK}GB | flips: ${WEEK_FLIPS} | files cleaned: ${WEEK_FILES}" + line "Sessions: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD" + + PEAK_INT=$(printf "%.0f" "$WEEK_PEAK" 2>/dev/null || echo 0) + WARN_INT=$(printf "%.0f" "$RAMDISK_WARN_GB" 2>/dev/null || echo 0) + if [[ "$PEAK_INT" -ge "$WARN_INT" ]]; then + finding "Transcode peak ${WEEK_PEAK}GB reached warn threshold — consider increasing RAMDISK_SIZE" + fi + if [[ "${WEEK_FLIPS:-0}" -ge "${TRANSCODE_FLIP_WARN:-3}" ]]; then + finding "Transcode flips this week: $WEEK_FLIPS — monitor ramdisk headroom" + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ 🎵 Media Activity ━━━ +# ----------------------------------------------------------------------------------------------- +section "🎵 MEDIA ACTIVITY" + +# Arr cleanup stats +if [[ -f "$ARR_CLEANUP_STATS" ]]; then + for arr in lidarr sonarr radarr; do + WEEK_ORPHANS=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \ + '$1 >= cutoff && $2 == a {sum+=$3} END {print sum+0}' "$ARR_CLEANUP_STATS") + WEEK_BYTES=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \ + '$1 >= cutoff && $2 == a {sum+=$4} END {print sum+0}' "$ARR_CLEANUP_STATS") + WEEK_TRACKED=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \ + 'BEGIN{max=0} $1 >= cutoff && $2 == a && $8 > max {max=$8} END {print max+0}' \ + "$ARR_CLEANUP_STATS") + if [[ "${WEEK_ORPHANS:-0}" -gt 0 ]]; then + FREED=$(format_bytes "${WEEK_BYTES:-0}") + line "${arr^} cleanup: $WEEK_ORPHANS orphans removed ($FREED freed) | tracked: $WEEK_TRACKED files" + else + line "${arr^} cleanup: clean ✅ (tracked: $WEEK_TRACKED files)" + fi + done +else + line "Arr cleanup stats: no data yet (runs after first cleanup)" +fi + +# Arr recovery stats +if [[ -f "$ARR_RECOVERY_STATS" ]]; then + WEEK_ACTIONED=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff {sum+=$3} END {print sum+0}' "$ARR_RECOVERY_STATS") + WEEK_RUNS=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff {count++} END {print count+0}' "$ARR_RECOVERY_STATS") + if [[ "${WEEK_ACTIONED:-0}" -gt 0 ]]; then + line "Arr recovery: $WEEK_ACTIONED items auto-recovered across $WEEK_RUNS runs" + else + line "Arr recovery: no failed imports this week ✅" + fi +fi + +# Arr queue depth — current snapshot +for arr_name in "Sonarr|${HOST1_SONARR_URL}|${HOST1_SONARR_API_KEY}|v3" \ + "Radarr|${HOST1_RADARR_URL}|${HOST1_RADARR_API_KEY}|v3" \ + "Lidarr|${HOST1_LIDARR_URL}|${HOST1_LIDARR_API_KEY}|v1"; do + IFS='|' read -r name url key ver <<< "$arr_name" + if [[ "$url" == *"your-"* ]] || [[ -z "$key" ]]; then continue; fi + QUEUE=$(curl -sf --max-time 5 -H "X-Api-Key: $key" \ + "${url}/api/${ver}/queue?pageSize=1" 2>/dev/null | \ + python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('totalRecords',0))" \ + 2>/dev/null || echo "?") + if [[ "$QUEUE" == "0" ]] || [[ -z "$QUEUE" ]]; then + line "$name queue: empty ✅" + else + line "$name queue: $QUEUE items" + fi +done + +# ----------------------------------------------------------------------------------------------- +# ━━━ 🌐 Rsync ━━━ +# ----------------------------------------------------------------------------------------------- +section "🌐 RSYNC" + +if [[ -f "$BANDWIDTH_LOG" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then + WEEK_TOTAL_BYTES=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff {sum+=$3} END {print sum+0}' "$BANDWIDTH_LOG") + WEEK_SYNCS=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff' "$BANDWIDTH_LOG" | wc -l) + WEEK_FAILED=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff && $5 != "success"' "$BANDWIDTH_LOG" | wc -l) + WEEK_GB=$(awk "BEGIN {printf \"%.1f\", $WEEK_TOTAL_BYTES / 1073741824}") + + line "Total transferred: ${WEEK_GB}GB across $WEEK_SYNCS syncs" + if [[ "${WEEK_FAILED:-0}" -gt 0 ]]; then + issue "Failed syncs this week: $WEEK_FAILED" + else + line "Sync failures: none ✅" + fi + + # Top 3 shares by transfer this week + TOP_SHARES=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$1 >= cutoff {bytes[$4]+=$3} END {for(s in bytes) print bytes[s], s}' \ + "$BANDWIDTH_LOG" | sort -rn | head -3) + if [[ -n "$TOP_SHARES" ]]; then + while IFS=' ' read -r bytes share; do + [[ -z "$share" ]] && continue + SIZE=$(format_bytes "$bytes") + line " → $share: $SIZE" + done <<< "$TOP_SHARES" + fi +else + line "No bandwidth data yet" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ 🛡️ Watchdog ━━━ +# ----------------------------------------------------------------------------------------------- +section "🛡️ WATCHDOG" + +# Container watchdog strikes +if [[ -f "$WATCHDOG_STATE_FILE" ]]; then + ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$" | wc -l) + if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then + STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ') + issue "Container watchdog: $ACTIVE_STRIKES active strikes — $STRIKE_LIST" + else + line "Container watchdog: no active strikes ✅" + fi +fi + +# System watchdog strikes +if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then + SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$" | wc -l) + if [[ "$SYS_STRIKES" -gt 0 ]]; then + SYS_LIST=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ') + issue "System watchdog: $SYS_STRIKES active strikes — $SYS_LIST" + else + line "System watchdog: no active strikes ✅" + fi +fi + +# Container skip list +if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then + SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE") + SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ') + issue "Skip list: $SKIP_COUNT containers — $SKIP_LIST — manual intervention needed" +else + line "Skip list: empty ✅" +fi + +# Container restarts this week +if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then + WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l) + if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then + RESTARTED=$(awk -F'|' -v cutoff="$WEEK_START" \ + '$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \ + sort | uniq -c | sort -rn | head -5 | \ + awk '{print $2 "(" $1 ")"}' | tr '\n' ' ') + finding "Watchdog restarted $WEEK_RESTARTS containers this week: $RESTARTED" + else + line "Watchdog restarts: none this week ✅" + fi +fi + +# Docker container count +if command -v docker >/dev/null 2>&1; then + RUNNING=$(docker ps -q 2>/dev/null | wc -l) + TOTAL=$(docker ps -aq 2>/dev/null | wc -l) + STOPPED=$(( TOTAL - RUNNING )) + if [[ "$STOPPED" -gt 0 ]]; then + STOPPED_NAMES=$(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null | \ + head -5 | tr '\n' ' ') + finding "Docker: $RUNNING/$TOTAL running — $STOPPED stopped: $STOPPED_NAMES" + else + line "Docker: $RUNNING/$TOTAL containers running ✅" + fi +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ 🔐 Security ━━━ +# ----------------------------------------------------------------------------------------------- +section "🔐 SECURITY" + +for domain in "${CERT_MONITOR_DOMAINS[@]:-}"; do + [[ -z "$domain" ]] && continue + EXPIRY=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \ + -connect "${domain}:443" -servername "$domain" 2>/dev/null | \ + openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) + if [[ -z "$EXPIRY" ]]; then + issue "$domain: could not check certificate" + continue + fi + EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s 2>/dev/null || echo 0) + DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW) / 86400 )) + if [[ "$DAYS_LEFT" -le "${CERT_CRIT_DAYS:-7}" ]]; then + issue "$domain: ${DAYS_LEFT} days remaining — CRITICAL, renew now" + elif [[ "$DAYS_LEFT" -le "${CERT_WARN_DAYS:-30}" ]]; then + finding "$domain: ${DAYS_LEFT} days remaining — renew soon" + else + line "$domain: ${DAYS_LEFT} days remaining ✅" + fi +done + +# ----------------------------------------------------------------------------------------------- +# ━━━ 📊 Emby ━━━ +# ----------------------------------------------------------------------------------------------- +section "📊 EMBY" + +# Select correct Emby URL and key for this host +if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then + EMBY_URL="$HOST1_EMBY_URL" + EMBY_KEY="$HOST1_EMBY_API_KEY" +else + EMBY_URL="$HOST2_EMBY_URL" + EMBY_KEY="$HOST2_EMBY_API_KEY" +fi + +if [[ "$EMBY_KEY" != *"your-"* ]] && [[ -n "$EMBY_KEY" ]]; then + WEEK_MS=$(( 7 * 24 * 3600 * 10000000 )) + ACTIVITY=$(curl -sf --max-time 10 \ + -H "X-Emby-Token: $EMBY_KEY" \ + "${EMBY_URL}/Sessions?ControllableByUserId=&api_key=$EMBY_KEY" \ + 2>/dev/null) + + # Use activity log for weekly stats + ACTIVITY_LOG=$(curl -sf --max-time 10 \ + -H "X-Emby-Token: $EMBY_KEY" \ + "${EMBY_URL}/user_usage_stats/user_activity?days=7&api_key=$EMBY_KEY" \ + 2>/dev/null) + + if [[ -n "$ACTIVITY_LOG" ]]; then + TOTAL_PLAYS=$(echo "$ACTIVITY_LOG" | \ + python3 -c "import sys,json; d=json.load(sys.stdin); \ + print(sum(u.get('total_plays',0) for u in d))" 2>/dev/null || echo "?") + line "Streams this week: $TOTAL_PLAYS total plays" + + # Top 3 users + TOP_USERS=$(echo "$ACTIVITY_LOG" | \ + python3 -c " +import sys,json +d=json.load(sys.stdin) +users=sorted(d,key=lambda x:x.get('total_plays',0),reverse=True)[:3] +for u in users: + print(f\" → {u.get('user_name','?')}: {u.get('total_plays',0)} plays\") +" 2>/dev/null) + [[ -n "$TOP_USERS" ]] && echo "$TOP_USERS" | while IFS= read -r l; do line "$l"; done + else + # Fallback — just show active sessions + ACTIVE=$(curl -sf --max-time 5 \ + -H "X-Emby-Token: $EMBY_KEY" \ + "${EMBY_URL}/Sessions?api_key=$EMBY_KEY" 2>/dev/null | \ + python3 -c "import sys,json; \ + d=json.load(sys.stdin); \ + active=[s for s in d if s.get('NowPlayingItem')]; \ + print(f'{len(active)} active streams now')" 2>/dev/null || echo "API unavailable") + line "Emby: $ACTIVE" + fi +else + line "Emby: API key not configured" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ ⚙️ System Health ━━━ +# ----------------------------------------------------------------------------------------------- +section "⚙️ SYSTEM HEALTH" + +# SMART summary +if command -v smartctl >/dev/null 2>&1; then + SMART_ISSUES=0 + for disk in /dev/sd? /dev/nvme?; do + [[ ! -e "$disk" ]] && continue + DISK_NAME=$(basename "$disk") + SKIP=false + for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do + [[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break + done + [[ "$SKIP" == true ]] && continue + HEALTH=$(smartctl -H "$disk" 2>/dev/null | grep "SMART overall-health" | \ + awk '{print $NF}') + if [[ "$HEALTH" != "PASSED" ]] && [[ -n "$HEALTH" ]]; then + issue "SMART $DISK_NAME: $HEALTH — check immediately" + ((SMART_ISSUES++)) + fi + done + if [[ "$SMART_ISSUES" -eq 0 ]]; then + line "SMART: all drives PASSED ✅" + fi +fi + +# Gitea sync status +if command -v git >/dev/null 2>&1 && [[ -d "$TARGET_DIR/.git" ]]; then + CURRENT_COMMIT=$(git -C "$TARGET_DIR" rev-parse --short HEAD 2>/dev/null || echo "unknown") + LAST_PULL=$(git -C "$TARGET_DIR" log -1 --format="%ar" 2>/dev/null || echo "unknown") + line "Gitea: commit $CURRENT_COMMIT (pulled $LAST_PULL)" +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ ⚠️ Issues Requiring Attention ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ ${#ISSUES[@]} -gt 0 ]]; then + REPORT+=("") + REPORT+=("⚠️ ISSUES REQUIRING ATTENTION") + REPORT+=("$(printf '%.0s─' {1..50})") + for issue_line in "${ISSUES[@]}"; do + REPORT+=(" ❌ $issue_line") + done +fi + +if [[ ${#FINDINGS[@]} -gt 0 ]]; then + REPORT+=("") + REPORT+=("ℹ️ NOTABLE") + REPORT+=("$(printf '%.0s─' {1..50})") + for finding_line in "${FINDINGS[@]}"; do + REPORT+=(" → $finding_line") + done +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ Footer ━━━ +# ----------------------------------------------------------------------------------------------- +REPORT+=("") +if [[ ${#ISSUES[@]} -eq 0 ]]; then + STATUS="✅ All systems healthy — enjoy your Sunday" +else + STATUS="⚠️ ${#ISSUES[@]} issue(s) need attention" +fi +REPORT+=("$STATUS") +REPORT+=("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━") + +# ----------------------------------------------------------------------------------------------- +# ━━━ Output and Send ━━━ +# ----------------------------------------------------------------------------------------------- +HEADER="☕ SUNDAY MORNING COFFEE REPORT — $REPORT_DATE" +DIVIDER="$(printf '%.0s━' {1..50})" + +echo "" +echo "$DIVIDER" +echo "$HEADER" +echo "$DIVIDER" + +for report_line in "${REPORT[@]}"; do + echo "$report_line" +done + +echo "" + +# Send notification +BODY=$(printf '%s\n' "$HEADER" "${REPORT[@]}") + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — notification not sent" +else + notify "$BODY" "☕ Weekly Report" "normal" + success "Report sent" +fi \ No newline at end of file diff --git a/Orchestrators/weekly_sync_maintenance.sh b/Orchestrators/weekly_sync_maintenance.sh index 8695a41..3074ce6 100644 --- a/Orchestrators/weekly_sync_maintenance.sh +++ b/Orchestrators/weekly_sync_maintenance.sh @@ -1,38 +1,45 @@ #!/bin/bash # ----------------------------------------------------------------------------------------------- -# ----------------------------- Weekly Sync Maintenance ------------------------------------ +# ----------------------------- Weekly Sync Maintenance ---------------------------------------- # ----------------------------------------------------------------------------------------------- -# Maintenance window orchestrator for Emby and the auth stack (Critical-Data). -# Both local and remote containers are stopped for the entire window — clean state -# guaranteed for sync, updates, and restart. +# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts. +# Schedule: 30 2 * * 0 (Sunday 2:30am — fits before 3am network reboot) # -# What it does (in order): -# 1. Stop local containers — auth stack + Emby stopped locally -# 2. Stop remote containers — auth stack + Emby stopped remotely via SSH -# 3. Pull updates locally — if CRITICAL_SYNC_UPDATES=true -# 4. Pull updates remotely — if CRITICAL_SYNC_UPDATES_REMOTE=true -# 5. rsync Emby — clean full sync, both sides down -# 6. rsync Critical-Data — clean full sync, databases fully flushed -# 7. Start remote containers — starts on new images, correct order -# 8. Start local containers — starts on new images, correct order +# Execution order: +# 1. Stop local containers — Emby + auth stack stopped locally +# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH +# 3. Pull updates locally — if CRITICAL_SYNC_UPDATES=true +# 4. Pull updates remotely — if CRITICAL_SYNC_UPDATES_REMOTE=true +# 5. rsync Emby — full clean mirror, both instances stopped +# 6. rsync Critical-Data — auth stack clean sync, databases flushed +# 7. Start remote containers — correct order, delayed start respected +# 8. Start local containers — correct order, delayed start respected +# 9. docker_weekly_restart.sh — weekly container restarts # -# Why this is better than separate update + sync jobs: -# Containers are already stopped for the sync — no extra downtime for updates -# Both sides on identical image versions after restart -# Databases synced before first start on new version — clean state guaranteed -# One maintenance window handles sync + updates + ordered restart +# Synced shares (WEEKLY_SYNC_JOBS in Master.conf): +# /mnt/user/Media_Server/Emby — emby profile — full mirror, cache resets weekly +# /mnt/user/appdata-Failover/Critical-Data — critical-data — auth stack clean state # -# Toggle updates on/off in Master.conf: -# CRITICAL_SYNC_UPDATES=true/false — local updates -# CRITICAL_SYNC_UPDATES_REMOTE=true/false — remote updates -# Both false = sync only (sync only, no updates) +# Why weekly instead of nightly for Emby: +# Emby builds a warm image cache on HOST2 throughout the week +# Syncing nightly resets cache — cold loads every morning for users +# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep +# emby-failover dirty sync covers watch states + library every 30-60min between syncs # -# Container lists and startup order from PROFILE_CRITICAL_CONTAINER_NAMES -# Delayed containers (Authelia, Authelia-Secondary) respected on restart -# Containers not found on a server skipped gracefully -# Only containers that were running get restarted — stopped containers stay stopped +# Container updates during the window: +# Containers already stopped for sync — updates pull at zero extra downtime +# Both servers start on identical image versions after the window completes +# Toggle: CRITICAL_SYNC_UPDATES / CRITICAL_SYNC_UPDATES_REMOTE in Master.conf # -# Recommended schedule: 30 2 * * 0 (Sunday 2:30am) +# What triggers weekly_health_digest.sh: +# NOT this script — weekly_health_digest.sh runs on its own Saturday schedule +# +# Configuration in Master.conf: +# WEEKLY_SYNC_JOBS — shares synced during the maintenance window +# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync (docker_weekly_restart) +# CRITICAL_SYNC_UPDATES — toggle container updates on/off +# CRITICAL_SYNC_UPDATES_REMOTE — toggle remote container updates on/off +# ----------------------------------------------------------------------------------------------- # All configuration in Master.conf. # Supports --dry-run to walk through without stopping containers, syncing, or updating. # ----------------------------------------------------------------------------------------------- diff --git a/common.sh b/common.sh index d5219c4..9ee3907 100644 --- a/common.sh +++ b/common.sh @@ -672,6 +672,12 @@ get_rsync_opts() { # ----------------------------------------------------------------------------------------------- LOCK_DIR="/tmp/unraid_locks" + +# Ensure DATA_DIR exists — created here so every script that sources common.sh +# can safely write to it without checking first +if [[ -n "${DATA_DIR:-}" ]] && [[ ! -d "$DATA_DIR" ]]; then + mkdir -p "$DATA_DIR" 2>/dev/null || true +fi RSYNC_COUNT_FILE="$LOCK_DIR/rsync_active_count" RSYNC_MAX_CONCURRENT=3 LOCK_WARN_AGE=300 # seconds — warn if lock older than this (5min default) diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index 70076fd..44151e9 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -49,6 +49,14 @@ # ├── user_script_plug-in.sh # This file — copy into User Scripts plugin # ├── git_pull_execute.sh # Pulls latest scripts from Gitea repo # │ +# ├── Orchestrators/ +# │ ├── array_start.sh # Single entry point — launches all array-start scripts +# │ ├── daily_sync_maintenance.sh # Daily — git pull, media sync, media mgmt, docker restart +# │ ├── weekly_sync_maintenance.sh # Weekly — critical sync + updates, docker weekly restart +# │ ├── media_management.sh # Permissions + cleaners + arr cleanup — run manually +# │ ├── transcode_management.sh # Cleanup then manager every 3min + daily stats +# │ └── README-Orchestrators.md +# │ # ├── Failover/ # │ ├── failover.sh # Mutual container failover — runs continuously # │ ├── failover_test.sh # Controlled failover simulation — run manually @@ -64,16 +72,6 @@ # │ ├── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report # │ └── README-Monitors.md # │ -# ├── Orchestrators/ -# │ ├── array_start.sh # Single entry point — launches all array-start scripts -# │ ├── daily_sync_maintenance.sh # Daily — git pull, media sync, media mgmt, docker restart -# │ ├── weekly_sync_maintenance.sh # Weekly — critical sync + updates, docker weekly restart -# │ ├── daily_sync_maintenance.sh # Media shares sync both directions -# │ ├── weekly_sync_maintenance.sh # Clean sync + container updates (Emby + auth stack) -# │ ├── media_management.sh # Permissions + cleaners + arr cleanup — run manually -# │ ├── transcode_management.sh # Cleanup then manager every 3min + daily stats -# │ └── README-Orchestrators.md -# │ # ├── Rsync/ # │ ├── rsync.sh # Core rsync script — called per share or profile # │ └── README-Rsync_Setup.md # Rsync-specific setup guide @@ -137,6 +135,47 @@ # in real time — symlink flips work correctly for the lifetime of the container. # # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# ━━━ 🚀 Orchestrators — Uncomment the one below you want to run ━━━ +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# ━━━ Orchestrators ━━━ +# Orchestrators are the main scripts — they handle most if not all needed scripts +# in a specific order of operations. Each orchestrator owns a domain and runs +# everything within it automatically. All scripts below orchestrators can still +# be run separately for testing or manual maintenance. +# +# array_start.sh — At Startup of Array — single entry, launches everything +# ramdisk, syslog filter, php-fpm, network connect (one-shot) +# system_watchdog, docker_watchdog, failover (continuous loops) +# +# daily_sync_maintenance.sh — 0 1 * * * (1am daily) +# git pull → media share sync → permissions → cleaners → +# arr cleanup → docker daily restart +# +# weekly_sync_maintenance.sh — 30 2 * * 0 (Sunday 2:30am) +# stop containers → pull updates → sync Emby + Critical-Data → +# start containers → docker weekly restart +# +# transcode_management.sh — */3 * * * * (every 3 minutes) +# transcode cleanup → transcode manager (correct order) +# +# sunday_morning_coffee_report.sh — 0 7 * * 0 (Sunday 7am) +# full weekly system health digest — ready when you wake up ☕ +# system, array, failover, transcodes, media activity, rsync, +# watchdog, security, Emby stats, health summary +# +# arrs_failed_stalled_recovery.sh — 0 */6 * * * (every 6 hours) +# blocklist + re-search failed imports and stalled downloads +# across Sonarr, Radarr, and Lidarr automatically +# ━━━━━━━━━━━━━━━━━━━━━━ +# +#/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh +#/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh +#/mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh +#/mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh +#/mnt/user/appdata/unraid_scripts/Orchestrators/sunday_morning_coffee_report.sh +#/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh +# +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # ━━━ 🚀 Script Commands — Uncomment the one you want to run ━━━ # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ # @@ -162,18 +201,6 @@ #/mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh #/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh # -# ━━━ Orchestrators ━━━ -# array_start.sh is the single User Scripts entry for array startup. -# All other orchestrators are scheduled via cron — not started at array start. -# -#/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh -# ^^ set to: At Startup of Array -#/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh -#/mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh -#/mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh -# ^^ media_management.sh is absorbed into daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS -# ^^ run manually: bash Orchestrators/media_management.sh --dry-run -# # ━━━ Rsync — Appdata Profiles ━━━ #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack #/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Critical-Data @@ -325,16 +352,13 @@ # emby-failover — every 30-60min via frequent cron above # # ━━━ Weekly — Sunday morning block ━━━ -# 30 2 * * 0 weekly_sync_maintenance.sh (clean Emby + auth stack — cache resets weekly) -# 0 3 * * 0 docker_weekly_restart.sh -# 0 5 * * 0 clear_logs.sh -# 0 6 * * 0 zfs_memory_snapshot.sh -# 0 7 * * 0 smart_health.sh -# 0 8 * * 0 weekly_health_digest.sh (weekly profile sends today) -# 0 9 * * 0 cert_monitor.sh -# 0 10 * * 0 backup_verify.sh -# 0 11 * * 0 emby_session_report.sh -# 0 11 * * 0 bandwidth_monitor.sh --report +# 30 2 * * 0 weekly_sync_maintenance.sh (clean sync + updates + docker weekly restart) +# 0 7 * * 0 sunday_morning_coffee_report.sh (weekly health digest — ready when you wake up) +# +# Individual monitor scripts still available for manual runs: +# clear_logs.sh, zfs_memory_snapshot.sh, smart_health.sh +# cert_monitor.sh, backup_verify.sh, emby_session_report.sh +# weekly_health_digest.sh, bandwidth_monitor.sh --report # # ━━━ Bandwidth monitor ━━━ # bandwidth_monitor.sh --log-transfer called automatically by rsync.sh