added manual neuclear option to arrs cleanup
This commit is contained in:
+19
-5
@@ -16,7 +16,7 @@
|
||||
#
|
||||
# Section Description
|
||||
# ───────────────────────────────────────────────────────────────────────────────────────────
|
||||
# HOST CONFIGURATION Server hostnames, SSH keys, and Emby connection details
|
||||
# HOST CONFIGURATION Server hostnames, SSH keys, Emby connection details, DATA_DIR
|
||||
# LOGGING Enable or disable verbose logging
|
||||
# NOTIFICATIONS unRAID native and Discord webhook settings
|
||||
# GIT / REPO Gitea repository and SSH settings
|
||||
@@ -90,6 +90,12 @@
|
||||
HOST1="unRAID-Gmer4Lfe"
|
||||
HOST2="unRAID-Jayred365"
|
||||
|
||||
# Data directory — persistent script state and statistics files.
|
||||
# Array share — survives reboots, no flash drive wear.
|
||||
# Created automatically if it doesn't exist.
|
||||
# Only truly critical files (failover state, watchdog reboot log) stay on /boot/config.
|
||||
DATA_DIR="/mnt/user/appdata/unraid_scripts/data"
|
||||
|
||||
# SSH keys for server-to-server rsync and failover container operations.
|
||||
# Both keys must be in /root/.ssh/ and authorised in the remote server's authorized_keys.
|
||||
HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
|
||||
@@ -735,7 +741,8 @@ WATCHDOG_SCAN_IGNORE=(
|
||||
# Skip list auto-clears when container recovers healthy
|
||||
WATCHDOG_CONTAINER_RESTART_LIMIT=3 # max watchdog restarts allowed in window
|
||||
WATCHDOG_CONTAINER_RESTART_WINDOW=1 # rolling window in hours
|
||||
WATCHDOG_CONTAINER_RESTART_LOG="/boot/config/container_restart_history.db"
|
||||
WATCHDOG_CONTAINER_RESTART_LOG="$DATA_DIR/container_restart_history.db"
|
||||
# rolling restart history for loop detection
|
||||
|
||||
# Notification batching — one clean summary per cycle instead of one ping per event
|
||||
# true = batch all events into one notification at end of cycle
|
||||
@@ -918,7 +925,7 @@ LIDARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.lrc")
|
||||
LIDARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this
|
||||
LIDARR_MIN_TRACKED_PCT=80 # abort if tracked count drops below this % of last run
|
||||
# protects against API returning partial data on a bad day
|
||||
LIDARR_TRACKED_COUNT_FILE="/boot/config/lidarr_tracked.count"
|
||||
LIDARR_TRACKED_COUNT_FILE="$DATA_DIR/lidarr_tracked.count"
|
||||
# persists last known tracked count for percentage comparison
|
||||
|
||||
# ── Sonarr ────────────────────────────────────────────────────────────────────────────────────
|
||||
@@ -946,6 +953,7 @@ declare -A HOST2_SONARR_PATH_MAP=(
|
||||
)
|
||||
|
||||
SONARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion
|
||||
SONARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this
|
||||
SONARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "ts" "wmv" "mov")
|
||||
SONARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa")
|
||||
# NEVER deleted — cover art, metadata, subtitles
|
||||
@@ -976,6 +984,7 @@ declare -A HOST2_RADARR_PATH_MAP=(
|
||||
)
|
||||
|
||||
RADARR_ORPHAN_AGE=7 # days — files must be older than this before eligible for deletion
|
||||
RADARR_MAX_DELETE_GB=1 # require --i-know-what-im-doing if deletion exceeds this
|
||||
RADARR_EXTENSIONS=("mkv" "mp4" "avi" "m4v" "wmv" "mov")
|
||||
RADARR_PROTECTED_PATTERNS=("*.jpg" "*.jpeg" "*.png" "*.nfo" "*.srt" "*.sub" "*.ass" "*.ssa")
|
||||
# NEVER deleted — cover art, metadata, subtitles
|
||||
@@ -1081,7 +1090,7 @@ HOST2_RADARR_RECOVERY=true # Anime_Movies import recovery
|
||||
# Daily statistics log — read by weekly_health_digest.sh for transcode summary
|
||||
# Tracks peak usage, flip count, session ratio, files cleaned per day
|
||||
# Bounded to TRANSCODE_LOG_RETENTION days — auto-purges old entries on each write
|
||||
TRANSCODE_DAILY_LOG="/boot/config/transcode_daily.db"
|
||||
TRANSCODE_DAILY_LOG="$DATA_DIR/transcode_daily.db"
|
||||
TRANSCODE_LOG_RETENTION=90 # days before old entries are purged
|
||||
|
||||
# ━━━ Transcode Server Array ━━━
|
||||
@@ -1164,10 +1173,15 @@ ZFS_REPORT_IGNORE_POOLS=(
|
||||
# Tracks transfer size, duration and profile per sync for weekly summary reporting.
|
||||
# BANDWIDTH_LOG_RETENTION = days to keep entries before auto-purging old records
|
||||
# BANDWIDTH_WARN_GB = flag in weekly summary if a single sync exceeded this size
|
||||
BANDWIDTH_LOG="/boot/config/bandwidth_history.db"
|
||||
BANDWIDTH_LOG="$DATA_DIR/bandwidth_history.db"
|
||||
BANDWIDTH_LOG_RETENTION=90 # days before old entries are purged
|
||||
BANDWIDTH_WARN_GB=50 # flag syncs larger than this in weekly report
|
||||
|
||||
# Stats files — written by cleanup and recovery scripts, read by coffee report
|
||||
# All in DATA_DIR — array always running when these are written
|
||||
ARR_CLEANUP_STATS="$DATA_DIR/arr_cleanup_stats.db" # lidarr/sonarr/radarr orphan stats
|
||||
ARR_RECOVERY_STATS="$DATA_DIR/arr_recovery_stats.db" # blocklist + re-search stats
|
||||
|
||||
# ━━━ Health Digest ━━━
|
||||
# Aggregated system health summary — reads existing state files, no new writes.
|
||||
# Three profiles control when the digest email is sent:
|
||||
|
||||
@@ -1,10 +1,18 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Arrs Failed Stalled Recovery ----------------------------------------
|
||||
# --------------------------------- Arrs Failed Stalled Recovery --------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Automatically detects and recovers from failed imports and stalled downloads
|
||||
# across Sonarr, Radarr, and Lidarr. Blocklists the bad release and triggers
|
||||
# a new search — hands free recovery while you sleep.
|
||||
# Schedule: 0 */6 * * * (every 6 hours)
|
||||
#
|
||||
# What it checks (per-arr toggles in Master.conf):
|
||||
# HOST1: Sonarr (Tv_Shows) — /api/v3/
|
||||
# HOST1: Radarr (Movies) — /api/v3/
|
||||
# HOST1: Lidarr (Music) — /api/v1/ ← HOST1 only, exits cleanly on HOST2
|
||||
# HOST2: Sonarr (Anime_Shows) — /api/v3/
|
||||
# HOST2: Radarr (Anime_Movies) — /api/v3/
|
||||
#
|
||||
# Targets four problem types from the queue API:
|
||||
# importFailed — downloaded successfully but arr couldn't import the file
|
||||
@@ -12,6 +20,7 @@
|
||||
# error status — serious failure not covered by importFailed/importPending
|
||||
# stalled — download stuck with no connections or no progress
|
||||
#
|
||||
# Items newer than ARR_IMPORT_RECOVERY_AGE (6hr) are skipped — gives arr time to retry.
|
||||
# Never touches items with state "downloading" or "imported" — safe to run anytime.
|
||||
#
|
||||
# Action per problem item:
|
||||
@@ -19,6 +28,12 @@
|
||||
# 2. Remove from queue — cleans up the failed item
|
||||
# 3. Trigger new search — finds a different release automatically
|
||||
#
|
||||
# Configuration in Master.conf:
|
||||
# ARR_IMPORT_RECOVERY_AGE — hours before item is eligible
|
||||
# HOST1/2_SONARR_RECOVERY — enable/disable per arr
|
||||
# HOST1/2_RADARR_RECOVERY — enable/disable per arr
|
||||
# HOST1_LIDARR_RECOVERY — enable/disable Lidarr (HOST1 only)
|
||||
#
|
||||
# Age threshold (ARR_IMPORT_RECOVERY_AGE):
|
||||
# Items newer than threshold are skipped — gives the arr time to retry on its own
|
||||
# Items older than threshold have not self-resolved — safe to intervene
|
||||
@@ -357,4 +372,12 @@ elif [[ "$TOTAL_ACTIONED" -gt 0 ]]; then
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE — nothing to recover"
|
||||
fi
|
||||
|
||||
# Write stats for sunday_morning_coffee_report.sh
|
||||
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_RECOVERY_STATS:-}" ]]; then
|
||||
DATE=$(date '+%Y-%m-%d')
|
||||
TIME=$(date '+%H:%M')
|
||||
echo "${DATE}|${TIME}|${TOTAL_ACTIONED}|${TOTAL_SKIPPED}" \
|
||||
>> "$ARR_RECOVERY_STATS" 2>/dev/null || true
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+53
-3
@@ -31,6 +31,14 @@
|
||||
# Required when deletion would exceed LIDARR_MAX_DELETE_GB
|
||||
# Long and annoying by design — cannot be added accidentally
|
||||
#
|
||||
# --skip-strike-list flag:
|
||||
# Bypasses the LIDARR_ORPHAN_AGE age check — deletes recent files too
|
||||
# Combined with --i-know-what-im-doing activates NUCLEAR MODE:
|
||||
# Age check bypassed, size threshold bypassed, deletes on first pass
|
||||
# Use when Soularr/other tool has filled the gaps — clean one-pass wipe
|
||||
# ⚠️ The script author takes NO responsibility for data loss with both flags active
|
||||
# The user accepts full responsibility — this is 100% intentional by design
|
||||
#
|
||||
# Lidarr runs on HOST1 only — music library is HOST1's source of truth.
|
||||
# If run on HOST2 this script exits cleanly with no action.
|
||||
# All configuration in Master.conf under Arr Cleanup section.
|
||||
@@ -42,12 +50,16 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
# Check for --i-know-what-im-doing flag before parse_args
|
||||
# Check for --i-know-what-im-doing and --skip-strike-list flags before parse_args
|
||||
# These flags are filtered out before parse_args sees them to avoid unknown flag errors
|
||||
I_KNOW=false
|
||||
SKIP_STRIKES=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
if [[ "$arg" == "--i-know-what-im-doing" ]]; then
|
||||
I_KNOW=true
|
||||
elif [[ "$arg" == "--skip-strike-list" ]]; then
|
||||
SKIP_STRIKES=true
|
||||
else
|
||||
FILTERED_ARGS+=("$arg")
|
||||
fi
|
||||
@@ -55,6 +67,32 @@ done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# Nuclear mode — both override flags active
|
||||
# Strike system AND size threshold bypassed — deletes on first pass
|
||||
# Script author takes no responsibility for data loss when both flags are used.
|
||||
# This combination is 100% intentional and the user accepts full responsibility.
|
||||
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " Flags: --i-know-what-im-doing --skip-strike-list"
|
||||
echo " Strike system: BYPASSED — deletes on first pass"
|
||||
echo " Size threshold: BYPASSED — no GB limit"
|
||||
echo " Data recovery: NOT POSSIBLE after deletion"
|
||||
echo ""
|
||||
echo " The script author takes no responsibility for data"
|
||||
echo " loss when both flags are used together. This is a"
|
||||
echo " 100% intentional action by the user."
|
||||
echo ""
|
||||
echo " Review the dry run output before proceeding."
|
||||
echo " You have 10 seconds to cancel (Ctrl+C)..."
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
sleep 10
|
||||
echo " Proceeding..."
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
@@ -117,6 +155,9 @@ if [[ ! -d "$LIDARR_MUSIC_ROOT" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Large library scans take time — override default lock warn age
|
||||
[[ -n "${LIDARR_LOCK_WARN_AGE:-}" ]] && LOCK_WARN_AGE="$LIDARR_LOCK_WARN_AGE"
|
||||
|
||||
acquire_lock "wait"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
@@ -156,6 +197,7 @@ esac
|
||||
success "All safety checks passed"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
||||
[[ "$I_KNOW" == true ]] && warn "OVERRIDE — --i-know-what-im-doing flag active"
|
||||
[[ "$SKIP_STRIKES" == true ]] && warn "OVERRIDE — --skip-strike-list flag active — strike system bypassed"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
@@ -349,7 +391,7 @@ while IFS= read -r filepath; do
|
||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||
|
||||
if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]]; then
|
||||
if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then
|
||||
log "RECENT (skipping): $filepath"
|
||||
((RECENT_COUNT++))
|
||||
continue
|
||||
@@ -379,6 +421,7 @@ if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
|
||||
error "Deletion would exceed ${LIDARR_MAX_DELETE_GB}GB threshold — $TOTAL_HUMAN would be deleted"
|
||||
error "Review the ORPHAN lines above carefully before proceeding"
|
||||
error "If this is expected, rerun with: --i-know-what-im-doing"
|
||||
error "To also bypass age check and delete on first pass: add --skip-strike-list"
|
||||
notify "Lidarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} deletion requires --i-know-what-im-doing" "Lidarr Cleanup" "warning"
|
||||
exit 1
|
||||
else
|
||||
@@ -398,7 +441,7 @@ if [[ "$DRY_RUN" == false ]]; then
|
||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||
|
||||
if is_music_file "$filepath"; then
|
||||
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && continue
|
||||
[[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]] && continue
|
||||
fi
|
||||
|
||||
rm -f "$filepath" 2>/dev/null || error "Failed to delete: $filepath"
|
||||
@@ -448,4 +491,11 @@ else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)"
|
||||
notify "Lidarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Lidarr Cleanup" "normal"
|
||||
fi
|
||||
|
||||
# Write stats for sunday_morning_coffee_report.sh
|
||||
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then
|
||||
DATE=$(date '+%Y-%m-%d')
|
||||
echo "${DATE}|lidarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \
|
||||
>> "$ARR_CLEANUP_STATS" 2>/dev/null || true
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+64
-2
@@ -30,7 +30,44 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
# Check for --i-know-what-im-doing and --skip-strike-list flags before parse_args
|
||||
I_KNOW=false
|
||||
SKIP_STRIKES=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
if [[ "$arg" == "--i-know-what-im-doing" ]]; then
|
||||
I_KNOW=true
|
||||
elif [[ "$arg" == "--skip-strike-list" ]]; then
|
||||
SKIP_STRIKES=true
|
||||
else
|
||||
FILTERED_ARGS+=("$arg")
|
||||
fi
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# Nuclear mode disclaimer
|
||||
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " Flags: --i-know-what-im-doing --skip-strike-list"
|
||||
echo " Strike system: BYPASSED — deletes on first pass"
|
||||
echo " Size threshold: BYPASSED — no GB limit"
|
||||
echo " Data recovery: NOT POSSIBLE after deletion"
|
||||
echo ""
|
||||
echo " The script author takes no responsibility for data"
|
||||
echo " loss when both flags are used together. This is a"
|
||||
echo " 100% intentional action by the user."
|
||||
echo ""
|
||||
echo " Review the dry run output before proceeding."
|
||||
echo " You have 10 seconds to cancel (Ctrl+C)..."
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
sleep 10
|
||||
echo " Proceeding..."
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
@@ -243,7 +280,7 @@ while IFS= read -r filepath; do
|
||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||
|
||||
if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]]; then
|
||||
if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then
|
||||
log "RECENT (skipping): $filepath"
|
||||
((RECENT_COUNT++))
|
||||
continue
|
||||
@@ -305,6 +342,24 @@ format_bytes() {
|
||||
ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES)
|
||||
JUNK_HUMAN=$(format_bytes $JUNK_BYTES)
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", ${RADARR_MAX_DELETE_GB:-1} * 1073741824}")
|
||||
|
||||
# Size threshold check
|
||||
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
|
||||
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
|
||||
if [[ "$I_KNOW" != true ]]; then
|
||||
echo ""
|
||||
error "Deletion would exceed ${RADARR_MAX_DELETE_GB:-1}GB threshold — $TOTAL_HUMAN would be deleted"
|
||||
error "Review the ORPHAN lines above carefully before proceeding"
|
||||
error "If this is expected, rerun with: --i-know-what-im-doing"
|
||||
error "To also bypass age check and delete on first pass: add --skip-strike-list"
|
||||
notify "Radarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} deletion requires --i-know-what-im-doing" "Radarr Cleanup" "warning"
|
||||
exit 1
|
||||
else
|
||||
warn "OVERRIDE — deletion is ${TOTAL_HUMAN} — proceeding because --i-know-what-im-doing"
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
@@ -327,4 +382,11 @@ else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed"
|
||||
notify "Radarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Radarr Cleanup" "normal"
|
||||
fi
|
||||
|
||||
# Write stats for sunday_morning_coffee_report.sh
|
||||
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then
|
||||
DATE=$(date '+%Y-%m-%d')
|
||||
echo "${DATE}|radarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \
|
||||
>> "$ARR_CLEANUP_STATS" 2>/dev/null || true
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
+64
-2
@@ -30,7 +30,44 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
# Check for --i-know-what-im-doing and --skip-strike-list flags before parse_args
|
||||
I_KNOW=false
|
||||
SKIP_STRIKES=false
|
||||
FILTERED_ARGS=()
|
||||
for arg in "$@"; do
|
||||
if [[ "$arg" == "--i-know-what-im-doing" ]]; then
|
||||
I_KNOW=true
|
||||
elif [[ "$arg" == "--skip-strike-list" ]]; then
|
||||
SKIP_STRIKES=true
|
||||
else
|
||||
FILTERED_ARGS+=("$arg")
|
||||
fi
|
||||
done
|
||||
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# Nuclear mode disclaimer
|
||||
if [[ "$I_KNOW" == true ]] && [[ "$SKIP_STRIKES" == true ]] && [[ "$DRY_RUN" != true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "⚠️ WARNING — NUCLEAR MODE ACTIVE"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " Flags: --i-know-what-im-doing --skip-strike-list"
|
||||
echo " Strike system: BYPASSED — deletes on first pass"
|
||||
echo " Size threshold: BYPASSED — no GB limit"
|
||||
echo " Data recovery: NOT POSSIBLE after deletion"
|
||||
echo ""
|
||||
echo " The script author takes no responsibility for data"
|
||||
echo " loss when both flags are used together. This is a"
|
||||
echo " 100% intentional action by the user."
|
||||
echo ""
|
||||
echo " Review the dry run output before proceeding."
|
||||
echo " You have 10 seconds to cancel (Ctrl+C)..."
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
sleep 10
|
||||
echo " Proceeding..."
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
@@ -243,7 +280,7 @@ while IFS= read -r filepath; do
|
||||
FILE_MTIME=$(stat -c %Y "$filepath" 2>/dev/null || echo 0)
|
||||
FILE_AGE=$(( NOW - FILE_MTIME ))
|
||||
|
||||
if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]]; then
|
||||
if [[ "$FILE_AGE" -lt "$AGE_SECONDS" ]] && [[ "$SKIP_STRIKES" != true ]]; then
|
||||
log "RECENT (skipping): $filepath"
|
||||
((RECENT_COUNT++))
|
||||
continue
|
||||
@@ -305,6 +342,24 @@ format_bytes() {
|
||||
ORPHAN_HUMAN=$(format_bytes $ORPHAN_BYTES)
|
||||
JUNK_HUMAN=$(format_bytes $JUNK_BYTES)
|
||||
TOTAL_REMOVED=$(( ORPHAN_COUNT + JUNK_COUNT ))
|
||||
TOTAL_DELETE_BYTES=$(( ORPHAN_BYTES + JUNK_BYTES ))
|
||||
MAX_DELETE_BYTES=$(awk "BEGIN {printf \"%d\", ${SONARR_MAX_DELETE_GB:-1} * 1073741824}")
|
||||
|
||||
# Size threshold check
|
||||
if [[ "$TOTAL_DELETE_BYTES" -gt "$MAX_DELETE_BYTES" ]]; then
|
||||
TOTAL_HUMAN=$(awk "BEGIN {printf \"%.1fGB\", $TOTAL_DELETE_BYTES / 1073741824}")
|
||||
if [[ "$I_KNOW" != true ]]; then
|
||||
echo ""
|
||||
error "Deletion would exceed ${SONARR_MAX_DELETE_GB:-1}GB threshold — $TOTAL_HUMAN would be deleted"
|
||||
error "Review the ORPHAN lines above carefully before proceeding"
|
||||
error "If this is expected, rerun with: --i-know-what-im-doing"
|
||||
error "To also bypass age check and delete on first pass: add --skip-strike-list"
|
||||
notify "Sonarr cleanup halted on $(hostname) — ${TOTAL_HUMAN} deletion requires --i-know-what-im-doing" "Sonarr Cleanup" "warning"
|
||||
exit 1
|
||||
else
|
||||
warn "OVERRIDE — deletion is ${TOTAL_HUMAN} — proceeding because --i-know-what-im-doing"
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
@@ -327,4 +382,11 @@ else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE — $TOTAL_REMOVED files removed"
|
||||
notify "Sonarr cleanup on $(hostname) — removed $TOTAL_REMOVED files (orphans: $ORPHAN_HUMAN junk: $JUNK_HUMAN)" "Sonarr Cleanup" "normal"
|
||||
fi
|
||||
|
||||
# Write stats for sunday_morning_coffee_report.sh
|
||||
if [[ "$DRY_RUN" == false ]] && [[ -n "${ARR_CLEANUP_STATS:-}" ]]; then
|
||||
DATE=$(date '+%Y-%m-%d')
|
||||
echo "${DATE}|sonarr|${ORPHAN_COUNT}|${ORPHAN_BYTES}|${JUNK_COUNT}|${JUNK_BYTES}|${RECENT_COUNT}|${TRACKED_COUNT}" \
|
||||
>> "$ARR_CLEANUP_STATS" 2>/dev/null || true
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -2,21 +2,25 @@
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Array Start Orchestrator -----------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Launches all scripts configured in ARRAY_START_SCRIPTS when the unRAID array comes online.
|
||||
# Set this script to run at "Startup of Array" in the User Scripts plugin.
|
||||
# Single entry point for "At Startup of Array" in User Scripts plugin.
|
||||
# Launches everything configured in ARRAY_START_SCRIPTS in Master.conf.
|
||||
#
|
||||
# Each script is launched as a background process:
|
||||
# One-shot scripts (ramdisk_setup, syslog_filter etc.) run and exit naturally
|
||||
# Continuous scripts (system_watchdog, docker_watchdog, failover) run until array stops
|
||||
# What it launches (configured in Master.conf ARRAY_START_SCRIPTS):
|
||||
# Transcodes/ramdisk_setup.sh — creates tmpfs + symlink before Emby starts (one-shot)
|
||||
# unRAID_Essentials/docker_syslog_filter.sh — suppress veth log noise (one-shot)
|
||||
# unRAID_Essentials/php_fpm_max_children.sh — WebGUI performance tuning (one-shot)
|
||||
# Docker_Essentials/docker_network_connect.sh — connect containers to extra networks (one-shot)
|
||||
# unRAID_Essentials/system_watchdog.sh — system health monitor (continuous loop)
|
||||
# Docker_Essentials/docker_watchdog.sh — container health monitor (continuous loop)
|
||||
# Failover/failover.sh — mutual failover monitor (continuous loop)
|
||||
#
|
||||
# Scripts are launched in the order defined in ARRAY_START_SCRIPTS in Master.conf.
|
||||
# Order matters — ramdisk before Emby, network before watchdogs, watchdogs before failover.
|
||||
#
|
||||
# To add or remove a script: edit ARRAY_START_SCRIPTS in Master.conf.
|
||||
# No changes to this script needed.
|
||||
#
|
||||
# Logs: each script logs its own output independently.
|
||||
# One-shot scripts run and exit naturally — array_start.sh confirms completion.
|
||||
# Continuous scripts run until array stops or SIGTERM received.
|
||||
# This orchestrator exits after launching all scripts — unRAID sees it complete normally.
|
||||
#
|
||||
# Add or remove scripts: edit ARRAY_START_SCRIPTS in Master.conf.
|
||||
# Order matters — ramdisk first, network before watchdogs, watchdogs before failover.
|
||||
# No changes to this script ever needed.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
@@ -3,23 +3,29 @@
|
||||
# --------------------------------- Daily Sync Maintenance ------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Daily orchestrator — runs the full daily maintenance window in the correct order.
|
||||
# Schedule: 0 1 * * * (1am daily)
|
||||
#
|
||||
# What it does:
|
||||
# 1. Iterates DAILY_MAINTENANCE_SCRIPTS — runs git pull first, then additional jobs
|
||||
# 2. Syncs all media shares in the correct direction for the local server
|
||||
# 3. Additional scripts in DAILY_MAINTENANCE_SCRIPTS run after the sync completes
|
||||
# Execution order:
|
||||
# 1. git_pull_execute.sh — pull latest scripts first, always
|
||||
# 2. Media share sync — HOST*_DAILY_SYNC_SHARES pushed to remote
|
||||
# 3. HOST*_PERSONAL_SHARES — personal encrypted shares after media
|
||||
# 4. media_shares_permissions.sh — fix ownership before arr cleanup
|
||||
# 5. media_cleaner.sh anime — remove junk from anime shares
|
||||
# 6. media_cleaner.sh media — remove junk from media shares
|
||||
# 7. lidarr_cleanup.sh — remove orphaned music files
|
||||
# 8. sonarr_cleanup.sh — remove orphaned TV files
|
||||
# 9. radarr_cleanup.sh — remove orphaned movie files
|
||||
# 10. docker_daily_restart.sh — restart containers that need daily restart
|
||||
#
|
||||
# Media share sync:
|
||||
# Each server pushes only the shares it owns (source of truth) — direction is automatic.
|
||||
# HOST1 pushes: Movies, Tv_Shows, Music, Books etc. → HOST2
|
||||
# HOST2 pushes: Anime_Shows, Anime_Movies → HOST1
|
||||
# Personal encrypted shares synced after media shares.
|
||||
# detect_hosts() determines which server is running — no script changes needed.
|
||||
# Share lists configured in Master.conf ORCHESTRATORS section.
|
||||
# Configuration in Master.conf:
|
||||
# DAILY_MAINTENANCE_SCRIPTS — pre/post-sync scripts (git pull, docker restart)
|
||||
# MEDIA_MANAGEMENT_JOBS — media maintenance jobs run after sync
|
||||
# HOST1_DAILY_SYNC_SHARES — shares HOST1 pushes to HOST2
|
||||
# HOST2_DAILY_SYNC_SHARES — shares HOST2 pushes to HOST1
|
||||
# HOST1/2_PERSONAL_SHARES — encrypted personal shares
|
||||
#
|
||||
# Bidirectional — same script runs on both servers, correct direction automatic.
|
||||
# Per-share rsync handled by rsync.sh — this script tracks pass/fail and total time.
|
||||
# All job lists configured in Master.conf — no script changes needed to add or remove jobs.
|
||||
# Schedule: 0 1 * * * (1am daily — configured in User Scripts plugin)
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
@@ -0,0 +1,633 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------------------- Sunday Morning Coffee Report -----------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Weekly system overview — everything that happened this week in one clean read.
|
||||
# Designed to be read over coffee Sunday morning while the system is fully caught up
|
||||
# from the 2:30am maintenance window.
|
||||
# Schedule: 0 7 * * 0 (7am Sunday — after weekly_sync_maintenance.sh finishes at ~3am)
|
||||
# Maintenance window completes → 4 hours of fresh data → report ready ☕
|
||||
#
|
||||
# Sections:
|
||||
# 🖥️ System — uptime, memory, boot drive, cache drive, reboots
|
||||
# 📀 Array — disk count, parity status, ZFS health, drive temps
|
||||
# 🔀 Failover — state, last change, Tailscale connectivity
|
||||
# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split
|
||||
# 🎵 Media Activity — arr cleanup stats, arr recovery stats, library health
|
||||
# 🌐 Rsync — weekly transfer totals, per-share breakdown
|
||||
# 🛡️ Watchdog — container strikes, restarts, system strikes, skip list
|
||||
# 🔐 Security — SSL cert expiry per domain
|
||||
# 📊 Emby — weekly stream count, top users, top content
|
||||
# ⚙️ Health — SMART summary, docker container count, Gitea sync status
|
||||
# ⚠️ Issues — anything requiring attention collected above
|
||||
#
|
||||
# Data sources (reads only — no writes except the notification):
|
||||
# DATA_DIR stats files — arr cleanup, recovery, transcode, bandwidth history
|
||||
# /boot/config — failover state, watchdog reboot log
|
||||
# /tmp — watchdog strike state files
|
||||
# /proc, /sys — system memory, uptime
|
||||
# /var/local/emhttp/ — unRAID array info
|
||||
# Emby API — session history
|
||||
# Arr APIs — current queue depth
|
||||
# tailscale — network status
|
||||
# openssl — live SSL cert check
|
||||
# smartctl — drive health
|
||||
#
|
||||
# All configuration in Master.conf.
|
||||
# Supports --dry-run to preview report without sending notification.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
REPORT_DATE=$(date '+%A, %B %-d, %Y')
|
||||
WEEK_START=$(date -d "7 days ago" '+%Y-%m-%d')
|
||||
TODAY=$(date '+%Y-%m-%d')
|
||||
NOW=$(date +%s)
|
||||
|
||||
REPORT=() # all report lines
|
||||
ISSUES=() # items needing attention
|
||||
FINDINGS=() # notable but not critical
|
||||
|
||||
section() {
|
||||
REPORT+=("")
|
||||
REPORT+=("$1")
|
||||
REPORT+=("$(printf '%.0s─' {1..50})")
|
||||
}
|
||||
|
||||
line() {
|
||||
REPORT+=(" $1")
|
||||
}
|
||||
|
||||
issue() {
|
||||
ISSUES+=("$1")
|
||||
REPORT+=(" ⚠️ $1")
|
||||
}
|
||||
|
||||
finding() {
|
||||
FINDINGS+=("$1")
|
||||
REPORT+=(" ℹ️ $1")
|
||||
}
|
||||
|
||||
format_bytes() {
|
||||
local bytes=$1
|
||||
if (( bytes > 1073741824 )); then
|
||||
awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}"
|
||||
elif (( bytes > 1048576 )); then
|
||||
awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}"
|
||||
else
|
||||
echo "${bytes}B"
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ 🖥️ System ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
section "🖥️ SYSTEM"
|
||||
|
||||
# Uptime
|
||||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||||
UPTIME_DAYS=$(( UPTIME_SECONDS / 86400 ))
|
||||
UPTIME_HOURS=$(( (UPTIME_SECONDS % 86400) / 3600 ))
|
||||
BOOT_TIME=$(date -d "@$(( NOW - UPTIME_SECONDS ))" '+%A %-d %b at %-I:%M%p')
|
||||
line "Uptime: ${UPTIME_DAYS}d ${UPTIME_HOURS}hr (up since $BOOT_TIME)"
|
||||
|
||||
# Reboot history this week
|
||||
REBOOT_COUNT=0
|
||||
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
|
||||
WEEK_EPOCH=$(date -d "$WEEK_START" +%s)
|
||||
REBOOT_COUNT=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \
|
||||
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
|
||||
fi
|
||||
if [[ "$REBOOT_COUNT" -gt 0 ]]; then
|
||||
issue "Reboots this week: $REBOOT_COUNT (system watchdog triggered)"
|
||||
else
|
||||
line "Reboots this week: 0 ✅"
|
||||
fi
|
||||
|
||||
# Memory
|
||||
MEM_TOTAL_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
|
||||
MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_USED_KB=$(( MEM_TOTAL_KB - MEM_AVAIL_KB ))
|
||||
MEM_TOTAL_GB=$(awk "BEGIN {printf \"%.0f\", $MEM_TOTAL_KB / 1048576}")
|
||||
MEM_USED_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_USED_KB / 1048576}")
|
||||
MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}")
|
||||
|
||||
ARC_SIZE=0
|
||||
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||||
ARC_BYTES=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||||
ARC_SIZE=$(awk "BEGIN {printf \"%.1f\", $ARC_BYTES / 1073741824}")
|
||||
fi
|
||||
line "Memory: ${MEM_USED_GB}GB used / ${MEM_TOTAL_GB}GB total (ARC: ${ARC_SIZE}GB)"
|
||||
|
||||
# Boot drive
|
||||
BOOT_PCT=$(df /boot --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||||
BOOT_USED=$(df /boot -h --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||||
BOOT_SIZE=$(df /boot -h --output=size 2>/dev/null | tail -1 | tr -d ' ')
|
||||
if [[ "${BOOT_PCT:-0}" -ge 80 ]]; then
|
||||
issue "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE}) — getting full"
|
||||
else
|
||||
line "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE})"
|
||||
fi
|
||||
|
||||
# Cache drive
|
||||
CACHE_PCT=$(df /mnt/cache --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||||
CACHE_AVAIL=$(df /mnt/cache -h --output=avail 2>/dev/null | tail -1 | tr -d ' ')
|
||||
CACHE_SIZE=$(df /mnt/cache -h --output=size 2>/dev/null | tail -1 | tr -d ' ')
|
||||
if [[ "${CACHE_PCT:-0}" -ge 85 ]]; then
|
||||
issue "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})"
|
||||
else
|
||||
line "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ 📀 Array ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
section "📀 ARRAY"
|
||||
|
||||
# unRAID array info
|
||||
if [[ -f /var/local/emhttp/disks.ini ]]; then
|
||||
DISK_COUNT=$(grep -c "^\[disk" /var/local/emhttp/disks.ini 2>/dev/null || echo "?")
|
||||
line "Array disks: $DISK_COUNT"
|
||||
fi
|
||||
|
||||
# Parity
|
||||
if [[ -f /var/local/emhttp/parity-date.txt ]]; then
|
||||
PARITY_INFO=$(cat /var/local/emhttp/parity-date.txt 2>/dev/null)
|
||||
if echo "$PARITY_INFO" | grep -q "progress"; then
|
||||
finding "Parity check in progress"
|
||||
else
|
||||
PARITY_DATE=$(echo "$PARITY_INFO" | grep -oE '[0-9]{4}-[0-9]{2}-[0-9]{2}' | head -1)
|
||||
PARITY_ERRORS=$(echo "$PARITY_INFO" | grep -oE 'errors=[0-9]+' | cut -d= -f2 || echo 0)
|
||||
if [[ "${PARITY_ERRORS:-0}" -gt 0 ]]; then
|
||||
issue "Parity: $PARITY_ERRORS errors on last check ($PARITY_DATE)"
|
||||
else
|
||||
line "Parity: OK (last check: ${PARITY_DATE:-unknown}, 0 errors)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ZFS pool health
|
||||
if command -v zpool >/dev/null 2>&1; then
|
||||
POOLS=$(zpool list -H -o name,health 2>/dev/null)
|
||||
while IFS=$'\t' read -r pool health; do
|
||||
[[ -z "$pool" ]] && continue
|
||||
# Skip single-disk unRAID array pools
|
||||
if [[ "$health" == "ONLINE" ]]; then
|
||||
line "ZFS $pool: ONLINE ✅"
|
||||
else
|
||||
issue "ZFS $pool: $health — check immediately"
|
||||
fi
|
||||
done <<< "$POOLS"
|
||||
fi
|
||||
|
||||
# Drive temperatures from SMART
|
||||
if command -v smartctl >/dev/null 2>&1; then
|
||||
TEMP_WARN=false
|
||||
TEMP_SUMMARY=""
|
||||
for disk in /dev/sd? /dev/nvme?; do
|
||||
[[ ! -e "$disk" ]] && continue
|
||||
DISK_NAME=$(basename "$disk")
|
||||
# Skip ignored drives
|
||||
SKIP=false
|
||||
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
|
||||
[[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break
|
||||
done
|
||||
[[ "$SKIP" == true ]] && continue
|
||||
TEMP=$(smartctl -A "$disk" 2>/dev/null | \
|
||||
awk '/Temperature_Celsius|Temperature:/ {print $NF; exit}' | tr -d '°C')
|
||||
[[ -z "$TEMP" ]] && continue
|
||||
TEMP_INT=$(printf "%.0f" "$TEMP" 2>/dev/null || echo 0)
|
||||
if [[ "$TEMP_INT" -ge "${SMART_TEMP_CRIT:-55}" ]]; then
|
||||
issue "Drive $DISK_NAME: ${TEMP_INT}°C — CRITICAL"
|
||||
TEMP_WARN=true
|
||||
elif [[ "$TEMP_INT" -ge "${SMART_TEMP_WARN:-45}" ]]; then
|
||||
finding "Drive $DISK_NAME: ${TEMP_INT}°C — warm"
|
||||
TEMP_WARN=true
|
||||
fi
|
||||
[[ -n "$TEMP_SUMMARY" ]] && TEMP_SUMMARY+=", "
|
||||
TEMP_SUMMARY+="${DISK_NAME}:${TEMP_INT}°C"
|
||||
done
|
||||
if [[ "$TEMP_WARN" == false ]]; then
|
||||
line "Drive temps: all normal"
|
||||
fi
|
||||
[[ -n "$TEMP_SUMMARY" ]] && line "Temps: $TEMP_SUMMARY"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ 🔀 Failover ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
section "🔀 FAILOVER"
|
||||
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
FAILOVER_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
if [[ "$FAILOVER_STATE" == "NORMAL" ]]; then
|
||||
line "State: NORMAL ✅"
|
||||
else
|
||||
issue "Failover state: $FAILOVER_STATE — not normal"
|
||||
fi
|
||||
line "Last change: ${FAILOVER_CHANGE:-unknown}"
|
||||
else
|
||||
issue "Failover state file not found"
|
||||
fi
|
||||
|
||||
# Tailscale connectivity
|
||||
if command -v tailscale >/dev/null 2>&1; then
|
||||
TS_STATUS=$(tailscale status 2>/dev/null)
|
||||
REMOTE_VISIBLE=$(echo "$TS_STATUS" | grep -c "$HOST2" 2>/dev/null || echo 0)
|
||||
if [[ "$REMOTE_VISIBLE" -gt 0 ]]; then
|
||||
REMOTE_IP=$(tailscale ip -4 "$HOST2" 2>/dev/null || echo "unknown")
|
||||
line "Tailscale: $HOST2 visible at $REMOTE_IP ✅"
|
||||
else
|
||||
issue "Tailscale: $HOST2 not visible — check connectivity"
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ 🎬 Transcodes ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
section "🎬 TRANSCODES"
|
||||
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
|
||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
|
||||
RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail | tail -1 | tr -d ' ')
|
||||
RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}")
|
||||
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null | xargs basename 2>/dev/null || echo "unknown")
|
||||
line "Ramdisk now: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET"
|
||||
else
|
||||
issue "Ramdisk not mounted"
|
||||
fi
|
||||
|
||||
if [[ -f "$TRANSCODE_DAILY_LOG" ]]; then
|
||||
WEEK_PEAK=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff {if ($2 > max) max=$2} END {printf "%.2f", max+0}' \
|
||||
"$TRANSCODE_DAILY_LOG")
|
||||
WEEK_FLIPS=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff {sum+=$3} END {print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||||
WEEK_RAM=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff {sum+=$4} END {print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||||
WEEK_SSD=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff {sum+=$5} END {print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||||
WEEK_FILES=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff {sum+=$6} END {print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||||
|
||||
line "Week peak: ${WEEK_PEAK}GB | flips: ${WEEK_FLIPS} | files cleaned: ${WEEK_FILES}"
|
||||
line "Sessions: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD"
|
||||
|
||||
PEAK_INT=$(printf "%.0f" "$WEEK_PEAK" 2>/dev/null || echo 0)
|
||||
WARN_INT=$(printf "%.0f" "$RAMDISK_WARN_GB" 2>/dev/null || echo 0)
|
||||
if [[ "$PEAK_INT" -ge "$WARN_INT" ]]; then
|
||||
finding "Transcode peak ${WEEK_PEAK}GB reached warn threshold — consider increasing RAMDISK_SIZE"
|
||||
fi
|
||||
if [[ "${WEEK_FLIPS:-0}" -ge "${TRANSCODE_FLIP_WARN:-3}" ]]; then
|
||||
finding "Transcode flips this week: $WEEK_FLIPS — monitor ramdisk headroom"
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ 🎵 Media Activity ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
section "🎵 MEDIA ACTIVITY"
|
||||
|
||||
# Arr cleanup stats
|
||||
if [[ -f "$ARR_CLEANUP_STATS" ]]; then
|
||||
for arr in lidarr sonarr radarr; do
|
||||
WEEK_ORPHANS=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \
|
||||
'$1 >= cutoff && $2 == a {sum+=$3} END {print sum+0}' "$ARR_CLEANUP_STATS")
|
||||
WEEK_BYTES=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \
|
||||
'$1 >= cutoff && $2 == a {sum+=$4} END {print sum+0}' "$ARR_CLEANUP_STATS")
|
||||
WEEK_TRACKED=$(awk -F'|' -v cutoff="$WEEK_START" -v a="$arr" \
|
||||
'BEGIN{max=0} $1 >= cutoff && $2 == a && $8 > max {max=$8} END {print max+0}' \
|
||||
"$ARR_CLEANUP_STATS")
|
||||
if [[ "${WEEK_ORPHANS:-0}" -gt 0 ]]; then
|
||||
FREED=$(format_bytes "${WEEK_BYTES:-0}")
|
||||
line "${arr^} cleanup: $WEEK_ORPHANS orphans removed ($FREED freed) | tracked: $WEEK_TRACKED files"
|
||||
else
|
||||
line "${arr^} cleanup: clean ✅ (tracked: $WEEK_TRACKED files)"
|
||||
fi
|
||||
done
|
||||
else
|
||||
line "Arr cleanup stats: no data yet (runs after first cleanup)"
|
||||
fi
|
||||
|
||||
# Arr recovery stats
|
||||
if [[ -f "$ARR_RECOVERY_STATS" ]]; then
|
||||
WEEK_ACTIONED=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff {sum+=$3} END {print sum+0}' "$ARR_RECOVERY_STATS")
|
||||
WEEK_RUNS=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff {count++} END {print count+0}' "$ARR_RECOVERY_STATS")
|
||||
if [[ "${WEEK_ACTIONED:-0}" -gt 0 ]]; then
|
||||
line "Arr recovery: $WEEK_ACTIONED items auto-recovered across $WEEK_RUNS runs"
|
||||
else
|
||||
line "Arr recovery: no failed imports this week ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Arr queue depth — current snapshot
|
||||
for arr_name in "Sonarr|${HOST1_SONARR_URL}|${HOST1_SONARR_API_KEY}|v3" \
|
||||
"Radarr|${HOST1_RADARR_URL}|${HOST1_RADARR_API_KEY}|v3" \
|
||||
"Lidarr|${HOST1_LIDARR_URL}|${HOST1_LIDARR_API_KEY}|v1"; do
|
||||
IFS='|' read -r name url key ver <<< "$arr_name"
|
||||
if [[ "$url" == *"your-"* ]] || [[ -z "$key" ]]; then continue; fi
|
||||
QUEUE=$(curl -sf --max-time 5 -H "X-Api-Key: $key" \
|
||||
"${url}/api/${ver}/queue?pageSize=1" 2>/dev/null | \
|
||||
python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('totalRecords',0))" \
|
||||
2>/dev/null || echo "?")
|
||||
if [[ "$QUEUE" == "0" ]] || [[ -z "$QUEUE" ]]; then
|
||||
line "$name queue: empty ✅"
|
||||
else
|
||||
line "$name queue: $QUEUE items"
|
||||
fi
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ 🌐 Rsync ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
section "🌐 RSYNC"
|
||||
|
||||
if [[ -f "$BANDWIDTH_LOG" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
|
||||
WEEK_TOTAL_BYTES=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff {sum+=$3} END {print sum+0}' "$BANDWIDTH_LOG")
|
||||
WEEK_SYNCS=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff' "$BANDWIDTH_LOG" | wc -l)
|
||||
WEEK_FAILED=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff && $5 != "success"' "$BANDWIDTH_LOG" | wc -l)
|
||||
WEEK_GB=$(awk "BEGIN {printf \"%.1f\", $WEEK_TOTAL_BYTES / 1073741824}")
|
||||
|
||||
line "Total transferred: ${WEEK_GB}GB across $WEEK_SYNCS syncs"
|
||||
if [[ "${WEEK_FAILED:-0}" -gt 0 ]]; then
|
||||
issue "Failed syncs this week: $WEEK_FAILED"
|
||||
else
|
||||
line "Sync failures: none ✅"
|
||||
fi
|
||||
|
||||
# Top 3 shares by transfer this week
|
||||
TOP_SHARES=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$1 >= cutoff {bytes[$4]+=$3} END {for(s in bytes) print bytes[s], s}' \
|
||||
"$BANDWIDTH_LOG" | sort -rn | head -3)
|
||||
if [[ -n "$TOP_SHARES" ]]; then
|
||||
while IFS=' ' read -r bytes share; do
|
||||
[[ -z "$share" ]] && continue
|
||||
SIZE=$(format_bytes "$bytes")
|
||||
line " → $share: $SIZE"
|
||||
done <<< "$TOP_SHARES"
|
||||
fi
|
||||
else
|
||||
line "No bandwidth data yet"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ 🛡️ Watchdog ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
section "🛡️ WATCHDOG"
|
||||
|
||||
# Container watchdog strikes
|
||||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$" | wc -l)
|
||||
if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then
|
||||
STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
|
||||
issue "Container watchdog: $ACTIVE_STRIKES active strikes — $STRIKE_LIST"
|
||||
else
|
||||
line "Container watchdog: no active strikes ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
# System watchdog strikes
|
||||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||||
SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$" | wc -l)
|
||||
if [[ "$SYS_STRIKES" -gt 0 ]]; then
|
||||
SYS_LIST=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
|
||||
issue "System watchdog: $SYS_STRIKES active strikes — $SYS_LIST"
|
||||
else
|
||||
line "System watchdog: no active strikes ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Container skip list
|
||||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||||
SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ')
|
||||
issue "Skip list: $SKIP_COUNT containers — $SKIP_LIST — manual intervention needed"
|
||||
else
|
||||
line "Skip list: empty ✅"
|
||||
fi
|
||||
|
||||
# Container restarts this week
|
||||
if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
|
||||
WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l)
|
||||
if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then
|
||||
RESTARTED=$(awk -F'|' -v cutoff="$WEEK_START" \
|
||||
'$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
|
||||
sort | uniq -c | sort -rn | head -5 | \
|
||||
awk '{print $2 "(" $1 ")"}' | tr '\n' ' ')
|
||||
finding "Watchdog restarted $WEEK_RESTARTS containers this week: $RESTARTED"
|
||||
else
|
||||
line "Watchdog restarts: none this week ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Docker container count
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
RUNNING=$(docker ps -q 2>/dev/null | wc -l)
|
||||
TOTAL=$(docker ps -aq 2>/dev/null | wc -l)
|
||||
STOPPED=$(( TOTAL - RUNNING ))
|
||||
if [[ "$STOPPED" -gt 0 ]]; then
|
||||
STOPPED_NAMES=$(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null | \
|
||||
head -5 | tr '\n' ' ')
|
||||
finding "Docker: $RUNNING/$TOTAL running — $STOPPED stopped: $STOPPED_NAMES"
|
||||
else
|
||||
line "Docker: $RUNNING/$TOTAL containers running ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ 🔐 Security ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
section "🔐 SECURITY"
|
||||
|
||||
for domain in "${CERT_MONITOR_DOMAINS[@]:-}"; do
|
||||
[[ -z "$domain" ]] && continue
|
||||
EXPIRY=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \
|
||||
-connect "${domain}:443" -servername "$domain" 2>/dev/null | \
|
||||
openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||||
if [[ -z "$EXPIRY" ]]; then
|
||||
issue "$domain: could not check certificate"
|
||||
continue
|
||||
fi
|
||||
EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s 2>/dev/null || echo 0)
|
||||
DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW) / 86400 ))
|
||||
if [[ "$DAYS_LEFT" -le "${CERT_CRIT_DAYS:-7}" ]]; then
|
||||
issue "$domain: ${DAYS_LEFT} days remaining — CRITICAL, renew now"
|
||||
elif [[ "$DAYS_LEFT" -le "${CERT_WARN_DAYS:-30}" ]]; then
|
||||
finding "$domain: ${DAYS_LEFT} days remaining — renew soon"
|
||||
else
|
||||
line "$domain: ${DAYS_LEFT} days remaining ✅"
|
||||
fi
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ 📊 Emby ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
section "📊 EMBY"
|
||||
|
||||
# Select correct Emby URL and key for this host
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
EMBY_URL="$HOST1_EMBY_URL"
|
||||
EMBY_KEY="$HOST1_EMBY_API_KEY"
|
||||
else
|
||||
EMBY_URL="$HOST2_EMBY_URL"
|
||||
EMBY_KEY="$HOST2_EMBY_API_KEY"
|
||||
fi
|
||||
|
||||
if [[ "$EMBY_KEY" != *"your-"* ]] && [[ -n "$EMBY_KEY" ]]; then
|
||||
WEEK_MS=$(( 7 * 24 * 3600 * 10000000 ))
|
||||
ACTIVITY=$(curl -sf --max-time 10 \
|
||||
-H "X-Emby-Token: $EMBY_KEY" \
|
||||
"${EMBY_URL}/Sessions?ControllableByUserId=&api_key=$EMBY_KEY" \
|
||||
2>/dev/null)
|
||||
|
||||
# Use activity log for weekly stats
|
||||
ACTIVITY_LOG=$(curl -sf --max-time 10 \
|
||||
-H "X-Emby-Token: $EMBY_KEY" \
|
||||
"${EMBY_URL}/user_usage_stats/user_activity?days=7&api_key=$EMBY_KEY" \
|
||||
2>/dev/null)
|
||||
|
||||
if [[ -n "$ACTIVITY_LOG" ]]; then
|
||||
TOTAL_PLAYS=$(echo "$ACTIVITY_LOG" | \
|
||||
python3 -c "import sys,json; d=json.load(sys.stdin); \
|
||||
print(sum(u.get('total_plays',0) for u in d))" 2>/dev/null || echo "?")
|
||||
line "Streams this week: $TOTAL_PLAYS total plays"
|
||||
|
||||
# Top 3 users
|
||||
TOP_USERS=$(echo "$ACTIVITY_LOG" | \
|
||||
python3 -c "
|
||||
import sys,json
|
||||
d=json.load(sys.stdin)
|
||||
users=sorted(d,key=lambda x:x.get('total_plays',0),reverse=True)[:3]
|
||||
for u in users:
|
||||
print(f\" → {u.get('user_name','?')}: {u.get('total_plays',0)} plays\")
|
||||
" 2>/dev/null)
|
||||
[[ -n "$TOP_USERS" ]] && echo "$TOP_USERS" | while IFS= read -r l; do line "$l"; done
|
||||
else
|
||||
# Fallback — just show active sessions
|
||||
ACTIVE=$(curl -sf --max-time 5 \
|
||||
-H "X-Emby-Token: $EMBY_KEY" \
|
||||
"${EMBY_URL}/Sessions?api_key=$EMBY_KEY" 2>/dev/null | \
|
||||
python3 -c "import sys,json; \
|
||||
d=json.load(sys.stdin); \
|
||||
active=[s for s in d if s.get('NowPlayingItem')]; \
|
||||
print(f'{len(active)} active streams now')" 2>/dev/null || echo "API unavailable")
|
||||
line "Emby: $ACTIVE"
|
||||
fi
|
||||
else
|
||||
line "Emby: API key not configured"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ ⚙️ System Health ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
section "⚙️ SYSTEM HEALTH"
|
||||
|
||||
# SMART summary
|
||||
if command -v smartctl >/dev/null 2>&1; then
|
||||
SMART_ISSUES=0
|
||||
for disk in /dev/sd? /dev/nvme?; do
|
||||
[[ ! -e "$disk" ]] && continue
|
||||
DISK_NAME=$(basename "$disk")
|
||||
SKIP=false
|
||||
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
|
||||
[[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break
|
||||
done
|
||||
[[ "$SKIP" == true ]] && continue
|
||||
HEALTH=$(smartctl -H "$disk" 2>/dev/null | grep "SMART overall-health" | \
|
||||
awk '{print $NF}')
|
||||
if [[ "$HEALTH" != "PASSED" ]] && [[ -n "$HEALTH" ]]; then
|
||||
issue "SMART $DISK_NAME: $HEALTH — check immediately"
|
||||
((SMART_ISSUES++))
|
||||
fi
|
||||
done
|
||||
if [[ "$SMART_ISSUES" -eq 0 ]]; then
|
||||
line "SMART: all drives PASSED ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Gitea sync status
|
||||
if command -v git >/dev/null 2>&1 && [[ -d "$TARGET_DIR/.git" ]]; then
|
||||
CURRENT_COMMIT=$(git -C "$TARGET_DIR" rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||||
LAST_PULL=$(git -C "$TARGET_DIR" log -1 --format="%ar" 2>/dev/null || echo "unknown")
|
||||
line "Gitea: commit $CURRENT_COMMIT (pulled $LAST_PULL)"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ ⚠️ Issues Requiring Attention ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ ${#ISSUES[@]} -gt 0 ]]; then
|
||||
REPORT+=("")
|
||||
REPORT+=("⚠️ ISSUES REQUIRING ATTENTION")
|
||||
REPORT+=("$(printf '%.0s─' {1..50})")
|
||||
for issue_line in "${ISSUES[@]}"; do
|
||||
REPORT+=(" ❌ $issue_line")
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ ${#FINDINGS[@]} -gt 0 ]]; then
|
||||
REPORT+=("")
|
||||
REPORT+=("ℹ️ NOTABLE")
|
||||
REPORT+=("$(printf '%.0s─' {1..50})")
|
||||
for finding_line in "${FINDINGS[@]}"; do
|
||||
REPORT+=(" → $finding_line")
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ Footer ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
REPORT+=("")
|
||||
if [[ ${#ISSUES[@]} -eq 0 ]]; then
|
||||
STATUS="✅ All systems healthy — enjoy your Sunday"
|
||||
else
|
||||
STATUS="⚠️ ${#ISSUES[@]} issue(s) need attention"
|
||||
fi
|
||||
REPORT+=("$STATUS")
|
||||
REPORT+=("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ Output and Send ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
HEADER="☕ SUNDAY MORNING COFFEE REPORT — $REPORT_DATE"
|
||||
DIVIDER="$(printf '%.0s━' {1..50})"
|
||||
|
||||
echo ""
|
||||
echo "$DIVIDER"
|
||||
echo "$HEADER"
|
||||
echo "$DIVIDER"
|
||||
|
||||
for report_line in "${REPORT[@]}"; do
|
||||
echo "$report_line"
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
||||
# Send notification
|
||||
BODY=$(printf '%s\n' "$HEADER" "${REPORT[@]}")
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — notification not sent"
|
||||
else
|
||||
notify "$BODY" "☕ Weekly Report" "normal"
|
||||
success "Report sent"
|
||||
fi
|
||||
@@ -1,38 +1,45 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------------------- Weekly Sync Maintenance ------------------------------------
|
||||
# ----------------------------- Weekly Sync Maintenance ----------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Maintenance window orchestrator for Emby and the auth stack (Critical-Data).
|
||||
# Both local and remote containers are stopped for the entire window — clean state
|
||||
# guaranteed for sync, updates, and restart.
|
||||
# Weekly maintenance window orchestrator — clean sync, container updates, weekly restarts.
|
||||
# Schedule: 30 2 * * 0 (Sunday 2:30am — fits before 3am network reboot)
|
||||
#
|
||||
# What it does (in order):
|
||||
# 1. Stop local containers — auth stack + Emby stopped locally
|
||||
# 2. Stop remote containers — auth stack + Emby stopped remotely via SSH
|
||||
# 3. Pull updates locally — if CRITICAL_SYNC_UPDATES=true
|
||||
# 4. Pull updates remotely — if CRITICAL_SYNC_UPDATES_REMOTE=true
|
||||
# 5. rsync Emby — clean full sync, both sides down
|
||||
# 6. rsync Critical-Data — clean full sync, databases fully flushed
|
||||
# 7. Start remote containers — starts on new images, correct order
|
||||
# 8. Start local containers — starts on new images, correct order
|
||||
# Execution order:
|
||||
# 1. Stop local containers — Emby + auth stack stopped locally
|
||||
# 2. Stop remote containers — Emby + auth stack stopped remotely via SSH
|
||||
# 3. Pull updates locally — if CRITICAL_SYNC_UPDATES=true
|
||||
# 4. Pull updates remotely — if CRITICAL_SYNC_UPDATES_REMOTE=true
|
||||
# 5. rsync Emby — full clean mirror, both instances stopped
|
||||
# 6. rsync Critical-Data — auth stack clean sync, databases flushed
|
||||
# 7. Start remote containers — correct order, delayed start respected
|
||||
# 8. Start local containers — correct order, delayed start respected
|
||||
# 9. docker_weekly_restart.sh — weekly container restarts
|
||||
#
|
||||
# Why this is better than separate update + sync jobs:
|
||||
# Containers are already stopped for the sync — no extra downtime for updates
|
||||
# Both sides on identical image versions after restart
|
||||
# Databases synced before first start on new version — clean state guaranteed
|
||||
# One maintenance window handles sync + updates + ordered restart
|
||||
# Synced shares (WEEKLY_SYNC_JOBS in Master.conf):
|
||||
# /mnt/user/Media_Server/Emby — emby profile — full mirror, cache resets weekly
|
||||
# /mnt/user/appdata-Failover/Critical-Data — critical-data — auth stack clean state
|
||||
#
|
||||
# Toggle updates on/off in Master.conf:
|
||||
# CRITICAL_SYNC_UPDATES=true/false — local updates
|
||||
# CRITICAL_SYNC_UPDATES_REMOTE=true/false — remote updates
|
||||
# Both false = sync only (sync only, no updates)
|
||||
# Why weekly instead of nightly for Emby:
|
||||
# Emby builds a warm image cache on HOST2 throughout the week
|
||||
# Syncing nightly resets cache — cold loads every morning for users
|
||||
# Weekly sync: cache stays warm 6 days, resets Sunday night while users sleep
|
||||
# emby-failover dirty sync covers watch states + library every 30-60min between syncs
|
||||
#
|
||||
# Container lists and startup order from PROFILE_CRITICAL_CONTAINER_NAMES
|
||||
# Delayed containers (Authelia, Authelia-Secondary) respected on restart
|
||||
# Containers not found on a server skipped gracefully
|
||||
# Only containers that were running get restarted — stopped containers stay stopped
|
||||
# Container updates during the window:
|
||||
# Containers already stopped for sync — updates pull at zero extra downtime
|
||||
# Both servers start on identical image versions after the window completes
|
||||
# Toggle: CRITICAL_SYNC_UPDATES / CRITICAL_SYNC_UPDATES_REMOTE in Master.conf
|
||||
#
|
||||
# Recommended schedule: 30 2 * * 0 (Sunday 2:30am)
|
||||
# What triggers weekly_health_digest.sh:
|
||||
# NOT this script — weekly_health_digest.sh runs on its own Saturday schedule
|
||||
#
|
||||
# Configuration in Master.conf:
|
||||
# WEEKLY_SYNC_JOBS — shares synced during the maintenance window
|
||||
# WEEKLY_MAINTENANCE_SCRIPTS — scripts run after sync (docker_weekly_restart)
|
||||
# CRITICAL_SYNC_UPDATES — toggle container updates on/off
|
||||
# CRITICAL_SYNC_UPDATES_REMOTE — toggle remote container updates on/off
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# All configuration in Master.conf.
|
||||
# Supports --dry-run to walk through without stopping containers, syncing, or updating.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
@@ -672,6 +672,12 @@ get_rsync_opts() {
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
LOCK_DIR="/tmp/unraid_locks"
|
||||
|
||||
# Ensure DATA_DIR exists — created here so every script that sources common.sh
|
||||
# can safely write to it without checking first
|
||||
if [[ -n "${DATA_DIR:-}" ]] && [[ ! -d "$DATA_DIR" ]]; then
|
||||
mkdir -p "$DATA_DIR" 2>/dev/null || true
|
||||
fi
|
||||
RSYNC_COUNT_FILE="$LOCK_DIR/rsync_active_count"
|
||||
RSYNC_MAX_CONCURRENT=3
|
||||
LOCK_WARN_AGE=300 # seconds — warn if lock older than this (5min default)
|
||||
|
||||
+56
-32
@@ -49,6 +49,14 @@
|
||||
# ├── user_script_plug-in.sh # This file — copy into User Scripts plugin
|
||||
# ├── git_pull_execute.sh # Pulls latest scripts from Gitea repo
|
||||
# │
|
||||
# ├── Orchestrators/
|
||||
# │ ├── array_start.sh # Single entry point — launches all array-start scripts
|
||||
# │ ├── daily_sync_maintenance.sh # Daily — git pull, media sync, media mgmt, docker restart
|
||||
# │ ├── weekly_sync_maintenance.sh # Weekly — critical sync + updates, docker weekly restart
|
||||
# │ ├── media_management.sh # Permissions + cleaners + arr cleanup — run manually
|
||||
# │ ├── transcode_management.sh # Cleanup then manager every 3min + daily stats
|
||||
# │ └── README-Orchestrators.md
|
||||
# │
|
||||
# ├── Failover/
|
||||
# │ ├── failover.sh # Mutual container failover — runs continuously
|
||||
# │ ├── failover_test.sh # Controlled failover simulation — run manually
|
||||
@@ -64,16 +72,6 @@
|
||||
# │ ├── zfs_memory_snapshot.sh # Weekly ZFS health and memory diagnostic report
|
||||
# │ └── README-Monitors.md
|
||||
# │
|
||||
# ├── Orchestrators/
|
||||
# │ ├── array_start.sh # Single entry point — launches all array-start scripts
|
||||
# │ ├── daily_sync_maintenance.sh # Daily — git pull, media sync, media mgmt, docker restart
|
||||
# │ ├── weekly_sync_maintenance.sh # Weekly — critical sync + updates, docker weekly restart
|
||||
# │ ├── daily_sync_maintenance.sh # Media shares sync both directions
|
||||
# │ ├── weekly_sync_maintenance.sh # Clean sync + container updates (Emby + auth stack)
|
||||
# │ ├── media_management.sh # Permissions + cleaners + arr cleanup — run manually
|
||||
# │ ├── transcode_management.sh # Cleanup then manager every 3min + daily stats
|
||||
# │ └── README-Orchestrators.md
|
||||
# │
|
||||
# ├── Rsync/
|
||||
# │ ├── rsync.sh # Core rsync script — called per share or profile
|
||||
# │ └── README-Rsync_Setup.md # Rsync-specific setup guide
|
||||
@@ -137,6 +135,47 @@
|
||||
# in real time — symlink flips work correctly for the lifetime of the container.
|
||||
#
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# ━━━ 🚀 Orchestrators — Uncomment the one below you want to run ━━━
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# ━━━ Orchestrators ━━━
|
||||
# Orchestrators are the main scripts — they handle most if not all needed scripts
|
||||
# in a specific order of operations. Each orchestrator owns a domain and runs
|
||||
# everything within it automatically. All scripts below orchestrators can still
|
||||
# be run separately for testing or manual maintenance.
|
||||
#
|
||||
# array_start.sh — At Startup of Array — single entry, launches everything
|
||||
# ramdisk, syslog filter, php-fpm, network connect (one-shot)
|
||||
# system_watchdog, docker_watchdog, failover (continuous loops)
|
||||
#
|
||||
# daily_sync_maintenance.sh — 0 1 * * * (1am daily)
|
||||
# git pull → media share sync → permissions → cleaners →
|
||||
# arr cleanup → docker daily restart
|
||||
#
|
||||
# weekly_sync_maintenance.sh — 30 2 * * 0 (Sunday 2:30am)
|
||||
# stop containers → pull updates → sync Emby + Critical-Data →
|
||||
# start containers → docker weekly restart
|
||||
#
|
||||
# transcode_management.sh — */3 * * * * (every 3 minutes)
|
||||
# transcode cleanup → transcode manager (correct order)
|
||||
#
|
||||
# sunday_morning_coffee_report.sh — 0 7 * * 0 (Sunday 7am)
|
||||
# full weekly system health digest — ready when you wake up ☕
|
||||
# system, array, failover, transcodes, media activity, rsync,
|
||||
# watchdog, security, Emby stats, health summary
|
||||
#
|
||||
# arrs_failed_stalled_recovery.sh — 0 */6 * * * (every 6 hours)
|
||||
# blocklist + re-search failed imports and stalled downloads
|
||||
# across Sonarr, Radarr, and Lidarr automatically
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━
|
||||
#
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/sunday_morning_coffee_report.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Media/arrs_failed_stalled_recovery.sh
|
||||
#
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
# ━━━ 🚀 Script Commands — Uncomment the one you want to run ━━━
|
||||
# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
#
|
||||
@@ -162,18 +201,6 @@
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/weekly_health_digest.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Monitors/zfs_memory_snapshot.sh
|
||||
#
|
||||
# ━━━ Orchestrators ━━━
|
||||
# array_start.sh is the single User Scripts entry for array startup.
|
||||
# All other orchestrators are scheduled via cron — not started at array start.
|
||||
#
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/array_start.sh
|
||||
# ^^ set to: At Startup of Array
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/daily_sync_maintenance.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/weekly_sync_maintenance.sh
|
||||
#/mnt/user/appdata/unraid_scripts/Orchestrators/transcode_management.sh
|
||||
# ^^ media_management.sh is absorbed into daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS
|
||||
# ^^ run manually: bash Orchestrators/media_management.sh --dry-run
|
||||
#
|
||||
# ━━━ Rsync — Appdata Profiles ━━━
|
||||
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Arrs_Stack
|
||||
#/mnt/user/appdata/unraid_scripts/Rsync/rsync.sh /mnt/user/appdata-Failover/Critical-Data
|
||||
@@ -325,16 +352,13 @@
|
||||
# emby-failover — every 30-60min via frequent cron above
|
||||
#
|
||||
# ━━━ Weekly — Sunday morning block ━━━
|
||||
# 30 2 * * 0 weekly_sync_maintenance.sh (clean Emby + auth stack — cache resets weekly)
|
||||
# 0 3 * * 0 docker_weekly_restart.sh
|
||||
# 0 5 * * 0 clear_logs.sh
|
||||
# 0 6 * * 0 zfs_memory_snapshot.sh
|
||||
# 0 7 * * 0 smart_health.sh
|
||||
# 0 8 * * 0 weekly_health_digest.sh (weekly profile sends today)
|
||||
# 0 9 * * 0 cert_monitor.sh
|
||||
# 0 10 * * 0 backup_verify.sh
|
||||
# 0 11 * * 0 emby_session_report.sh
|
||||
# 0 11 * * 0 bandwidth_monitor.sh --report
|
||||
# 30 2 * * 0 weekly_sync_maintenance.sh (clean sync + updates + docker weekly restart)
|
||||
# 0 7 * * 0 sunday_morning_coffee_report.sh (weekly health digest — ready when you wake up)
|
||||
#
|
||||
# Individual monitor scripts still available for manual runs:
|
||||
# clear_logs.sh, zfs_memory_snapshot.sh, smart_health.sh
|
||||
# cert_monitor.sh, backup_verify.sh, emby_session_report.sh
|
||||
# weekly_health_digest.sh, bandwidth_monitor.sh --report
|
||||
#
|
||||
# ━━━ Bandwidth monitor ━━━
|
||||
# bandwidth_monitor.sh --log-transfer called automatically by rsync.sh
|
||||
|
||||
Reference in New Issue
Block a user