massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
+890
-285
File diff suppressed because it is too large
Load Diff
+173
-88
@@ -1,79 +1,152 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Backup Verify ----------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= Backup Verify ==============================================
|
||||
# ==============================================================================================
|
||||
# Verifies the rsync mirror is healthy by comparing random file samples between
|
||||
# local and remote servers using MD5 checksums.
|
||||
#
|
||||
# Randomly samples BACKUP_VERIFY_SAMPLE files per share, computes checksums locally,
|
||||
# then computes the same checksums on the remote via SSH and compares results.
|
||||
# ── WHAT IT DOES ──────────────────────────────────────────────────────────────────────────────
|
||||
# Randomly samples BACKUP_VERIFY_SAMPLE files per share above BACKUP_VERIFY_MIN_SIZE,
|
||||
# computes MD5 checksums locally, then computes the same checksums on the remote via SSH
|
||||
# and compares results. Catches silent corruption or incomplete syncs that rsync itself
|
||||
# would not detect.
|
||||
#
|
||||
# Results per file:
|
||||
# MATCH — checksums identical, file is correctly mirrored
|
||||
# MISMATCH — file exists on both but checksums differ — sync may have failed
|
||||
# MISSING — file exists locally but not on remote — not yet synced or deleted
|
||||
# ── RESULTS PER FILE ──────────────────────────────────────────────────────────────────────────
|
||||
# MATCH — checksums identical, file is correctly mirrored ✅
|
||||
# MISMATCH — file exists on both but checksums differ — sync may have partially failed
|
||||
# MISSING — file exists locally but not on remote — not yet synced or deleted on remote
|
||||
#
|
||||
# Silent when all files match. Notifies on any mismatch or missing file.
|
||||
# Uses existing SSH keys — no additional configuration needed beyond share list.
|
||||
# ── SHARE SELECTION ───────────────────────────────────────────────────────────────────────────
|
||||
# Uses HOST*_BACKUP_VERIFY_SHARES if defined, falls back to HOST*_DAILY_SYNC_SHARES.
|
||||
# Both aliased by detect_hosts() — no manual HOST1/HOST2 selection needed.
|
||||
#
|
||||
# All configuration in Master.conf under Backup Verify section.
|
||||
# Supports --dry-run to show what would be checked without running checksums.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents concurrent runs producing duplicate/conflicting results
|
||||
# check_connectivity() — verifies remote reachable before attempting SSH calls
|
||||
# check_remote_array() — verifies remote array mounted before checksums
|
||||
# remote array down = all files "missing" = false alarm ✅
|
||||
# version parity — verifies both servers on compatible unRAID before trusting results
|
||||
# SSH_TIMEOUT — all SSH calls protected against hangs
|
||||
# validate_unraid_cmd — notify script validated before use
|
||||
# Silent by default — only issues produce output, all-match runs are silent
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_BACKUP_VERIFY_SHARES — override share list (empty = use DAILY_SYNC_SHARES)
|
||||
# HOST*_DAILY_SYNC_SHARES — fallback share list
|
||||
# All aliased by detect_hosts()
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# BACKUP_VERIFY_SAMPLE — random files to check per share (default 10)
|
||||
# BACKUP_VERIFY_MIN_SIZE — minimum file size to include in sample (default 1M)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# backup_verify.sh — normal run
|
||||
# backup_verify.sh --dry-run — show sample selection only, no checksums
|
||||
# backup_verify.sh --log — verbose output
|
||||
# backup_verify.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
SSH_TIMEOUT=15
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases BACKUP_VERIFY_SHARES + DAILY_SYNC_SHARES
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
# Use BACKUP_VERIFY_SHARES if defined, fall back to DAILY_SYNC_SHARES
|
||||
# Share selection — configured list or fallback to daily sync shares
|
||||
if [[ ${#BACKUP_VERIFY_SHARES[@]} -gt 0 ]]; then
|
||||
VERIFY_SHARES=("${BACKUP_VERIFY_SHARES[@]}")
|
||||
info "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)"
|
||||
log "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)"
|
||||
else
|
||||
VERIFY_SHARES=("${DAILY_SYNC_SHARES[@]}")
|
||||
info "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)"
|
||||
log "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)"
|
||||
fi
|
||||
|
||||
if [[ ${#VERIFY_SHARES[@]} -eq 0 ]]; then
|
||||
warn "No shares configured — nothing to verify"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_VERIFY Shares: ${#VERIFY_SHARES[@]}"
|
||||
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share"
|
||||
echo "$ICON_VERIFY Min size: $BACKUP_VERIFY_MIN_SIZE"
|
||||
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
warn "No shares configured for $MY_ID — nothing to verify"
|
||||
warn "Check HOST*_BACKUP_VERIFY_SHARES or HOST*_DAILY_SYNC_SHARES in master_host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing sample selection only, no checksums computed"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_VERIFY Backup Verification ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME — $REMOTE_SERVER)"
|
||||
echo "$ICON_VERIFY Shares: ${#VERIFY_SHARES[@]}"
|
||||
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share"
|
||||
echo "$ICON_VERIFY Min size: $BACKUP_VERIFY_MIN_SIZE"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
echo " Shares to verify:"
|
||||
for share in "${VERIFY_SHARES[@]}"; do
|
||||
echo " $share"
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Pre-flight ━━━"
|
||||
|
||||
# Connectivity — no point making 100+ SSH calls if remote is unreachable
|
||||
check_connectivity
|
||||
|
||||
# Version parity — mismatched unRAID could cause md5sum path differences
|
||||
check_unraid_version_parity || {
|
||||
warn "Version parity check failed — proceeding with caution"
|
||||
warn "Checksum results may be unreliable if md5sum path changed between versions"
|
||||
}
|
||||
|
||||
# Remote array — if array is down all files appear "missing" = false alarm
|
||||
if ! check_remote_array; then
|
||||
error "Remote array not mounted on $REMOTE_SERVER_NAME"
|
||||
error "All files would appear as MISSING — aborting to prevent false alarm"
|
||||
notify "Backup verify aborted on $(hostname) — remote array not mounted on $REMOTE_SERVER_NAME" \
|
||||
"Backup Verify" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Pre-flight passed ✅"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Backup Verification ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_VERIFY Backup Verification — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
||||
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share (min size: $BACKUP_VERIFY_MIN_SIZE)"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME) → $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share (min: $BACKUP_VERIFY_MIN_SIZE)"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
@@ -93,69 +166,74 @@ for share in "${VERIFY_SHARES[@]}"; do
|
||||
continue
|
||||
fi
|
||||
|
||||
# Find files above minimum size and randomly sample
|
||||
SAMPLE_FILES=$(find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \
|
||||
shuf | head -n "$BACKUP_VERIFY_SAMPLE")
|
||||
# Sample random files above minimum size
|
||||
mapfile -t SAMPLE_FILES < <(
|
||||
find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \
|
||||
shuf | head -n "$BACKUP_VERIFY_SAMPLE"
|
||||
)
|
||||
|
||||
SAMPLE_COUNT=$(echo "$SAMPLE_FILES" | grep -c "." 2>/dev/null || echo 0)
|
||||
|
||||
if [[ "$SAMPLE_COUNT" -eq 0 ]]; then
|
||||
info "No files found above $BACKUP_VERIFY_MIN_SIZE — skipping"
|
||||
if [[ ${#SAMPLE_FILES[@]} -eq 0 ]]; then
|
||||
log "$SHARE_NAME — no files found above $BACKUP_VERIFY_MIN_SIZE"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
info "Sampled $SAMPLE_COUNT files"
|
||||
log "$SHARE_NAME — sampled ${#SAMPLE_FILES[@]} files"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$SAMPLE_FILES" | while IFS= read -r f; do
|
||||
for f in "${SAMPLE_FILES[@]}"; do
|
||||
warn "DRY RUN — would check: $(basename "$f")"
|
||||
done
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
SHARE_MATCH=0
|
||||
SHARE_MISMATCH=0
|
||||
SHARE_MISSING=0
|
||||
SHARE_MATCH=0
|
||||
|
||||
while IFS= read -r local_file; do
|
||||
for local_file in "${SAMPLE_FILES[@]}"; do
|
||||
[[ -z "$local_file" ]] && continue
|
||||
|
||||
# Compute local checksum
|
||||
# Local checksum
|
||||
local_md5=$(md5sum "$local_file" 2>/dev/null | awk '{print $1}')
|
||||
|
||||
if [[ -z "$local_md5" ]]; then
|
||||
warn "Could not checksum: $local_file — skipping"
|
||||
warn "Could not checksum locally: $(basename "$local_file") — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Compute remote checksum via SSH
|
||||
remote_md5=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||||
# Remote checksum via SSH — timeout protected
|
||||
remote_md5=$(timeout "$SSH_TIMEOUT" ssh -i "$SSH_KEY" \
|
||||
-o ConnectTimeout="$SSH_TIMEOUT" \
|
||||
-o StrictHostKeyChecking=no \
|
||||
root@"$REMOTE_SERVER" \
|
||||
"md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null)
|
||||
|
||||
((TOTAL_CHECKED++))
|
||||
(( TOTAL_CHECKED++ ))
|
||||
|
||||
if [[ -z "$remote_md5" ]]; then
|
||||
warn "$ICON_ERROR MISSING: $(basename "$local_file")"
|
||||
((SHARE_MISSING++))
|
||||
((TOTAL_MISSING++))
|
||||
(( SHARE_MISSING++ ))
|
||||
(( TOTAL_MISSING++ ))
|
||||
elif [[ "$local_md5" == "$remote_md5" ]]; then
|
||||
log "MATCH: $(basename "$local_file")"
|
||||
((SHARE_MATCH++))
|
||||
((TOTAL_MATCH++))
|
||||
(( SHARE_MATCH++ ))
|
||||
(( TOTAL_MATCH++ ))
|
||||
else
|
||||
error "$ICON_ERROR MISMATCH: $(basename "$local_file")"
|
||||
((SHARE_MISMATCH++))
|
||||
((TOTAL_MISMATCH++))
|
||||
error "MISMATCH: $(basename "$local_file")"
|
||||
error " local: $local_md5"
|
||||
error " remote: $remote_md5"
|
||||
(( SHARE_MISMATCH++ ))
|
||||
(( TOTAL_MISMATCH++ ))
|
||||
fi
|
||||
done
|
||||
|
||||
done <<< "$SAMPLE_FILES"
|
||||
|
||||
echo " $ICON_SUCCESS Match: $SHARE_MATCH $ICON_WARN Missing: $SHARE_MISSING $ICON_ERROR Mismatch: $SHARE_MISMATCH"
|
||||
|
||||
# Per-share result — only visible if issues found
|
||||
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then
|
||||
warn "$SHARE_NAME — match: $SHARE_MATCH missing: $SHARE_MISSING mismatch: $SHARE_MISMATCH"
|
||||
SHARES_WITH_ISSUES+=("$SHARE_NAME")
|
||||
else
|
||||
log "$SHARE_NAME — all $SHARE_MATCH files match ✅"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
@@ -163,25 +241,32 @@ done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━━━ $ICON_SUMMARY BACKUP VERIFY SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME"
|
||||
echo "$ICON_HOST My ID: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_HOST Remote: $REMOTE_ID ($REMOTE_SERVER_NAME)"
|
||||
echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files"
|
||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||
echo "$ICON_WARN Missing: $TOTAL_MISSING"
|
||||
echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no checksums computed"
|
||||
elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention"
|
||||
notify "Backup verify failed on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" "Backup Verify" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL FILES MATCH"
|
||||
notify "Backup verify passed on $(hostname) → $REMOTE_SERVER_NAME — $TOTAL_CHECKED files checked across ${#VERIFY_SHARES[@]} shares" "Backup Verify" "normal"
|
||||
if [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||
warn "Missing: $TOTAL_MISSING"
|
||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no checksums computed"
|
||||
elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention: ${SHARES_WITH_ISSUES[*]}"
|
||||
notify "Backup verify FAILED on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" \
|
||||
"Backup Verify" "warning"
|
||||
else
|
||||
log "$ICON_DONE Status: all $TOTAL_CHECKED files match across ${#VERIFY_SHARES[@]} shares ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_MISMATCH" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+149
-49
@@ -1,38 +1,63 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Bandwidth Monitor ------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= Bandwidth Monitor ==========================================
|
||||
# ==============================================================================================
|
||||
# Logs rsync transfer history and generates weekly summary reports.
|
||||
# Designed for minimal flash drive impact — one bounded write per rsync run.
|
||||
#
|
||||
# Two modes:
|
||||
# --log-transfer "profile" duration status — called by rsync.sh after each sync
|
||||
# appends one line, trims old entries
|
||||
# --report (or no args) — generates summary from log
|
||||
# ── TWO MODES ─────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# Log format — one line per transfer, version-proof, never needs rsync output parsing:
|
||||
# YYYY-MM-DD|HH:MM|profile|duration_seconds|status
|
||||
# --log-transfer "profile" duration_seconds status
|
||||
# Called automatically by rsync.sh after each sync completes.
|
||||
# Appends one line to the log and trims entries older than BANDWIDTH_LOG_RETENTION.
|
||||
# Flags syncs exceeding BANDWIDTH_WARN_GB in the log for weekly report highlighting.
|
||||
#
|
||||
# Log file stays bounded to BANDWIDTH_LOG_RETENTION days — old entries trimmed on every write.
|
||||
# --report (or no args)
|
||||
# Generates a summary from the accumulated log.
|
||||
# Shows per-profile breakdown, last 7 days, and overall totals.
|
||||
# This is a monitor script — SILENT_MODE=false — output is the point.
|
||||
#
|
||||
# ── LOG FORMAT ────────────────────────────────────────────────────────────────────────────────
|
||||
# One line per transfer — version-proof, never needs rsync output parsing:
|
||||
# YYYY-MM-DD|HH:MM|profile|duration_seconds|status|bytes_transferred
|
||||
#
|
||||
# Log file stays bounded to BANDWIDTH_LOG_RETENTION days — trimmed on every write.
|
||||
# Minimal flash drive impact: one append + one trim per rsync run.
|
||||
#
|
||||
# All configuration in Master.conf under Bandwidth Monitor section.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock "wait" — prevents log corruption from concurrent rsync completions
|
||||
# validate_unraid_cmd — notify script validated before use
|
||||
# Atomic log write — temp file + mv prevents partial writes on trim
|
||||
# Log existence check — creates log directory if needed, exits cleanly if unwritable
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# BANDWIDTH_LOG — log file path
|
||||
# BANDWIDTH_LOG_RETENTION — days before old entries are purged (default 90)
|
||||
# BANDWIDTH_WARN_GB — flag syncs larger than this in report (default 50)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# bandwidth_monitor.sh — generate report
|
||||
# bandwidth_monitor.sh --report — generate report (explicit)
|
||||
# bandwidth_monitor.sh --log-transfer profile secs ok — log a transfer (called by rsync.sh)
|
||||
# bandwidth_monitor.sh --status — show config and exit
|
||||
# bandwidth_monitor.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Monitor script — output is the point
|
||||
SILENT_MODE=false
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Parse mode from PARSED_ARGS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── Parse mode from PARSED_ARGS ───────────────────────────────────────────────────────────────
|
||||
LOG_TRANSFER_MODE=false
|
||||
TRANSFER_PROFILE=""
|
||||
TRANSFER_DURATION=0
|
||||
TRANSFER_STATUS="success"
|
||||
TRANSFER_BYTES=0
|
||||
|
||||
for arg in "${PARSED_ARGS[@]}"; do
|
||||
case "$arg" in
|
||||
@@ -42,79 +67,131 @@ for arg in "${PARSED_ARGS[@]}"; do
|
||||
if [[ "$LOG_TRANSFER_MODE" == true ]]; then
|
||||
if [[ -z "$TRANSFER_PROFILE" ]]; then
|
||||
TRANSFER_PROFILE="$arg"
|
||||
elif [[ "$TRANSFER_DURATION" -eq 0 ]]; then
|
||||
elif [[ "$TRANSFER_DURATION" -eq 0 && "$arg" =~ ^[0-9]+$ ]]; then
|
||||
TRANSFER_DURATION="$arg"
|
||||
else
|
||||
elif [[ "$arg" == "success" || "$arg" == "failed" ]]; then
|
||||
TRANSFER_STATUS="$arg"
|
||||
elif [[ "$arg" =~ ^[0-9]+$ ]]; then
|
||||
TRANSFER_BYTES="$arg"
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Ensure log directory and file exist
|
||||
# ── Ensure log file exists and is writable ────────────────────────────────────────────────────
|
||||
mkdir -p "$(dirname "$BANDWIDTH_LOG")"
|
||||
touch "$BANDWIDTH_LOG" 2>/dev/null || {
|
||||
error "Cannot write to bandwidth log: $BANDWIDTH_LOG"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# LOG TRANSFER MODE
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# detect_hosts() sets MY_ID for report header
|
||||
detect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_BANDWIDTH Log file: $BANDWIDTH_LOG"
|
||||
echo "$ICON_BANDWIDTH Retention: ${BANDWIDTH_LOG_RETENTION} days"
|
||||
echo "$ICON_BANDWIDTH Warn GB: ${BANDWIDTH_WARN_GB}GB"
|
||||
local entry_count=0
|
||||
[[ -f "$BANDWIDTH_LOG" ]] && entry_count=$(wc -l < "$BANDWIDTH_LOG")
|
||||
echo "$ICON_BANDWIDTH Log entries: $entry_count"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Log Transfer Mode ━━━
|
||||
# ==============================================================================================
|
||||
# Called by rsync.sh after each sync — appends one line and trims old entries.
|
||||
# Usage: bandwidth_monitor.sh --log-transfer "profile" duration_seconds status
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Uses "wait" lock — if two rsync jobs finish simultaneously, wait and write in order.
|
||||
# Usage: bandwidth_monitor.sh --log-transfer "profile" duration_seconds status [bytes]
|
||||
|
||||
if [[ "$LOG_TRANSFER_MODE" == true ]]; then
|
||||
[[ -z "$TRANSFER_PROFILE" ]] && error "No profile specified for --log-transfer" && exit 1
|
||||
[[ -z "$TRANSFER_PROFILE" ]] && { error "No profile specified for --log-transfer"; exit 1; }
|
||||
|
||||
acquire_lock "wait"
|
||||
|
||||
TODAY=$(date '+%Y-%m-%d')
|
||||
NOW=$(date '+%H:%M')
|
||||
DURATION_FMT=$(format_duration "$TRANSFER_DURATION")
|
||||
|
||||
# Append entry
|
||||
echo "${TODAY}|${NOW}|${TRANSFER_PROFILE}|${TRANSFER_DURATION}|${TRANSFER_STATUS}" >> "$BANDWIDTH_LOG"
|
||||
log "$ICON_BANDWIDTH Logged: $TRANSFER_PROFILE — ${DURATION_FMT} — $TRANSFER_STATUS"
|
||||
# Check if transfer exceeds warn threshold
|
||||
WARN_FLAG=""
|
||||
if [[ -n "$TRANSFER_BYTES" && "$TRANSFER_BYTES" -gt 0 ]]; then
|
||||
WARN_BYTES=$(awk "BEGIN {printf \"%d\", $BANDWIDTH_WARN_GB * 1073741824}")
|
||||
[[ "$TRANSFER_BYTES" -gt "$WARN_BYTES" ]] && WARN_FLAG="LARGE"
|
||||
fi
|
||||
|
||||
# Trim entries older than retention period — keeps file bounded
|
||||
# Append entry — format: date|time|profile|duration|status|bytes|warn_flag
|
||||
echo "${TODAY}|${NOW}|${TRANSFER_PROFILE}|${TRANSFER_DURATION}|${TRANSFER_STATUS}|${TRANSFER_BYTES}|${WARN_FLAG}" \
|
||||
>> "$BANDWIDTH_LOG"
|
||||
log "$ICON_BANDWIDTH Logged: $TRANSFER_PROFILE — ${DURATION_FMT} — $TRANSFER_STATUS${WARN_FLAG:+ [$WARN_FLAG]}"
|
||||
|
||||
# Trim entries older than retention — atomic write via temp file
|
||||
CUTOFF=$(date -d "${BANDWIDTH_LOG_RETENTION} days ago" '+%Y-%m-%d')
|
||||
TEMP_FILE="${BANDWIDTH_LOG}.tmp"
|
||||
awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' "$BANDWIDTH_LOG" > "$TEMP_FILE" && \
|
||||
mv "$TEMP_FILE" "$BANDWIDTH_LOG"
|
||||
log "$ICON_BANDWIDTH Log trimmed — retaining entries from $CUTOFF onwards"
|
||||
log "$ICON_BANDWIDTH Log trimmed — retaining entries from $CUTOFF"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REPORT MODE — generate summary from log
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Report Mode ━━━
|
||||
# ==============================================================================================
|
||||
acquire_lock "wait"
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_BANDWIDTH Bandwidth Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
|
||||
if [[ ! -s "$BANDWIDTH_LOG" ]]; then
|
||||
warn "No bandwidth data yet — log is empty"
|
||||
warn "Data accumulates as rsync jobs complete"
|
||||
warn "Data accumulates as rsync jobs complete via rsync.sh"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
# Date range
|
||||
# ── Log overview ──────────────────────────────────────────────────────────────────────────────
|
||||
OLDEST=$(awk -F'|' 'NR==1{print $1}' "$BANDWIDTH_LOG")
|
||||
NEWEST=$(awk -F'|' 'END{print $1}' "$BANDWIDTH_LOG")
|
||||
ENTRY_COUNT=$(wc -l < "$BANDWIDTH_LOG")
|
||||
SUCCESS_COUNT=$(awk -F'|' '$5=="success"' "$BANDWIDTH_LOG" | wc -l)
|
||||
FAILED_COUNT=$(awk -F'|' '$5=="failed"' "$BANDWIDTH_LOG" | wc -l)
|
||||
LARGE_COUNT=$(awk -F'|' '$7=="LARGE"' "$BANDWIDTH_LOG" | wc -l)
|
||||
|
||||
info "Log covers: $OLDEST → $NEWEST ($ENTRY_COUNT runs)"
|
||||
echo ""
|
||||
|
||||
# ── Per-profile breakdown ────────────────────────────────────────────────────────────────────
|
||||
# ── Per-profile breakdown ─────────────────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_BANDWIDTH Per-Profile Summary ━━━"
|
||||
awk -F'|' '{
|
||||
runs[$3]++
|
||||
duration[$3] += $4
|
||||
if ($5 == "failed") fails[$3]++
|
||||
if ($7 == "LARGE") large[$3]++
|
||||
}
|
||||
END {
|
||||
for (profile in runs) {
|
||||
@@ -122,21 +199,22 @@ END {
|
||||
mins = int(avg / 60)
|
||||
secs = int(avg % 60)
|
||||
fail_count = (profile in fails) ? fails[profile] : 0
|
||||
printf " %-20s %3d runs avg %dm%ds failed: %d\n", \
|
||||
profile, runs[profile], mins, secs, fail_count
|
||||
large_count = (profile in large) ? large[profile] : 0
|
||||
large_str = (large_count > 0) ? " ⚠️ " large_count " large" : ""
|
||||
printf " %-22s %3d runs avg %dm%ds failed: %d%s\n", \
|
||||
profile, runs[profile], mins, secs, fail_count, large_str
|
||||
}
|
||||
}' "$BANDWIDTH_LOG" | sort
|
||||
|
||||
echo ""
|
||||
|
||||
# ── Last 7 days ──────────────────────────────────────────────────────────────────────────────
|
||||
# ── Last 7 days ───────────────────────────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_BANDWIDTH Last 7 Days ━━━"
|
||||
for i in 6 5 4 3 2 1 0; do
|
||||
day=$(date -d "$i days ago" '+%Y-%m-%d')
|
||||
day_name=$(date -d "$i days ago" '+%a')
|
||||
day_runs=$(awk -F'|' -v d="$day" '$1==d' "$BANDWIDTH_LOG" | wc -l)
|
||||
day_success=$(awk -F'|' -v d="$day" '$1==d && $5=="success"' "$BANDWIDTH_LOG" | wc -l)
|
||||
day_failed=$(awk -F'|' -v d="$day" '$1==d && $5=="failed"' "$BANDWIDTH_LOG" | wc -l)
|
||||
day_large=$(awk -F'|' -v d="$day" '$1==d && $7=="LARGE"' "$BANDWIDTH_LOG" | wc -l)
|
||||
day_duration=$(awk -F'|' -v d="$day" '$1==d{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG")
|
||||
day_duration_fmt=$(format_duration "$day_duration")
|
||||
|
||||
@@ -144,25 +222,47 @@ for i in 6 5 4 3 2 1 0; do
|
||||
echo " $ICON_TIME $day ($day_name) — no syncs"
|
||||
elif [[ "$day_failed" -gt 0 ]]; then
|
||||
echo " $ICON_WARN $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / $ICON_ERROR $day_failed failed"
|
||||
elif [[ "$day_large" -gt 0 ]]; then
|
||||
echo " $ICON_WARN $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / ⚠️ $day_large large"
|
||||
else
|
||||
echo " $ICON_DONE $day ($day_name) — $day_runs runs / ${day_duration_fmt} total"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
||||
# ── Totals ───────────────────────────────────────────────────────────────────────────────────
|
||||
# ── Large transfers ───────────────────────────────────────────────────────────────────────────
|
||||
if [[ "$LARGE_COUNT" -gt 0 ]]; then
|
||||
echo "━━━ $ICON_WARN Large Transfers (>${BANDWIDTH_WARN_GB}GB) ━━━"
|
||||
awk -F'|' '$7=="LARGE" {
|
||||
bytes=$6+0
|
||||
gb=bytes/1073741824
|
||||
printf " %s %s %-20s %.1fGB\n", $1, $2, $3, gb
|
||||
}' "$BANDWIDTH_LOG" | tail -10
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# ── Totals ────────────────────────────────────────────────────────────────────────────────────
|
||||
TOTAL_DURATION=$(awk -F'|' '{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG")
|
||||
TOTAL_DURATION_FMT=$(format_duration "$TOTAL_DURATION")
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
echo "━━━━━ $ICON_SUMMARY BANDWIDTH SUMMARY ━━━━━"
|
||||
echo "$ICON_BANDWIDTH Total runs: $ENTRY_COUNT ($SUCCESS_COUNT success / $FAILED_COUNT failed)"
|
||||
echo "$ICON_TIME Total time: $TOTAL_DURATION_FMT"
|
||||
echo "$ICON_TIME Log period: $OLDEST → $NEWEST"
|
||||
echo "$ICON_GEAR Retention: ${BANDWIDTH_LOG_RETENTION} days"
|
||||
echo "$ICON_TIME Generated in: $(format_duration $((END - START)))"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_BANDWIDTH Runs: $ENTRY_COUNT ($SUCCESS_COUNT success / $FAILED_COUNT failed)"
|
||||
[[ "$LARGE_COUNT" -gt 0 ]] && \
|
||||
warn "Large: $LARGE_COUNT transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB"
|
||||
echo "$ICON_TIME Total: $TOTAL_DURATION_FMT"
|
||||
echo "$ICON_TIME Period: $OLDEST → $NEWEST"
|
||||
echo "$ICON_GEAR Retention: ${BANDWIDTH_LOG_RETENTION} days"
|
||||
echo "$ICON_TIME Generated: $(format_duration $(( END - START )))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
notify "Bandwidth report on $(hostname) — $ENTRY_COUNT rsync runs ($SUCCESS_COUNT success / $FAILED_COUNT failed) over ${BANDWIDTH_LOG_RETENTION} day window" "Bandwidth Monitor" "normal"
|
||||
# Only notify if there are failures or large transfers worth flagging
|
||||
if [[ "$FAILED_COUNT" -gt 0 ]]; then
|
||||
notify "Bandwidth report on $(hostname) — $FAILED_COUNT failed sync(s) in ${BANDWIDTH_LOG_RETENTION} day window" \
|
||||
"Bandwidth Monitor" "warning"
|
||||
elif [[ "$LARGE_COUNT" -gt 0 ]]; then
|
||||
notify "Bandwidth report on $(hostname) — $LARGE_COUNT large transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB" \
|
||||
"Bandwidth Monitor" "normal"
|
||||
fi
|
||||
+129
-61
@@ -1,73 +1,130 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Certificate Monitor ----------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= Certificate Monitor ========================================
|
||||
# ==============================================================================================
|
||||
# Monitors SSL certificate expiry for all configured domains by connecting directly
|
||||
# via openssl — no dependency on NPM or any other service. Reads the actual certificate
|
||||
# the server is presenting to the outside world.
|
||||
#
|
||||
# This approach catches real-world cert issues that API-based checks miss:
|
||||
# - Cert renewed but server not reloaded
|
||||
# - Wrong cert being served
|
||||
# - Cert chain issues
|
||||
# ── WHY DIRECT OPENSSL ────────────────────────────────────────────────────────────────────────
|
||||
# Catches real-world cert issues that API-based checks miss:
|
||||
# - Cert renewed in NPM but server not reloaded (old cert still serving)
|
||||
# - Wrong cert being served to external clients
|
||||
# - Cert chain issues not visible from the internal network
|
||||
# - NPM reporting healthy while the world sees an expired cert
|
||||
#
|
||||
# Each domain and subdomain is a separate entry — they have independent certs.
|
||||
# Silent when all certs are healthy. Notifies when any approach warning threshold.
|
||||
# Notifications batched per severity — one message for warnings, one for criticals.
|
||||
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
|
||||
# Each domain is checked independently — they have independent certs.
|
||||
# Results per domain:
|
||||
# HEALTHY — > CERT_WARN_DAYS remaining — silent ✅
|
||||
# WARNING — <= CERT_WARN_DAYS remaining — notifies
|
||||
# CRITICAL — <= CERT_CRIT_DAYS remaining — notifies with urgency
|
||||
# FAILED — could not connect or parse cert — notifies
|
||||
#
|
||||
# All configuration in Master.conf under Certificate Monitor section.
|
||||
# Supports --dry-run to check certs and show results without sending notifications.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Notifications batched per severity — one message per severity level, not per domain.
|
||||
# This is a monitor script — SILENT_MODE=false — output is the point.
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS → CERT_MONITOR_DOMAINS.
|
||||
# Each server monitors its own domains — HOST1 monitors Gmer4Lfe.com etc.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents concurrent runs
|
||||
# detect_hosts() — correct domain list per host via MY_ID aliases
|
||||
# Empty array guard — warns and exits cleanly if no domains configured
|
||||
# CERT_TIMEOUT — openssl connects are time-limited per domain
|
||||
# validate_unraid_cmd — openssl and notify validated before use
|
||||
# Silent healthy certs — only problems produce visible output
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_CERT_MONITOR_DOMAINS — domains checked by this host
|
||||
# Aliased by detect_hosts() — script uses CERT_MONITOR_DOMAINS
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# CERT_WARN_DAYS — warn when cert expires within this many days (default 30)
|
||||
# CERT_CRIT_DAYS — critical alert within this many days (default 7)
|
||||
# CERT_TIMEOUT — seconds per domain before giving up (default 10)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# cert_monitor.sh — normal run
|
||||
# cert_monitor.sh --dry-run — check certs and show results, no notifications
|
||||
# cert_monitor.sh --log — verbose output
|
||||
# cert_monitor.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Monitor script — output is the point
|
||||
SILENT_MODE=false
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if ! command -v openssl >/dev/null 2>&1; then
|
||||
error "openssl not found — required for certificate checks"
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "openssl available"
|
||||
# Validate openssl — required for all cert checks
|
||||
validate_unraid_cmd \
|
||||
"$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \
|
||||
"version" "OpenSSL" \
|
||||
"openssl" || { error "openssl not found — required for certificate checks"; exit 1; }
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_CERT_MONITOR_DOMAINS
|
||||
detect_hosts
|
||||
|
||||
# Empty array guard
|
||||
if [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then
|
||||
warn "CERT_MONITOR_DOMAINS is empty in Master.conf — add your domains to enable monitoring"
|
||||
warn "CERT_MONITOR_DOMAINS is empty for $MY_ID"
|
||||
warn "Check HOST*_CERT_MONITOR_DOMAINS in master_host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
info "$ICON_CERT Domains to check: ${#CERT_MONITOR_DOMAINS[@]}"
|
||||
log "Domains to check: ${#CERT_MONITOR_DOMAINS[@]}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}"
|
||||
echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining"
|
||||
echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining"
|
||||
echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}"
|
||||
echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining"
|
||||
echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining"
|
||||
echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CERT CHECK FUNCTION
|
||||
# ==============================================================================================
|
||||
# ── CERT CHECK FUNCTION ───────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Connects to domain:443 via openssl, extracts expiry date, calculates days remaining.
|
||||
# Returns 0=healthy 1=warning 2=critical 3=failed
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Returns:
|
||||
# 0 = healthy (> CERT_WARN_DAYS)
|
||||
# 1 = warning (<= CERT_WARN_DAYS)
|
||||
# 2 = critical (<= CERT_CRIT_DAYS)
|
||||
# 3 = failed (could not connect or parse)
|
||||
|
||||
check_cert() {
|
||||
local domain="$1"
|
||||
local port="${2:-443}"
|
||||
@@ -79,7 +136,7 @@ check_cert() {
|
||||
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||||
|
||||
if [[ -z "$expiry_str" ]]; then
|
||||
error "$ICON_CERT $domain — could not retrieve certificate"
|
||||
error "$ICON_CERT $domain — could not retrieve certificate (unreachable or no TLS)"
|
||||
return 3
|
||||
fi
|
||||
|
||||
@@ -103,18 +160,19 @@ check_cert() {
|
||||
warn "$ICON_CERT $domain — WARNING: ${days_remaining} days remaining (expires $expiry_display)"
|
||||
return 1
|
||||
else
|
||||
success "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)"
|
||||
log "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CERT Certificate Monitor ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Certificate Monitor ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CERT Certificate Monitor — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_WARN Warn threshold: ${CERT_WARN_DAYS} days"
|
||||
echo "$ICON_ERROR Crit threshold: ${CERT_CRIT_DAYS} days"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
log "Warn threshold: ${CERT_WARN_DAYS} days"
|
||||
log "Crit threshold: ${CERT_CRIT_DAYS} days"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
@@ -126,7 +184,6 @@ declare -A DOMAIN_STATUS
|
||||
|
||||
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
||||
[[ -z "$domain" ]] && continue
|
||||
echo "━━━ $ICON_CERT $domain ━━━"
|
||||
check_cert "$domain"
|
||||
result=$?
|
||||
case $result in
|
||||
@@ -135,46 +192,57 @@ for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
||||
2) CRITICAL+=("$domain"); DOMAIN_STATUS["$domain"]="CRIT" ;;
|
||||
3) FAILED+=("$domain"); DOMAIN_STATUS["$domain"]="FAIL" ;;
|
||||
esac
|
||||
echo ""
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ── Send notifications — batched per severity ─────────────────────────────────────────────────
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
[[ ${#CRITICAL[@]} -gt 0 ]] && \
|
||||
notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" "Certificate Monitor" "warning"
|
||||
notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" \
|
||||
"Certificate Monitor" "warning"
|
||||
[[ ${#WARNING[@]} -gt 0 ]] && \
|
||||
notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" "Certificate Monitor" "warning"
|
||||
notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" \
|
||||
"Certificate Monitor" "warning"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && \
|
||||
notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" "Certificate Monitor" "warning"
|
||||
notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" \
|
||||
"Certificate Monitor" "warning"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CERTIFICATE MONITOR SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]} $ICON_WARN Warning: ${#WARNING[@]} $ICON_ERROR Critical: ${#CRITICAL[@]} Failed: ${#FAILED[@]}"
|
||||
echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]}"
|
||||
[[ ${#WARNING[@]} -gt 0 ]] && warn "Warning: ${#WARNING[@]} — renewal recommended"
|
||||
[[ ${#CRITICAL[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#CRITICAL[@]} — ACTION REQUIRED"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#FAILED[@]} — unreachable"
|
||||
echo ""
|
||||
|
||||
# Per-domain results — only show problems, healthy ones stay in log()
|
||||
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
||||
[[ -z "$domain" ]] && continue
|
||||
case "${DOMAIN_STATUS[$domain]:-UNKN}" in
|
||||
OK) echo " $ICON_SUCCESS $domain" ;;
|
||||
WARN) echo " $ICON_WARN $domain" ;;
|
||||
CRIT) echo " $ICON_ERROR $domain" ;;
|
||||
FAIL) echo " $ICON_ERROR $domain (unreachable)" ;;
|
||||
OK) log " $ICON_SUCCESS $domain — healthy" ;;
|
||||
WARN) warn " $ICON_WARN $domain — warning" ;;
|
||||
CRIT) echo " $ICON_ERROR $domain — CRITICAL" ;;
|
||||
FAIL) echo " $ICON_ERROR $domain — unreachable" ;;
|
||||
esac
|
||||
done
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no notifications sent"
|
||||
warn "DRY RUN — no notifications sent"
|
||||
elif [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ACTION REQUIRED"
|
||||
elif [[ ${#WARNING[@]} -gt 0 ]]; then
|
||||
echo "$ICON_WARN Status: WARNINGS — renewal recommended"
|
||||
warn "Status: WARNINGS — renewal recommended"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL CERTS HEALTHY"
|
||||
log "$ICON_DONE Status: all ${#HEALTHY[@]} certs healthy ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
|
||||
@@ -1,35 +1,58 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------------------- Continuous Scripts Status --------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ========================= Continuous Scripts Status ==========================================
|
||||
# ==============================================================================================
|
||||
# Live status dashboard for all continuously running scripts in the ecosystem.
|
||||
# Run manually anytime — no schedule, no cron.
|
||||
#
|
||||
# Covers all scripts started by array_start.sh that run until array stops:
|
||||
# system_watchdog.sh — system health monitor
|
||||
# docker_watchdog.sh — container health monitor
|
||||
# failover.sh — mutual failover monitor
|
||||
# ── WHAT IT SHOWS ─────────────────────────────────────────────────────────────────────────────
|
||||
# For each continuous script (system_watchdog, docker_watchdog, failover):
|
||||
# Running state, PID, uptime, approximate cycle count
|
||||
# Active strikes and skip list
|
||||
# Recent restart history
|
||||
# Live health snapshot
|
||||
#
|
||||
# Shows for each:
|
||||
# Running state, PID, uptime, current cycle
|
||||
# Active strikes, skip list, recent actions
|
||||
# Live system/container health snapshot
|
||||
# Failover state, tier status, Tailscale connectivity
|
||||
# system_watchdog — rootfs, RAM, ZFS ARC, load, zombie count, CPU temp
|
||||
# docker_watchdog — running/stopped/unhealthy containers, required containers,
|
||||
# monitored memory containers, recent restart history
|
||||
# failover — current state, tier status, remote Tailscale visibility
|
||||
#
|
||||
# If a script is mid-cycle state files are read as-is — reflects last completed cycle.
|
||||
# Run: bash Monitors/continuous_scripts_status.sh
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# If a script is mid-cycle, state files are read as-is — reflects last completed cycle.
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays.
|
||||
# Required containers, tier delays, and Tailscale checks use MY_ID/REMOTE_ID correctly.
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# continuous_scripts_status.sh — show dashboard
|
||||
# continuous_scripts_status.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Dashboard script — output is the point
|
||||
SILENT_MODE=false
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# HELPERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
|
||||
detect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
get_lock_pid() {
|
||||
local script_name="$1"
|
||||
@@ -51,11 +74,10 @@ get_lock_name() {
|
||||
fi
|
||||
}
|
||||
|
||||
is_watchdog_running() {
|
||||
is_script_running() {
|
||||
local script_name="$1"
|
||||
local pid
|
||||
local pid locked_name
|
||||
pid=$(get_lock_pid "$script_name")
|
||||
local locked_name
|
||||
locked_name=$(get_lock_name "$script_name")
|
||||
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$locked_name" == "$script_name" ]]
|
||||
}
|
||||
@@ -73,6 +95,7 @@ get_lock_age() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Human readable uptime — days/hours/mins
|
||||
format_uptime() {
|
||||
local seconds=$1
|
||||
local days=$(( seconds / 86400 ))
|
||||
@@ -87,34 +110,29 @@ format_uptime() {
|
||||
fi
|
||||
}
|
||||
|
||||
get_strikes() {
|
||||
local state_file="$1"
|
||||
local key="$2"
|
||||
grep -E "^${key}:" "$state_file" 2>/dev/null | cut -d: -f2
|
||||
}
|
||||
divider() { printf '%.0s─' {1..57}; echo; }
|
||||
section() { echo ""; echo " $1"; divider; }
|
||||
|
||||
divider() { printf '%.0s─' {1..55}; echo; }
|
||||
header() { echo ""; echo " $1"; divider; }
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ HEADER ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Header ━━━
|
||||
# ==============================================================================================
|
||||
clear
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " 🛡️ WATCHDOG STATUS — $(date '+%A, %B %-d at %-I:%M%p')"
|
||||
echo " 🖥️ $(hostname)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME"
|
||||
echo " $ICON_HOST Remote: $REMOTE_ID — $REMOTE_SERVER_NAME"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ System Watchdog ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
header "⚙️ SYSTEM WATCHDOG"
|
||||
# ==============================================================================================
|
||||
section "⚙️ SYSTEM WATCHDOG"
|
||||
|
||||
SYS_PID=$(get_lock_pid "system_watchdog")
|
||||
SYS_RUNNING=false
|
||||
|
||||
if is_watchdog_running "system_watchdog"; then
|
||||
if is_script_running "system_watchdog"; then
|
||||
SYS_RUNNING=true
|
||||
SYS_AGE=$(get_lock_age "system_watchdog")
|
||||
SYS_UPTIME=$(format_uptime "$SYS_AGE")
|
||||
@@ -128,7 +146,7 @@ fi
|
||||
|
||||
echo ""
|
||||
|
||||
# System watchdog strikes
|
||||
# System strikes
|
||||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||||
ACTIVE_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||||
if [[ -n "$ACTIVE_STRIKES" ]]; then
|
||||
@@ -149,8 +167,8 @@ if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
|
||||
TOTAL_REBOOTS=$(grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0)
|
||||
TOTAL_REBOOTS="${TOTAL_REBOOTS//[^0-9]/}"
|
||||
TOTAL_REBOOTS="${TOTAL_REBOOTS:-0}"
|
||||
WEEK_EPOCH=$(date -d "7 days ago" +%s)
|
||||
WEEK_REBOOTS=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \
|
||||
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
|
||||
WEEK_REBOOTS=$(awk -v cutoff="$WEEK_CUTOFF" '$0 >= cutoff' \
|
||||
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
|
||||
echo " 🔄 Watchdog reboots: $WEEK_REBOOTS this week / $TOTAL_REBOOTS total"
|
||||
fi
|
||||
@@ -159,7 +177,7 @@ fi
|
||||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||||
echo ""
|
||||
echo " ⛔ Skip list ($SKIP_COUNT containers — manual intervention needed):"
|
||||
echo " ⛔ Skip list ($SKIP_COUNT — manual intervention needed):"
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
echo " → $container"
|
||||
@@ -168,17 +186,15 @@ else
|
||||
echo " ✅ Skip list: empty"
|
||||
fi
|
||||
|
||||
# Current system health snapshot
|
||||
# Live system health snapshot
|
||||
echo ""
|
||||
echo " 📊 Current system state:"
|
||||
|
||||
# rootfs
|
||||
ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
|
||||
[[ "${ROOTFS_PCT:-0}" -ge "${SYS_WATCHDOG_ROOTFS_PCT:-95}" ]] && \
|
||||
ROOTFS_ICON="⚠️ " || ROOTFS_ICON="✅"
|
||||
echo " ${ROOTFS_ICON} rootfs: ${ROOTFS_PCT}% (threshold: ${SYS_WATCHDOG_ROOTFS_PCT}%)"
|
||||
|
||||
# RAM
|
||||
MEM_AVAIL_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_FREE_GB=$(awk "BEGIN {printf \"%.1f\", $MEM_AVAIL_KB / 1048576}")
|
||||
MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
|
||||
@@ -186,7 +202,6 @@ MEM_TOTAL_GB=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
|
||||
MEM_ICON="⚠️ " || MEM_ICON="✅"
|
||||
echo " ${MEM_ICON} RAM: ${MEM_FREE_GB}GB free / ${MEM_TOTAL_GB}GB total (threshold: ${SYS_WATCHDOG_MEM_GB}GB free)"
|
||||
|
||||
# ARC
|
||||
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||||
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
@@ -197,7 +212,6 @@ if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||||
echo " ${ARC_ICON} ZFS ARC: ${ARC_GB}GB (${ARC_PCT}% of max, threshold: ${SYS_WATCHDOG_ARC_PINNED_PCT}%)"
|
||||
fi
|
||||
|
||||
# Load
|
||||
LOAD=$(awk '{print $1}' /proc/loadavg)
|
||||
CORES=$(nproc)
|
||||
LOAD_THRESH=$(( CORES * ${SYS_WATCHDOG_LOAD_MULTIPLIER:-3} ))
|
||||
@@ -205,17 +219,13 @@ LOAD_INT=$(printf "%.0f" "$LOAD")
|
||||
[[ "$LOAD_INT" -ge "$LOAD_THRESH" ]] && LOAD_ICON="⚠️ " || LOAD_ICON="✅"
|
||||
echo " ${LOAD_ICON} Load avg: $LOAD (threshold: ${LOAD_THRESH} = ${SYS_WATCHDOG_LOAD_MULTIPLIER}x ${CORES} cores)"
|
||||
|
||||
# Zombies
|
||||
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0)
|
||||
ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"
|
||||
ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
|
||||
ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}"
|
||||
ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}"
|
||||
[[ "$ZOMBIE_COUNT" -ge "${SYS_WATCHDOG_ZOMBIE_LIMIT:-50}" ]] && \
|
||||
ZOMBIE_ICON="⚠️ " || ZOMBIE_ICON="✅"
|
||||
echo " ${ZOMBIE_ICON} Zombies: $ZOMBIE_COUNT (threshold: ${SYS_WATCHDOG_ZOMBIE_LIMIT})"
|
||||
|
||||
# CPU temp
|
||||
if command -v sensors >/dev/null 2>&1; then
|
||||
CPU_TEMP=$(sensors 2>/dev/null | \
|
||||
grep -i "Package id 0\|Tctl\|CPU Temp" | \
|
||||
@@ -228,15 +238,15 @@ if command -v sensors >/dev/null 2>&1; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
header "🐳 DOCKER WATCHDOG"
|
||||
# ==============================================================================================
|
||||
section "🐳 DOCKER WATCHDOG"
|
||||
|
||||
DOCKER_PID=$(get_lock_pid "docker_watchdog")
|
||||
DOCKER_RUNNING=false
|
||||
|
||||
if is_watchdog_running "docker_watchdog"; then
|
||||
if is_script_running "docker_watchdog"; then
|
||||
DOCKER_RUNNING=true
|
||||
DOCKER_AGE=$(get_lock_age "docker_watchdog")
|
||||
DOCKER_UPTIME=$(format_uptime "$DOCKER_AGE")
|
||||
@@ -250,7 +260,7 @@ fi
|
||||
|
||||
echo ""
|
||||
|
||||
# Container watchdog strikes
|
||||
# Container strikes
|
||||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
ACTIVE_CONTAINER_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
|
||||
if [[ -n "$ACTIVE_CONTAINER_STRIKES" ]]; then
|
||||
@@ -264,16 +274,15 @@ if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# Container restart history this week
|
||||
# Container restart history
|
||||
if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
|
||||
WEEK_EPOCH=$(date -d "7 days ago" +%s 2>/dev/null || date -v-7d +%s 2>/dev/null)
|
||||
WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_EPOCH" \
|
||||
'NR>0 {if ($2 >= cutoff) count++} END {print count+0}' \
|
||||
"$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null)
|
||||
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
|
||||
WEEK_RESTARTS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||||
'$2 >= cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l)
|
||||
if [[ "${WEEK_RESTARTS:-0}" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " 🔄 Container restarts this week: $WEEK_RESTARTS"
|
||||
awk -F'|' -v cutoff="$WEEK_EPOCH" \
|
||||
awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||||
'$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
|
||||
sort | uniq -c | sort -rn | head -5 | \
|
||||
while read -r count name; do
|
||||
@@ -284,31 +293,34 @@ if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# Docker container overview
|
||||
# Container overview
|
||||
echo ""
|
||||
echo " 📦 Container overview:"
|
||||
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
RUNNING=$(docker ps -q 2>/dev/null | wc -l)
|
||||
TOTAL=$(docker ps -aq 2>/dev/null | wc -l)
|
||||
UNHEALTHY=$(docker ps --filter health=unhealthy -q 2>/dev/null | wc -l)
|
||||
RUNNING=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l)
|
||||
TOTAL=$(timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l)
|
||||
UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
|
||||
--filter health=unhealthy -q 2>/dev/null | wc -l)
|
||||
|
||||
# Filter intentionally stopped containers from the stopped list
|
||||
# Stopped containers — filter intentionally ignored ones
|
||||
STOPPED_FILTERED=()
|
||||
while IFS= read -r name; do
|
||||
[[ -z "$name" ]] && continue
|
||||
SKIP=false
|
||||
local SKIP=false
|
||||
for ignore in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do
|
||||
[[ "$name" == "$ignore" ]] && SKIP=true && break
|
||||
done
|
||||
[[ "$SKIP" == false ]] && STOPPED_FILTERED+=("$name")
|
||||
done < <(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null)
|
||||
STOPPED_COUNT="${#STOPPED_FILTERED[@]}"
|
||||
done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \
|
||||
--format "{{.Names}}" 2>/dev/null)
|
||||
|
||||
STOPPED_COUNT="${#STOPPED_FILTERED[@]}"
|
||||
echo " Running: $RUNNING / $TOTAL total"
|
||||
[[ "$UNHEALTHY" -gt 0 ]] && echo " ⚠️ Unhealthy: $UNHEALTHY"
|
||||
|
||||
if [[ "$STOPPED_COUNT" -gt 0 ]]; then
|
||||
echo " ⚠️ Stopped containers (unexpected):"
|
||||
echo " ⚠️ Stopped (unexpected):"
|
||||
for name in "${STOPPED_FILTERED[@]}"; do
|
||||
echo " → $name"
|
||||
done
|
||||
@@ -316,40 +328,35 @@ if command -v docker >/dev/null 2>&1; then
|
||||
echo " ✅ All containers running"
|
||||
fi
|
||||
|
||||
# Check required containers
|
||||
detect_hosts 2>/dev/null
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
REQUIRED=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
else
|
||||
REQUIRED=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
fi
|
||||
|
||||
# Required containers — aliased by detect_hosts() → WATCHDOG_REQUIRED_CONTAINERS
|
||||
REQUIRED_ISSUES=0
|
||||
if [[ ${#REQUIRED[@]} -gt 0 ]]; then
|
||||
if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " 🔐 Required containers:"
|
||||
for container in "${REQUIRED[@]}"; do
|
||||
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
echo " ✅ $container"
|
||||
else
|
||||
echo " ❌ $container — $STATUS"
|
||||
((REQUIRED_ISSUES++))
|
||||
(( REQUIRED_ISSUES++ ))
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Tier 1 monitored containers from WATCHDOG_CONTAINERS
|
||||
# Memory-monitored containers — aliased by detect_hosts() → WATCHDOG_CONTAINERS
|
||||
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " 📊 Monitored containers (memory):"
|
||||
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
|
||||
LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}"
|
||||
LIMIT_GB=$(awk "BEGIN {printf \"%.0f\", $LIMIT_MB / 1024}")
|
||||
USAGE=$(docker stats --no-stream --format "{{.MemUsage}}" "$container" \
|
||||
2>/dev/null | awk '{print $1}')
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
|
||||
USAGE=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
|
||||
--format "{{.MemUsage}}" "$container" 2>/dev/null | awk '{print $1}')
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.Running}}' "$container" 2>/dev/null || echo "not found")
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
echo " ✅ $container: ${USAGE:-?} (limit: ${LIMIT_GB}GB)"
|
||||
else
|
||||
@@ -361,22 +368,22 @@ else
|
||||
echo " Docker not available"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Failover ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
header "🔀 FAILOVER"
|
||||
# ==============================================================================================
|
||||
section "🔀 FAILOVER"
|
||||
|
||||
FAILOVER_PID=$(get_lock_pid "failover")
|
||||
FAILOVER_RUNNING=false
|
||||
|
||||
if is_watchdog_running "failover"; then
|
||||
if is_script_running "failover"; then
|
||||
FAILOVER_RUNNING=true
|
||||
FAILOVER_AGE=$(get_lock_age "failover")
|
||||
FAILOVER_UPTIME=$(format_uptime "$FAILOVER_AGE")
|
||||
echo " ✅ Running │ PID: $FAILOVER_PID │ Uptime: $FAILOVER_UPTIME"
|
||||
else
|
||||
if [[ "${FAILOVER_ENABLED:-true}" == false ]]; then
|
||||
echo " ⏸️ Disabled — FAILOVER_ENABLED=false in Master.conf"
|
||||
echo " ⏸️ Disabled — FAILOVER_ENABLED=false in master.conf"
|
||||
else
|
||||
echo " ❌ NOT RUNNING — failover.sh is not active"
|
||||
echo " Start via: bash Orchestrators/array_start.sh"
|
||||
@@ -387,60 +394,55 @@ echo ""
|
||||
|
||||
# Failover state
|
||||
FAILOVER_STATE="UNKNOWN"
|
||||
FAILOVER_LAST_CHANGE=""
|
||||
FAILOVER_STATE_SECONDS=0
|
||||
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
FAILOVER_LAST_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
FAILOVER_LAST_EPOCH=$(grep "^last_change_epoch=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
if [[ -n "$FAILOVER_LAST_EPOCH" ]]; then
|
||||
FAILOVER_LAST_EPOCH=$(grep "^failover_start=" "$FAILOVER_STATE_FILE" \
|
||||
2>/dev/null | cut -d= -f2)
|
||||
if [[ -n "$FAILOVER_LAST_EPOCH" && "$FAILOVER_LAST_EPOCH" -gt 0 ]]; then
|
||||
FAILOVER_STATE_SECONDS=$(( $(date +%s) - FAILOVER_LAST_EPOCH ))
|
||||
fi
|
||||
fi
|
||||
|
||||
STATE_DURATION=$(format_uptime "${FAILOVER_STATE_SECONDS:-0}")
|
||||
|
||||
# Tier delays via REMOTE_ID — same logic as failover.sh
|
||||
REMOTE_TIER2_VAR="${REMOTE_ID}_TIER2_DELAY"
|
||||
REMOTE_TIER3_VAR="${REMOTE_ID}_TIER3_DELAY"
|
||||
REMOTE_TIER4_VAR="${REMOTE_ID}_TIER4_DELAY"
|
||||
TIER2_DELAY="${!REMOTE_TIER2_VAR:-240}"
|
||||
TIER3_DELAY="${!REMOTE_TIER3_VAR:-720}"
|
||||
TIER4_DELAY="${!REMOTE_TIER4_VAR:-1440}"
|
||||
|
||||
case "$FAILOVER_STATE" in
|
||||
NORMAL)
|
||||
echo " ✅ State: NORMAL"
|
||||
echo " 📅 In NORMAL state for: $STATE_DURATION"
|
||||
;;
|
||||
FAILOVER)
|
||||
echo " ⚠️ State: FAILOVER — remote server down"
|
||||
echo " ⚠️ State: FAILOVER — $REMOTE_SERVER_NAME is down"
|
||||
echo " ⏱️ Duration: $STATE_DURATION"
|
||||
# Show which tiers are active
|
||||
TIER1_DELAY=0
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
TIER2_DELAY=$HOST2_TIER2_DELAY
|
||||
TIER3_DELAY=$HOST2_TIER3_DELAY
|
||||
TIER4_DELAY=$HOST2_TIER4_DELAY
|
||||
else
|
||||
TIER2_DELAY=$HOST1_TIER2_DELAY
|
||||
TIER3_DELAY=$HOST1_TIER3_DELAY
|
||||
TIER4_DELAY=$HOST1_TIER4_DELAY
|
||||
fi
|
||||
FAILOVER_MINS=$(( FAILOVER_STATE_SECONDS / 60 ))
|
||||
echo ""
|
||||
echo " 🔄 Tier status:"
|
||||
echo " Tier 1 (immediate): ✅ active"
|
||||
echo " Tier 1 (immediate): ✅ active"
|
||||
if (( FAILOVER_MINS >= TIER2_DELAY )); then
|
||||
echo " Tier 2 (${TIER2_DELAY}min): ✅ active"
|
||||
echo " Tier 2 (${TIER2_DELAY}min): ✅ active"
|
||||
else
|
||||
REMAINING=$(( TIER2_DELAY - FAILOVER_MINS ))
|
||||
echo " Tier 2 (${TIER2_DELAY}min): ⏳ activates in ${REMAINING}min"
|
||||
echo " Tier 2 (${TIER2_DELAY}min): ⏳ in ${REMAINING}min"
|
||||
fi
|
||||
if (( FAILOVER_MINS >= TIER3_DELAY )); then
|
||||
echo " Tier 3 (${TIER3_DELAY}min): ✅ active"
|
||||
echo " Tier 3 (${TIER3_DELAY}min): ✅ active"
|
||||
else
|
||||
REMAINING=$(( TIER3_DELAY - FAILOVER_MINS ))
|
||||
echo " Tier 3 (${TIER3_DELAY}min): ⏳ activates in ${REMAINING}min"
|
||||
echo " Tier 3 (${TIER3_DELAY}min): ⏳ in ${REMAINING}min"
|
||||
fi
|
||||
if (( FAILOVER_MINS >= TIER4_DELAY )); then
|
||||
echo " Tier 4 (${TIER4_DELAY}min): ✅ active"
|
||||
echo " Tier 4 (${TIER4_DELAY}min): ✅ active"
|
||||
else
|
||||
REMAINING=$(( TIER4_DELAY - FAILOVER_MINS ))
|
||||
echo " Tier 4 (${TIER4_DELAY}min): ⏳ activates in ${REMAINING}min"
|
||||
echo " Tier 4 (${TIER4_DELAY}min): ⏳ in ${REMAINING}min"
|
||||
fi
|
||||
;;
|
||||
NO_INTERNET)
|
||||
@@ -448,7 +450,7 @@ case "$FAILOVER_STATE" in
|
||||
echo " ⏱️ Down for: $STATE_DURATION"
|
||||
;;
|
||||
DARK)
|
||||
echo " ❌ State: DARK — remote down AND no internet"
|
||||
echo " ❌ State: DARK — $REMOTE_SERVER_NAME down AND no internet"
|
||||
echo " ⏱️ Duration: $STATE_DURATION"
|
||||
;;
|
||||
*)
|
||||
@@ -456,19 +458,18 @@ case "$FAILOVER_STATE" in
|
||||
;;
|
||||
esac
|
||||
|
||||
# Tailscale remote visibility
|
||||
# Tailscale remote visibility — uses REMOTE_SERVER_NAME from detect_hosts()
|
||||
echo ""
|
||||
if command -v tailscale >/dev/null 2>&1; then
|
||||
REMOTE_IP=$(tailscale ip -4 "$HOST2" 2>/dev/null)
|
||||
REMOTE_IP=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null)
|
||||
if [[ -n "$REMOTE_IP" ]]; then
|
||||
# Try a quick ping to see last seen
|
||||
if ping -c 1 -W 2 "$REMOTE_IP" >/dev/null 2>&1; then
|
||||
echo " 🌐 Remote: $REMOTE_IP │ reachable ✅"
|
||||
echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_IP — reachable ✅"
|
||||
else
|
||||
echo " 🌐 Remote: $REMOTE_IP │ not responding ⚠️"
|
||||
echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_IP — not responding ⚠️"
|
||||
fi
|
||||
else
|
||||
echo " 🌐 Remote: $HOST2 not visible on Tailscale ❌"
|
||||
echo " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME) not visible on Tailscale ❌"
|
||||
fi
|
||||
else
|
||||
echo " 🌐 Tailscale: not available"
|
||||
@@ -476,28 +477,27 @@ fi
|
||||
|
||||
echo " 📡 Check interval: ${FAILOVER_CHECK_INTERVAL}s │ Handback strikes: ${FAILOVER_HANDBACK_STRIKES}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Footer ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Overall status
|
||||
ISSUES=0
|
||||
[[ "$SYS_RUNNING" == false ]] && ((ISSUES++))
|
||||
[[ "$DOCKER_RUNNING" == false ]] && ((ISSUES++))
|
||||
[[ "$FAILOVER_RUNNING" == false ]] && [[ "${FAILOVER_ENABLED:-true}" != false ]] && ((ISSUES++))
|
||||
[[ -n "$ACTIVE_STRIKES" ]] && ((ISSUES++))
|
||||
[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && ((ISSUES++))
|
||||
[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && ((ISSUES++))
|
||||
[[ "$FAILOVER_STATE" != "NORMAL" ]] && [[ "$FAILOVER_STATE" != "UNKNOWN" ]] && ((ISSUES++))
|
||||
[[ "$SYS_RUNNING" == false ]] && (( ISSUES++ ))
|
||||
[[ "$DOCKER_RUNNING" == false ]] && (( ISSUES++ ))
|
||||
[[ "$FAILOVER_RUNNING" == false && "${FAILOVER_ENABLED:-true}" != false ]] && (( ISSUES++ ))
|
||||
[[ -n "$ACTIVE_STRIKES" ]] && (( ISSUES++ ))
|
||||
[[ -n "$ACTIVE_CONTAINER_STRIKES" ]] && (( ISSUES++ ))
|
||||
[[ "${REQUIRED_ISSUES:-0}" -gt 0 ]] && (( ISSUES++ ))
|
||||
[[ "$FAILOVER_STATE" != "NORMAL" && "$FAILOVER_STATE" != "UNKNOWN" ]] && (( ISSUES++ ))
|
||||
|
||||
if [[ "$ISSUES" -eq 0 ]]; then
|
||||
echo " ✅ All continuous scripts healthy — no issues detected"
|
||||
echo " ✅ $MY_ID — all continuous scripts healthy"
|
||||
else
|
||||
echo " ⚠️ $ISSUES issue(s) detected — review above"
|
||||
fi
|
||||
|
||||
echo " 🕐 Checked at: $(date '+%H:%M:%S')"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " 🕐 Checked: $(date '+%H:%M:%S')"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
+214
-78
@@ -1,37 +1,75 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Emby Session Report ----------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Generates a weekly usage report from the Emby media server via its API.
|
||||
# Queries activity logs, session history and library stats to produce a
|
||||
# human-readable summary of what was watched, by whom and how.
|
||||
# ==============================================================================================
|
||||
# ================================= Emby Session Report ========================================
|
||||
# ==============================================================================================
|
||||
# Generates a usage report from the Emby media server via its API.
|
||||
# Queries activity logs and session history to produce a summary of what was
|
||||
# watched, by whom, and how over the configured report period.
|
||||
#
|
||||
# Report includes:
|
||||
# Total streams during the report period
|
||||
# Transcode vs direct play ratio
|
||||
# Live TV usage
|
||||
# Top N most watched content
|
||||
# Most active users
|
||||
# Peak concurrent streams
|
||||
# ── REPORT INCLUDES ───────────────────────────────────────────────────────────────────────────
|
||||
# Server info — name, version, uptime
|
||||
# Active sessions — current streams, direct play vs transcode
|
||||
# Library stats — movie, episode, song counts
|
||||
# Activity history — play events from the last EMBY_REPORT_DAYS days
|
||||
# Top content — most played items in the period (top EMBY_REPORT_TOP_N)
|
||||
# Most active users — who watched the most in the period
|
||||
# Transcode ratio — how often transcoding was needed vs direct play
|
||||
# Ramdisk status — current transcode location and usage
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases EMBY_URL and EMBY_API_KEY.
|
||||
# Each server reports on its own Emby instance automatically.
|
||||
#
|
||||
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
|
||||
# No persistent writes — queries API fresh each run.
|
||||
# All configuration in Master.conf under Emby Session Report section.
|
||||
# Supports --dry-run to test API connectivity without sending notification.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# This is a monitor/report script — SILENT_MODE=false — output is the point.
|
||||
# Silent when healthy (no notification on clean run).
|
||||
# Notifies only if transcoding is very high (>80% of streams) — may indicate config issue.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents duplicate reports running simultaneously
|
||||
# check_api() — verifies Emby reachable before queries
|
||||
# jq + curl validation — exits if either tool missing
|
||||
# validate_unraid_cmd — notify script validated before use
|
||||
# Per-section guards — API failure in one section does not abort others
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_EMBY_URL / HOST*_EMBY_API_KEY
|
||||
# Aliased by detect_hosts() — script uses EMBY_URL / EMBY_API_KEY
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# EMBY_REPORT_DAYS — days to include in the report period (default 7)
|
||||
# EMBY_REPORT_TOP_N — number of top content items to show (default 10)
|
||||
# RAMDISK_PATH — ramdisk mount path (for transcode status)
|
||||
# TRANSCODE_LINK — symlink path (for transcode location)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# emby_session_report.sh — generate report
|
||||
# emby_session_report.sh --dry-run — test API connectivity only, no notification
|
||||
# emby_session_report.sh --log — verbose output
|
||||
# emby_session_report.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Monitor/report script — output is the point
|
||||
SILENT_MODE=false
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v curl >/dev/null 2>&1; then
|
||||
error "curl not found — required for Emby API calls"
|
||||
exit 1
|
||||
@@ -43,28 +81,40 @@ if ! command -v jq >/dev/null 2>&1; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Select correct Emby instance based on which server is running this script
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases EMBY_URL, EMBY_API_KEY
|
||||
detect_hosts
|
||||
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
EMBY_URL="$HOST1_EMBY_URL"
|
||||
EMBY_API_KEY="$HOST1_EMBY_API_KEY"
|
||||
else
|
||||
EMBY_URL="$HOST2_EMBY_URL"
|
||||
EMBY_API_KEY="$HOST2_EMBY_API_KEY"
|
||||
fi
|
||||
|
||||
info "Emby instance: $LOCAL_SERVER_NAME → $EMBY_URL"
|
||||
|
||||
require_var EMBY_URL
|
||||
require_var EMBY_API_KEY
|
||||
|
||||
success "Config validated"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — API will be queried but no notification sent"
|
||||
log "Emby: $EMBY_URL"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — API queried but no notification sent"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# API HELPER
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_EMBY Emby URL: $EMBY_URL"
|
||||
echo "$ICON_TIME Period: Last ${EMBY_REPORT_DAYS} days"
|
||||
echo "$ICON_EMBY Top N: ${EMBY_REPORT_TOP_N} items"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── API HELPER ────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
emby_api() {
|
||||
local endpoint="$1"
|
||||
local response http_code body
|
||||
@@ -79,54 +129,66 @@ emby_api() {
|
||||
body=$(echo "$response" | head -n -1)
|
||||
|
||||
if [[ "$http_code" != "200" ]]; then
|
||||
error "Emby API returned HTTP $http_code for: $endpoint"
|
||||
error "Emby API HTTP $http_code for: $endpoint"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "$body"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_EMBY Emby Session Report ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Emby Session Report ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_EMBY Emby Session Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_EMBY URL: $EMBY_URL"
|
||||
echo "$ICON_TIME Period: Last ${EMBY_REPORT_DAYS} days"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Period: Last ${EMBY_REPORT_DAYS} days"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
# Test connectivity
|
||||
info "Testing Emby API connectivity..."
|
||||
# ── Connectivity and server info ──────────────────────────────────────────────────────────────
|
||||
if ! check_api "$EMBY_URL" "Emby" 10; then
|
||||
notify "Emby report failed on $(hostname) — cannot connect to Emby at $EMBY_URL" \
|
||||
"Emby Report" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SYSTEM_INFO=$(emby_api "System/Info" 2>/dev/null) || {
|
||||
error "Cannot connect to Emby at $EMBY_URL"
|
||||
notify "Emby report failed on $(hostname) — cannot connect to Emby" "Emby Report" "warning"
|
||||
exit 1
|
||||
}
|
||||
|
||||
SERVER_NAME=$(echo "$SYSTEM_INFO" | jq -r '.ServerName // "Unknown"' 2>/dev/null)
|
||||
SERVER_VERSION=$(echo "$SYSTEM_INFO" | jq -r '.Version // "Unknown"' 2>/dev/null)
|
||||
success "Connected to: $SERVER_NAME (v$SERVER_VERSION)"
|
||||
echo ""
|
||||
log "Connected to: $SERVER_NAME (v$SERVER_VERSION)"
|
||||
|
||||
# Calculate date range
|
||||
REPORT_START=$(date -d "${EMBY_REPORT_DAYS} days ago" '+%Y-%m-%dT00:00:00')
|
||||
|
||||
# ── Active Sessions ──────────────────────────────────────────────────────────────────────────
|
||||
# ── Active Sessions ───────────────────────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_EMBY Active Sessions ━━━"
|
||||
SESSIONS=$(emby_api "Sessions" 2>/dev/null) || { warn "Could not fetch sessions"; SESSIONS="[]"; }
|
||||
|
||||
ACTIVE_COUNT=$(echo "$SESSIONS" | jq '[.[] | select(.NowPlayingItem != null)] | length' 2>/dev/null || echo 0)
|
||||
TRANSCODE_COUNT=$(echo "$SESSIONS" | jq '[.[] | select(.NowPlayingItem != null) | select(.TranscodingInfo != null)] | length' 2>/dev/null || echo 0)
|
||||
DIRECT_COUNT=$(( ACTIVE_COUNT - TRANSCODE_COUNT ))
|
||||
ACTIVE_COUNT=$(echo "$SESSIONS" | \
|
||||
jq '[.[] | select(.NowPlayingItem != null)] | length' 2>/dev/null || echo 0)
|
||||
TRANSCODE_NOW=$(echo "$SESSIONS" | \
|
||||
jq '[.[] | select(.NowPlayingItem != null) | select(.TranscodingInfo != null)] | length' \
|
||||
2>/dev/null || echo 0)
|
||||
DIRECT_NOW=$(( ACTIVE_COUNT - TRANSCODE_NOW ))
|
||||
|
||||
echo " $ICON_EMBY Active streams: $ACTIVE_COUNT"
|
||||
echo " $ICON_EMBY Direct play: $DIRECT_COUNT"
|
||||
echo " $ICON_EMBY Transcoding: $TRANSCODE_COUNT"
|
||||
echo " $ICON_EMBY Active streams: $ACTIVE_COUNT"
|
||||
echo " $ICON_EMBY Direct play: $DIRECT_NOW"
|
||||
echo " $ICON_EMBY Transcoding: $TRANSCODE_NOW"
|
||||
|
||||
if [[ "$ACTIVE_COUNT" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " Now playing:"
|
||||
echo "$SESSIONS" | jq -r '
|
||||
.[] |
|
||||
select(.NowPlayingItem != null) |
|
||||
" \(.UserName // "Unknown") → \(.NowPlayingItem.Name // "Unknown") [\(if .TranscodingInfo != null then "transcode" else "direct" end)]"
|
||||
' 2>/dev/null || true
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ── Library Stats ────────────────────────────────────────────────────────────────────────────
|
||||
# ── Library Stats ─────────────────────────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_EMBY Library ━━━"
|
||||
ITEMS=$(emby_api "Items/Counts" 2>/dev/null) || { warn "Could not fetch library counts"; ITEMS="{}"; }
|
||||
|
||||
@@ -139,31 +201,105 @@ echo " $ICON_EMBY Episodes: $EPISODE_COUNT"
|
||||
echo " $ICON_EMBY Songs: $SONG_COUNT"
|
||||
echo ""
|
||||
|
||||
# ── Ramdisk Status (from state file) ────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_RAM Transcode Location ━━━"
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
|
||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
|
||||
SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
||||
echo " $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB"
|
||||
echo " $ICON_LINK Symlink target: $SYMLINK"
|
||||
# ── Activity History ──────────────────────────────────────────────────────────────────────────
|
||||
# Query activity log for the configured period
|
||||
echo "━━━ $ICON_EMBY Activity — Last ${EMBY_REPORT_DAYS} Days ━━━"
|
||||
|
||||
REPORT_START=$(date -d "${EMBY_REPORT_DAYS} days ago" '+%Y-%m-%dT00:00:00.000Z')
|
||||
|
||||
ACTIVITY=$(emby_api "System/ActivityLog/Entries?MinDate=${REPORT_START}&Limit=1000" \
|
||||
2>/dev/null) || { warn "Could not fetch activity log"; ACTIVITY="{}"; }
|
||||
|
||||
TOTAL_PLAYS=$(echo "$ACTIVITY" | \
|
||||
jq '[.Items // [] | .[] | select(.Type == "VideoPlayback" or .Type == "AudioPlayback")] | length' \
|
||||
2>/dev/null || echo 0)
|
||||
|
||||
TRANSCODE_PLAYS=$(echo "$ACTIVITY" | \
|
||||
jq '[.Items // [] | .[] | select(.Type == "VideoPlaybackUnplugged" or
|
||||
(.Type == "VideoPlayback" and (.Overview // "" | contains("Transcode"))))] | length' \
|
||||
2>/dev/null || echo 0)
|
||||
|
||||
echo " $ICON_EMBY Total play events: $TOTAL_PLAYS"
|
||||
|
||||
if [[ "$TOTAL_PLAYS" -gt 0 ]]; then
|
||||
TRANSCODE_PCT=$(awk "BEGIN {printf \"%.0f\", ($TRANSCODE_PLAYS / $TOTAL_PLAYS) * 100}")
|
||||
DIRECT_PCT=$(( 100 - TRANSCODE_PCT ))
|
||||
echo " $ICON_EMBY Direct play: ~${DIRECT_PCT}%"
|
||||
echo " $ICON_EMBY Transcoded: ~${TRANSCODE_PCT}%"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ── Top Content ───────────────────────────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_EMBY Top ${EMBY_REPORT_TOP_N} Content ━━━"
|
||||
|
||||
TOP_ITEMS=$(emby_api "Items?SortBy=DatePlayed&SortOrder=Descending&Limit=${EMBY_REPORT_TOP_N}&Recursive=true&Fields=Overview&IncludeItemTypes=Movie,Episode" \
|
||||
2>/dev/null) || { warn "Could not fetch top content"; TOP_ITEMS="{}"; }
|
||||
|
||||
TOP_COUNT=$(echo "$TOP_ITEMS" | jq '.Items // [] | length' 2>/dev/null || echo 0)
|
||||
if [[ "$TOP_COUNT" -gt 0 ]]; then
|
||||
echo "$TOP_ITEMS" | jq -r '
|
||||
.Items // [] |
|
||||
to_entries[] |
|
||||
" \(.key + 1). \(.value.Name // "Unknown") [\(.value.Type // "")]"
|
||||
' 2>/dev/null || warn "Could not parse top content"
|
||||
else
|
||||
echo " $ICON_RAM Ramdisk: not mounted"
|
||||
echo " No recent play history found"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ── Most Active Users ─────────────────────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_EMBY Most Active Users ━━━"
|
||||
USERS=$(emby_api "Users" 2>/dev/null) || { warn "Could not fetch users"; USERS="[]"; }
|
||||
|
||||
USER_COUNT=$(echo "$USERS" | jq 'length' 2>/dev/null || echo 0)
|
||||
echo " $ICON_EMBY Total users: $USER_COUNT"
|
||||
|
||||
if [[ "$USER_COUNT" -gt 0 ]]; then
|
||||
echo "$USERS" | jq -r '
|
||||
sort_by(.LastActivityDate // "0") |
|
||||
reverse |
|
||||
.[:5][] |
|
||||
" \(.Name // "Unknown") — last active: \(.LastActivityDate // "never" | split("T")[0])"
|
||||
' 2>/dev/null || true
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# ── Ramdisk / Transcode Status ────────────────────────────────────────────────────────────────
|
||||
echo "━━━ $ICON_RAM Transcode Status ━━━"
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
|
||||
SYMLINK=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
||||
echo " $ICON_RAM Ramdisk usage: ${RAMDISK_USED_GB}GB / ${RAMDISK_SIZE:-8G}"
|
||||
echo " $ICON_LINK Symlink target: $SYMLINK"
|
||||
if [[ "$SYMLINK" == *"ssd"* ]] || [[ "$SYMLINK" == *"cache"* ]]; then
|
||||
warn "Transcode link pointing at SSD — ramdisk may be full"
|
||||
fi
|
||||
else
|
||||
warn "Ramdisk not mounted at $RAMDISK_PATH"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━━━ $ICON_SUMMARY EMBY REPORT SUMMARY ━━━━━"
|
||||
echo "$ICON_EMBY Server: $SERVER_NAME (v$SERVER_VERSION)"
|
||||
echo "$ICON_EMBY Active: $ACTIVE_COUNT streams ($DIRECT_COUNT direct / $TRANSCODE_COUNT transcode)"
|
||||
echo "$ICON_EMBY Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes $SONG_COUNT songs"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_EMBY Server: $SERVER_NAME (v$SERVER_VERSION)"
|
||||
echo "$ICON_EMBY Active: $ACTIVE_COUNT streams ($DIRECT_NOW direct / $TRANSCODE_NOW transcode)"
|
||||
echo "$ICON_EMBY Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes $SONG_COUNT songs"
|
||||
echo "$ICON_EMBY Period: $TOTAL_PLAYS play events in last ${EMBY_REPORT_DAYS} days"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Only notify on issues — high transcode rate may indicate config problem
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
notify "Emby report on $(hostname) — $ACTIVE_COUNT active streams ($DIRECT_COUNT direct / $TRANSCODE_COUNT transcode) — Library: $MOVIE_COUNT movies $EPISODE_COUNT episodes" "Emby Report" "normal"
|
||||
fi
|
||||
if [[ "$TOTAL_PLAYS" -gt 10 && "${TRANSCODE_PCT:-0}" -gt 80 ]]; then
|
||||
notify "Emby report on $(hostname) — high transcode rate: ${TRANSCODE_PCT}% of $TOTAL_PLAYS plays — check direct play config" \
|
||||
"Emby Report" "warning"
|
||||
fi
|
||||
fi
|
||||
|
||||
exit 0
|
||||
+195
-95
@@ -1,35 +1,67 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- SMART Health Monitor ---------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= SMART Health Monitor =======================================
|
||||
# ==============================================================================================
|
||||
# Checks SMART health attributes for all drives on the system.
|
||||
# Reads data live from each drive via smartctl — no persistent writes.
|
||||
# Designed to run weekly as a scheduled report.
|
||||
#
|
||||
# Monitored attributes:
|
||||
# Reallocated_Sector_Ct — bad sectors remapped — any > 0 is concerning
|
||||
# Current_Pending_Sector — sectors waiting for reallocation — any > 0 is concerning
|
||||
# Offline_Uncorrectable — sectors that could not be corrected — any > 0 is critical
|
||||
# Temperature_Celsius — drive temperature vs SMART_TEMP_WARN / SMART_TEMP_CRIT
|
||||
# Power_On_Hours — informational — drive age estimation
|
||||
# SMART overall status — pass/fail per drive
|
||||
# ── MONITORED ATTRIBUTES ──────────────────────────────────────────────────────────────────────
|
||||
# Overall SMART status — PASSED/FAILED — immediate fail = drive is dying
|
||||
# Reallocated_Sector_Ct — bad sectors remapped — any > 0 is concerning
|
||||
# Current_Pending_Sector — sectors waiting for reallocation — any > 0 is concerning
|
||||
# Offline_Uncorrectable — sectors that could not be corrected — any > 0 is critical
|
||||
# Temperature_Celsius — vs thresholds from dynamix.cfg (or master.conf fallback)
|
||||
# Power_On_Hours — informational — drive age in days
|
||||
#
|
||||
# Discovers drives automatically — no configuration needed for drive list.
|
||||
# SMART_IGNORE_DRIVES allows skipping specific drives (e.g. USB flash drives).
|
||||
# ── DRIVE DISCOVERY ───────────────────────────────────────────────────────────────────────────
|
||||
# Discovers drives automatically via /dev/sd* and /dev/nvme* — no config needed.
|
||||
# NVMe drives use different attribute names — detected and handled automatically.
|
||||
# HOST*_SMART_IGNORE_DRIVES skips specific drives (e.g. boot USB flash drive).
|
||||
#
|
||||
# All configuration in Master.conf under SMART Health section.
|
||||
# Supports --dry-run to show which drives would be checked without running smartctl.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── TEMPERATURE THRESHOLDS ────────────────────────────────────────────────────────────────────
|
||||
# Reads hot/max/hotssd/maxssd from /boot/config/plugins/dynamix/dynamix.cfg at runtime.
|
||||
# Uses unRAID's own configured thresholds — no need to duplicate them here.
|
||||
# Falls back to SMART_TEMP_WARN / SMART_TEMP_CRIT from master.conf if dynamix.cfg not found.
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_SMART_IGNORE_DRIVES → SMART_IGNORE_DRIVES.
|
||||
# Each server monitors its own drives with its own ignore list.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — smartctl calls are slow, prevent duplicate runs
|
||||
# detect_hosts() — correct ignore list per host via MY_ID aliases
|
||||
# validate_unraid_cmd — smartctl and notify validated before use
|
||||
# Silent healthy drives — only problems produce output
|
||||
# Silent healthy run — no notify when all drives pass
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_SMART_IGNORE_DRIVES — drives skipped in SMART monitoring
|
||||
# Aliased by detect_hosts() — script uses SMART_IGNORE_DRIVES
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# SMART_TEMP_WARN — fallback warn threshold in °C (if dynamix.cfg not found)
|
||||
# SMART_TEMP_CRIT — fallback crit threshold in °C (if dynamix.cfg not found)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# smart_health.sh — normal run
|
||||
# smart_health.sh --dry-run — show which drives would be checked
|
||||
# smart_health.sh --log — verbose output
|
||||
# smart_health.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Monitor script — output is the point
|
||||
SILENT_MODE=false
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
@@ -38,35 +70,56 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
# Validate smartctl — required for all drive checks
|
||||
validate_unraid_cmd \
|
||||
"$(command -v smartctl 2>/dev/null || echo /usr/bin/smartctl)" \
|
||||
"--version" "smartmontools" \
|
||||
"smartctl" || {
|
||||
error "smartctl not found — install smartmontools"
|
||||
notify "SMART health check failed on $(hostname) — smartmontools not installed" \
|
||||
"SMART Health" "warning"
|
||||
exit 1
|
||||
}
|
||||
|
||||
if ! command -v smartctl >/dev/null 2>&1; then
|
||||
error "smartctl not found — install smartmontools"
|
||||
notify "SMART health check failed on $(hostname) — smartmontools not installed" "SMART Health" "warning"
|
||||
exit 1
|
||||
fi
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
success "smartctl available"
|
||||
acquire_lock
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_SMART_IGNORE_DRIVES
|
||||
detect_hosts
|
||||
|
||||
# Load temperature thresholds from dynamix.cfg — unRAID's own settings
|
||||
get_unraid_temp_thresholds
|
||||
|
||||
log "HDD warn: ${UNRAID_DISK_HOT}°C crit: ${UNRAID_DISK_MAX}°C"
|
||||
log "SSD warn: ${UNRAID_SSD_HOT}°C crit: ${UNRAID_SSD_MAX}°C"
|
||||
log "Ignore: ${SMART_IGNORE_DRIVES[*]:-none}"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing drive list only, no SMART data read"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_SMART Temp warn: ${SMART_TEMP_WARN}°C"
|
||||
echo "$ICON_SMART Temp crit: ${SMART_TEMP_CRIT}°C"
|
||||
echo "$ICON_SMART Ignore drives: ${SMART_IGNORE_DRIVES[*]:-none}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_SMART HDD warn: ${UNRAID_DISK_HOT}°C"
|
||||
echo "$ICON_SMART HDD crit: ${UNRAID_DISK_MAX}°C"
|
||||
echo "$ICON_SMART SSD warn: ${UNRAID_SSD_HOT}°C"
|
||||
echo "$ICON_SMART SSD crit: ${UNRAID_SSD_MAX}°C"
|
||||
echo "$ICON_SMART Ignore drives: ${SMART_IGNORE_DRIVES[*]:-none}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
|
||||
# Show drives that would be checked
|
||||
echo "━━━ Discovered Drives ━━━"
|
||||
for drive in /dev/sd? /dev/nvme?; do
|
||||
[[ ! -e "$drive" ]] && continue
|
||||
drive_name=$(basename "$drive")
|
||||
ignored=false
|
||||
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
|
||||
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
|
||||
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
|
||||
done
|
||||
if [[ "$ignored" == true ]]; then
|
||||
@@ -79,23 +132,55 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing drive list only, no SMART data read"
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# HELPER — extract SMART attribute value
|
||||
# Usage: get_smart_attr "/dev/sda" "Reallocated_Sector_Ct"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Extract a named SMART attribute value (column 10 — raw value)
|
||||
get_smart_attr() {
|
||||
local drive="$1" attr="$2"
|
||||
smartctl -A "$drive" 2>/dev/null | \
|
||||
awk -v attr="$attr" '$2 == attr {print $10}'
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SMART SMART Health Check ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Get drive temperature — handles HDD (attribute) and NVMe (different output format)
|
||||
get_drive_temp() {
|
||||
local drive="$1"
|
||||
local temp
|
||||
|
||||
# Standard HDD SMART attribute
|
||||
temp=$(get_smart_attr "$drive" "Temperature_Celsius")
|
||||
[[ -n "$temp" ]] && echo "$temp" && return
|
||||
|
||||
# NVMe — temperature in different section
|
||||
temp=$(smartctl -A "$drive" 2>/dev/null | \
|
||||
awk '/Temperature:/{gsub(/[^0-9]/,"",$2); if($2>0) print $2; exit}')
|
||||
[[ -n "$temp" ]] && echo "$temp" && return
|
||||
|
||||
# Fallback — any temperature line
|
||||
temp=$(smartctl -A "$drive" 2>/dev/null | \
|
||||
awk '/Temp/{gsub(/[^0-9]/,"",$NF); if($NF>0 && $NF<120) print $NF; exit}')
|
||||
echo "${temp:-}"
|
||||
}
|
||||
|
||||
# Detect if a drive is SSD/NVMe (rotational=0)
|
||||
is_ssd() {
|
||||
local drive="$1"
|
||||
local dev_name
|
||||
dev_name=$(basename "$drive" | sed 's/nvme[0-9]/nvme0/')
|
||||
local rotational="/sys/block/$(basename "$drive")/queue/rotational"
|
||||
[[ -f "$rotational" ]] && [[ "$(cat "$rotational" 2>/dev/null)" == "0" ]] && return 0
|
||||
# NVMe is always SSD
|
||||
[[ "$drive" == *nvme* ]] && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ SMART Health Check ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_SMART SMART Health Check — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
@@ -110,12 +195,12 @@ for drive in /dev/sd? /dev/nvme?; do
|
||||
|
||||
# Check ignore list
|
||||
ignored=false
|
||||
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
|
||||
for ignore in "${SMART_IGNORE_DRIVES[@]:-}"; do
|
||||
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
|
||||
done
|
||||
|
||||
if [[ "$ignored" == true ]]; then
|
||||
info "$drive_name — ignored (in SMART_IGNORE_DRIVES)"
|
||||
log "$drive_name — ignored (SMART_IGNORE_DRIVES)"
|
||||
DRIVES_SKIP+=("$drive_name")
|
||||
continue
|
||||
fi
|
||||
@@ -128,34 +213,40 @@ for drive in /dev/sd? /dev/nvme?; do
|
||||
continue
|
||||
fi
|
||||
|
||||
# Check if drive supports SMART
|
||||
# Check SMART support
|
||||
if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
|
||||
warn "$drive_name — SMART not enabled or not supported"
|
||||
warn "$drive_name — SMART not enabled or not supported — skipping"
|
||||
DRIVES_SKIP+=("$drive_name")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
# Overall SMART status
|
||||
SMART_STATUS=$(smartctl -H "$drive" 2>/dev/null | grep "overall-health" | awk '{print $NF}')
|
||||
if [[ "$SMART_STATUS" == "PASSED" ]]; then
|
||||
success "Overall status: PASSED"
|
||||
else
|
||||
error "Overall status: $SMART_STATUS"
|
||||
fi
|
||||
|
||||
# Key attributes
|
||||
DRIVE_WARN=false
|
||||
DRIVE_CRIT=false
|
||||
|
||||
# Overall SMART status
|
||||
SMART_STATUS=$(smartctl -H "$drive" 2>/dev/null | \
|
||||
grep "overall-health" | awk '{print $NF}')
|
||||
case "${SMART_STATUS:-}" in
|
||||
PASSED)
|
||||
log "$drive_name overall status: PASSED" ;;
|
||||
FAILED*)
|
||||
error "$drive_name overall status: FAILED — drive may be failing"
|
||||
DRIVE_CRIT=true ;;
|
||||
"")
|
||||
warn "$drive_name overall status: unknown — could not read SMART data" ;;
|
||||
*)
|
||||
warn "$drive_name overall status: $SMART_STATUS" ;;
|
||||
esac
|
||||
|
||||
# Reallocated sectors
|
||||
REALLOC=$(get_smart_attr "$drive" "Reallocated_Sector_Ct")
|
||||
if [[ -n "$REALLOC" ]]; then
|
||||
if [[ "$REALLOC" -gt 0 ]]; then
|
||||
warn "$ICON_SMART Reallocated sectors: $REALLOC — drive showing wear"
|
||||
warn "$ICON_SMART $drive_name — Reallocated sectors: $REALLOC (drive showing wear)"
|
||||
DRIVE_WARN=true
|
||||
else
|
||||
success "$ICON_SMART Reallocated sectors: $REALLOC"
|
||||
log "$drive_name reallocated sectors: 0 ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -163,47 +254,53 @@ for drive in /dev/sd? /dev/nvme?; do
|
||||
PENDING=$(get_smart_attr "$drive" "Current_Pending_Sector")
|
||||
if [[ -n "$PENDING" ]]; then
|
||||
if [[ "$PENDING" -gt 0 ]]; then
|
||||
warn "$ICON_SMART Pending sectors: $PENDING — sectors awaiting reallocation"
|
||||
warn "$ICON_SMART $drive_name — Pending sectors: $PENDING (awaiting reallocation)"
|
||||
DRIVE_WARN=true
|
||||
else
|
||||
success "$ICON_SMART Pending sectors: $PENDING"
|
||||
log "$drive_name pending sectors: 0 ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Uncorrectable sectors
|
||||
# Uncorrectable sectors — critical threshold
|
||||
UNCORR=$(get_smart_attr "$drive" "Offline_Uncorrectable")
|
||||
if [[ -n "$UNCORR" ]]; then
|
||||
if [[ "$UNCORR" -gt 0 ]]; then
|
||||
error "$ICON_SMART Uncorrectable sectors: $UNCORR — CRITICAL"
|
||||
error "$ICON_SMART $drive_name — Uncorrectable sectors: $UNCORR — CRITICAL"
|
||||
DRIVE_CRIT=true
|
||||
else
|
||||
success "$ICON_SMART Uncorrectable sectors: $UNCORR"
|
||||
log "$drive_name uncorrectable sectors: 0 ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Temperature
|
||||
TEMP=$(get_smart_attr "$drive" "Temperature_Celsius")
|
||||
# NVMe uses different attribute name
|
||||
[[ -z "$TEMP" ]] && TEMP=$(smartctl -A "$drive" 2>/dev/null | \
|
||||
awk '/Temperature/{print $2}' | head -1)
|
||||
# Temperature — use SSD/HDD thresholds from dynamix.cfg
|
||||
TEMP=$(get_drive_temp "$drive")
|
||||
if [[ -n "$TEMP" ]] && [[ "$TEMP" =~ ^[0-9]+$ ]]; then
|
||||
if is_ssd "$drive"; then
|
||||
WARN_THRESH="$UNRAID_SSD_HOT"
|
||||
CRIT_THRESH="$UNRAID_SSD_MAX"
|
||||
DRIVE_TYPE="SSD"
|
||||
else
|
||||
WARN_THRESH="$UNRAID_DISK_HOT"
|
||||
CRIT_THRESH="$UNRAID_DISK_MAX"
|
||||
DRIVE_TYPE="HDD"
|
||||
fi
|
||||
|
||||
if [[ -n "$TEMP" ]]; then
|
||||
if [[ "$TEMP" -ge "$SMART_TEMP_CRIT" ]]; then
|
||||
error "$ICON_SMART Temperature: ${TEMP}°C — CRITICAL (threshold: ${SMART_TEMP_CRIT}°C)"
|
||||
if [[ "$TEMP" -ge "$CRIT_THRESH" ]]; then
|
||||
error "$ICON_SMART $drive_name — ${DRIVE_TYPE} temp: ${TEMP}°C — CRITICAL (threshold: ${CRIT_THRESH}°C)"
|
||||
DRIVE_CRIT=true
|
||||
elif [[ "$TEMP" -ge "$SMART_TEMP_WARN" ]]; then
|
||||
warn "$ICON_SMART Temperature: ${TEMP}°C — warning (threshold: ${SMART_TEMP_WARN}°C)"
|
||||
elif [[ "$TEMP" -ge "$WARN_THRESH" ]]; then
|
||||
warn "$ICON_SMART $drive_name — ${DRIVE_TYPE} temp: ${TEMP}°C — warning (threshold: ${WARN_THRESH}°C)"
|
||||
DRIVE_WARN=true
|
||||
else
|
||||
success "$ICON_SMART Temperature: ${TEMP}°C"
|
||||
log "$drive_name temp: ${TEMP}°C ${DRIVE_TYPE} ✅"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Power on hours — informational
|
||||
# Power on hours — informational only
|
||||
POH=$(get_smart_attr "$drive" "Power_On_Hours")
|
||||
if [[ -n "$POH" ]]; then
|
||||
POH_DAYS=$(( POH / 24 ))
|
||||
info "$ICON_SMART Power on hours: $POH (${POH_DAYS} days)"
|
||||
log "$drive_name power on hours: $POH (${POH_DAYS} days)"
|
||||
fi
|
||||
|
||||
# Classify drive
|
||||
@@ -220,30 +317,33 @@ done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━━━ $ICON_SUMMARY SMART HEALTH SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
echo " $ICON_SUCCESS Healthy: ${#DRIVES_OK[@]} $ICON_WARN Warning: ${#DRIVES_WARN[@]} $ICON_ERROR Critical: ${#DRIVES_CRIT[@]} skipped: ${#DRIVES_SKIP[@]}"
|
||||
echo " $ICON_SUCCESS Healthy: ${#DRIVES_OK[@]}"
|
||||
[[ ${#DRIVES_WARN[@]} -gt 0 ]] && warn "Warning: ${#DRIVES_WARN[@]} — ${DRIVES_WARN[*]}"
|
||||
[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && echo "$ICON_ERROR Critical: ${#DRIVES_CRIT[@]} — ${DRIVES_CRIT[*]}"
|
||||
[[ ${#DRIVES_SKIP[@]} -gt 0 ]] && log "Skipped: ${#DRIVES_SKIP[@]} — ${DRIVES_SKIP[*]}"
|
||||
echo ""
|
||||
|
||||
[[ ${#DRIVES_OK[@]} -gt 0 ]] && echo " $ICON_SUCCESS ${DRIVES_OK[*]}"
|
||||
[[ ${#DRIVES_WARN[@]} -gt 0 ]] && echo " $ICON_WARN ${DRIVES_WARN[*]}"
|
||||
[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && echo " $ICON_ERROR ${DRIVES_CRIT[*]}"
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN"
|
||||
warn "DRY RUN — no SMART data read"
|
||||
elif [[ ${#DRIVES_CRIT[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: CRITICAL — ${DRIVES_CRIT[*]}"
|
||||
notify "SMART CRITICAL on $(hostname) — drives need immediate attention: ${DRIVES_CRIT[*]}" "SMART Health" "warning"
|
||||
notify "SMART CRITICAL on $(hostname) — immediate attention needed: ${DRIVES_CRIT[*]}" \
|
||||
"SMART Health" "warning"
|
||||
elif [[ ${#DRIVES_WARN[@]} -gt 0 ]]; then
|
||||
echo "$ICON_WARN Status: WARNING — ${DRIVES_WARN[*]}"
|
||||
notify "SMART WARNING on $(hostname) — drives showing wear: ${DRIVES_WARN[*]}" "SMART Health" "warning"
|
||||
warn "Status: WARNING — ${DRIVES_WARN[*]}"
|
||||
notify "SMART WARNING on $(hostname) — drives showing wear: ${DRIVES_WARN[*]}" \
|
||||
"SMART Health" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DRIVES HEALTHY"
|
||||
notify "SMART health check passed on $(hostname) — ${#DRIVES_OK[@]} drives healthy" "SMART Health" "normal"
|
||||
log "$ICON_DONE Status: all ${#DRIVES_OK[@]} drives healthy ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -0,0 +1,205 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ============================= System Tuning Monitor ==========================================
|
||||
# ==============================================================================================
|
||||
# Tracks inotify and php-fpm usage over time.
|
||||
# Snapshots written every 6 hours — read by sunday_morning_coffee_report.sh for weekly summary.
|
||||
# Schedule: 0 */6 * * * (every 6 hours via User Scripts)
|
||||
#
|
||||
# ── WHAT IT TRACKS ────────────────────────────────────────────────────────────────────────────
|
||||
# inotify instances:
|
||||
# Current in use vs kernel limit
|
||||
# % utilization — warns above INOTIFY_WARN_PCT (default 80%)
|
||||
# Top 5 consumers by instance count
|
||||
# Symptom of exhaustion: containers miss file events, downloads not detected,
|
||||
# Live TV stutter, library not updated
|
||||
#
|
||||
# php-fpm workers:
|
||||
# Active workers vs PHP_MAX_CHILDREN limit
|
||||
# % utilization — warns above PHP_FPM_WARN_PCT (default 80%)
|
||||
# Symptom: unRAID WebGUI slowdowns or timeouts under load
|
||||
#
|
||||
# ── LOG FORMAT ────────────────────────────────────────────────────────────────────────────────
|
||||
# DATE|TIME|INOTIFY_USED|INOTIFY_LIMIT|INOTIFY_PCT|INOTIFY_WARN|PHPFPM_ACTIVE|PHPFPM_MAX|PHPFPM_PCT|PHPFPM_WARN
|
||||
# Log trimmed to TUNING_LOG_RETENTION days on each write — bounded size.
|
||||
#
|
||||
# ── WHAT THE WEEKLY REPORT SHOWS ──────────────────────────────────────────────────────────────
|
||||
# inotify: peak, average, warning count over the week
|
||||
# php-fpm: peak workers, average workers, warning count over the week
|
||||
#
|
||||
# ── SILENT BY DEFAULT ─────────────────────────────────────────────────────────────────────────
|
||||
# Background snapshot script — no output when healthy.
|
||||
# Warns to stderr when thresholds exceeded — visible in User Scripts output log.
|
||||
# Does NOT notify on every snapshot — only when threshold exceeded.
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# Each server writes to its own DATA_DIR — no collision between servers.
|
||||
# MY_ID included in warning output for clarity in shared notification channels.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — prevents overlapping 6-hour snapshots
|
||||
# root check — /proc/*/fd requires root access
|
||||
# atomic log write — tmp file + mv prevents partial writes on trim
|
||||
# validate_unraid — notify script validated before use
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# INOTIFY_WARN_PCT — warn threshold % (default 80)
|
||||
# PHP_FPM_WARN_PCT — warn threshold % (default 80)
|
||||
# PHP_MAX_CHILDREN — max php-fpm workers (set by php_fpm_max_children.sh)
|
||||
# TUNING_MONITOR_LOG — log file path
|
||||
# TUNING_LOG_RETENTION — days before old entries purged (default 30)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# system_tuning_monitor.sh — normal snapshot run
|
||||
# system_tuning_monitor.sh --dry-run — measure and show, no log write
|
||||
# system_tuning_monitor.sh --log — verbose output
|
||||
# system_tuning_monitor.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root — /proc/*/fd requires root access"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID — used in warning output
|
||||
detect_hosts
|
||||
|
||||
DATE=$(date '+%Y-%m-%d')
|
||||
TIME=$(date '+%H:%M')
|
||||
INOTIFY_WARN=0
|
||||
PHPFPM_WARN=0
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR inotify warn: ${INOTIFY_WARN_PCT:-80}%"
|
||||
echo "$ICON_GEAR php-fpm warn: ${PHP_FPM_WARN_PCT:-80}%"
|
||||
echo "$ICON_GEAR php-fpm max: ${PHP_MAX_CHILDREN:-250}"
|
||||
echo "$ICON_GEAR Log file: ${TUNING_MONITOR_LOG:-not set}"
|
||||
echo "$ICON_GEAR Retention: ${TUNING_LOG_RETENTION:-30} days"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
|
||||
if [[ -f "$TUNING_MONITOR_LOG" ]]; then
|
||||
ENTRY_COUNT=$(wc -l < "$TUNING_MONITOR_LOG")
|
||||
OLDEST=$(awk -F'|' 'NR==1{print $1}' "$TUNING_MONITOR_LOG")
|
||||
NEWEST=$(awk -F'|' 'END{print $1}' "$TUNING_MONITOR_LOG")
|
||||
echo "$ICON_MONITOR Log entries: $ENTRY_COUNT ($OLDEST → $NEWEST)"
|
||||
else
|
||||
echo "$ICON_MONITOR Log entries: none yet"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — measuring only, no log write"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ inotify ━━━
|
||||
# ==============================================================================================
|
||||
INOTIFY_LIMIT=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 0)
|
||||
INOTIFY_USED=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l)
|
||||
INOTIFY_USED="${INOTIFY_USED//[^0-9]/}"
|
||||
INOTIFY_USED="${INOTIFY_USED:-0}"
|
||||
|
||||
if [[ "$INOTIFY_LIMIT" -gt 0 ]]; then
|
||||
INOTIFY_PCT=$(( INOTIFY_USED * 100 / INOTIFY_LIMIT ))
|
||||
else
|
||||
INOTIFY_PCT=0
|
||||
fi
|
||||
|
||||
[[ "$INOTIFY_PCT" -ge "${INOTIFY_WARN_PCT:-80}" ]] && INOTIFY_WARN=1
|
||||
|
||||
# Top 5 inotify consumers
|
||||
INOTIFY_TOP=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | \
|
||||
awk -F/ '{print $3}' | sort | uniq -c | sort -rn | head -5 | \
|
||||
while read -r count pid; do
|
||||
comm=$(cat "/proc/$pid/comm" 2>/dev/null || echo "?")
|
||||
echo "${count}×${comm}"
|
||||
done | tr '\n' ',' | sed 's/,$//')
|
||||
|
||||
if [[ "$INOTIFY_WARN" -eq 1 ]]; then
|
||||
warn "$MY_ID — inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) — above ${INOTIFY_WARN_PCT}% threshold"
|
||||
warn "Top consumers: ${INOTIFY_TOP:-unknown}"
|
||||
warn "Symptoms: containers missing file events, library not updating, Live TV stutter"
|
||||
notify "inotify at ${INOTIFY_PCT}% on $(hostname) — ${INOTIFY_USED}/${INOTIFY_LIMIT} in use — top: ${INOTIFY_TOP}" \
|
||||
"System Tuning" "warning"
|
||||
else
|
||||
log "inotify: ${INOTIFY_USED}/${INOTIFY_LIMIT} (${INOTIFY_PCT}%) ✅"
|
||||
log "inotify top consumers: ${INOTIFY_TOP:-none}"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ php-fpm ━━━
|
||||
# ==============================================================================================
|
||||
PHPFPM_MAX="${PHP_MAX_CHILDREN:-250}"
|
||||
|
||||
PHPFPM_ACTIVE=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0)
|
||||
PHPFPM_ACTIVE="${PHPFPM_ACTIVE//[^0-9]/}"
|
||||
PHPFPM_ACTIVE="${PHPFPM_ACTIVE:-0}"
|
||||
|
||||
if [[ "$PHPFPM_MAX" -gt 0 ]]; then
|
||||
PHPFPM_PCT=$(( PHPFPM_ACTIVE * 100 / PHPFPM_MAX ))
|
||||
else
|
||||
PHPFPM_PCT=0
|
||||
fi
|
||||
|
||||
[[ "$PHPFPM_PCT" -ge "${PHP_FPM_WARN_PCT:-80}" ]] && PHPFPM_WARN=1
|
||||
|
||||
if [[ "$PHPFPM_WARN" -eq 1 ]]; then
|
||||
warn "$MY_ID — php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers (${PHPFPM_PCT}%) — above ${PHP_FPM_WARN_PCT}% threshold"
|
||||
warn "Symptom: unRAID WebGUI slowdowns or timeouts under load"
|
||||
notify "php-fpm at ${PHPFPM_PCT}% on $(hostname) — ${PHPFPM_ACTIVE}/${PHPFPM_MAX} workers active" \
|
||||
"System Tuning" "warning"
|
||||
else
|
||||
log "php-fpm: ${PHPFPM_ACTIVE}/${PHPFPM_MAX} active workers (${PHPFPM_PCT}%) ✅"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Write Snapshot ━━━
|
||||
# ==============================================================================================
|
||||
if [[ -z "${TUNING_MONITOR_LOG:-}" ]]; then
|
||||
warn "TUNING_MONITOR_LOG not set — snapshot not written"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — snapshot not written"
|
||||
warn "Would write: ${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname "$TUNING_MONITOR_LOG")"
|
||||
|
||||
# Trim old entries — atomic write via temp file
|
||||
if [[ -f "$TUNING_MONITOR_LOG" ]]; then
|
||||
CUTOFF=$(date -d "${TUNING_LOG_RETENTION:-30} days ago" '+%Y-%m-%d')
|
||||
awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' \
|
||||
"$TUNING_MONITOR_LOG" > "${TUNING_MONITOR_LOG}.tmp" && \
|
||||
mv "${TUNING_MONITOR_LOG}.tmp" "$TUNING_MONITOR_LOG"
|
||||
fi
|
||||
|
||||
# Append snapshot
|
||||
echo "${DATE}|${TIME}|${INOTIFY_USED}|${INOTIFY_LIMIT}|${INOTIFY_PCT}|${INOTIFY_WARN}|${PHPFPM_ACTIVE}|${PHPFPM_MAX}|${PHPFPM_PCT}|${PHPFPM_WARN}" \
|
||||
>> "$TUNING_MONITOR_LOG"
|
||||
|
||||
log "Snapshot written: inotify ${INOTIFY_PCT}% php-fpm ${PHPFPM_PCT}%"
|
||||
+205
-161
@@ -1,50 +1,117 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Health Digest ----------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= Health Digest ==============================================
|
||||
# ==============================================================================================
|
||||
# Aggregates system health data from across the ecosystem into a single digest report.
|
||||
# Reads existing state files — no new writes to flash drive.
|
||||
# Reads existing state files — no new writes.
|
||||
#
|
||||
# ── THREE PROFILES ────────────────────────────────────────────────────────────────────────────
|
||||
# always — sends every run regardless of findings
|
||||
# schedule daily for a daily digest
|
||||
#
|
||||
# smart — sends only if something worth reporting was found
|
||||
# runs every run but stays silent when all healthy
|
||||
# DIGEST_SMART_ON_* toggles control what triggers a send
|
||||
#
|
||||
# Three profiles controlled by DIGEST_PROFILE in Master.conf:
|
||||
# always — sends every run regardless of findings (schedule daily for daily digest)
|
||||
# smart — sends only if something worth reporting was found (intelligent filtering)
|
||||
# weekly — sends once per week on DIGEST_DAY regardless of schedule frequency
|
||||
# run daily, digest only fires on DIGEST_DAY (default Sunday)
|
||||
#
|
||||
# The cron schedule stays the same regardless of profile — just change DIGEST_PROFILE
|
||||
# in Master.conf to switch behavior. Run daily, profile controls when it actually notifies.
|
||||
# The cron schedule stays the same regardless of profile — change DIGEST_PROFILE in
|
||||
# master.conf to switch behaviour. No cron changes needed.
|
||||
#
|
||||
# Data sources (reads only — no writes):
|
||||
# /tmp/transcode_state.db — ramdisk symlink and usage
|
||||
# /tmp/container_watchdog_state.db — active container strikes
|
||||
# /tmp/system_watchdog_state.db — active system strikes
|
||||
# /boot/config/failover_state.db — current failover state
|
||||
# /boot/config/system_watchdog_failed.db — container skip list
|
||||
# /boot/config/bandwidth_history.db — recent transfer totals
|
||||
# SSL certs via openssl (live check) — days remaining per domain
|
||||
# ── DATA SOURCES (reads only) ─────────────────────────────────────────────────────────────────
|
||||
# FAILOVER_STATE_FILE — current failover state
|
||||
# SYS_WATCHDOG_FAILED_FILE — container skip list (manual intervention needed)
|
||||
# WATCHDOG_STATE_FILE — active container watchdog strikes
|
||||
# SYS_WATCHDOG_STATE_FILE — active system watchdog strikes
|
||||
# BANDWIDTH_LOG — yesterday's transfer totals
|
||||
# TRANSCODE_DAILY_LOG — weekly transcode statistics
|
||||
# CERT_MONITOR_DOMAINS — live SSL cert check via openssl
|
||||
# RAMDISK_PATH / TRANSCODE_LINK — current transcode location and usage
|
||||
#
|
||||
# All configuration in Master.conf under Health Digest section.
|
||||
# Supports --dry-run to generate report without sending notification.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases CERT_MONITOR_DOMAINS, RAMDISK_WARN_GB,
|
||||
# RAMDISK_SIZE, RAMDISK_LOW_GB and all other host-specific vars used in this report.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — report takes time, prevent duplicate runs
|
||||
# detect_hosts() — correct vars per host
|
||||
# validate_unraid_cmd — notify and openssl validated before use
|
||||
# Per-section guards — missing state file skipped cleanly
|
||||
# Silent smart profile — completely silent when nothing to report
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# DIGEST_PROFILE — always | smart | weekly
|
||||
# DIGEST_DAY — day name for weekly profile (e.g. Sunday)
|
||||
# DIGEST_SMART_ON_WATCHDOG — send on active watchdog strikes
|
||||
# DIGEST_SMART_ON_FAILOVER — send on non-NORMAL failover state
|
||||
# DIGEST_SMART_ON_CERT_WARN — send on cert warning
|
||||
# DIGEST_SMART_ON_BANDWIDTH — send on high bandwidth day
|
||||
# CERT_WARN_DAYS / CERT_CRIT_DAYS / CERT_TIMEOUT
|
||||
# BANDWIDTH_WARN_GB
|
||||
# TRANSCODE_DAILY_LOG
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# weekly_health_digest.sh — normal run
|
||||
# weekly_health_digest.sh --dry-run — generate report, no notification
|
||||
# weekly_health_digest.sh --log — verbose output
|
||||
# weekly_health_digest.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Report/monitor script — output is the point when sending
|
||||
SILENT_MODE=false
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
validate_unraid_cmd \
|
||||
"$(command -v openssl 2>/dev/null || echo /usr/bin/openssl)" \
|
||||
"version" "OpenSSL" \
|
||||
"openssl" || warn "openssl not found — SSL cert checks will be skipped"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases all host-specific vars used in this report
|
||||
detect_hosts
|
||||
|
||||
success "Health Digest — profile: $DIGEST_PROFILE"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — report generated but no notification sent"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Profile check — should we send today?
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_DIGEST Profile: $DIGEST_PROFILE"
|
||||
echo "$ICON_DIGEST Digest day: $DIGEST_DAY"
|
||||
echo "$ICON_DIGEST Smart triggers: watchdog=$DIGEST_SMART_ON_WATCHDOG failover=$DIGEST_SMART_ON_FAILOVER cert=$DIGEST_SMART_ON_CERT_WARN bandwidth=$DIGEST_SMART_ON_BANDWIDTH"
|
||||
echo "$ICON_CERT Cert domains: ${CERT_MONITOR_DOMAINS[*]:-none}"
|
||||
echo "$ICON_BANDWIDTH Bandwidth warn: ${BANDWIDTH_WARN_GB}GB"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ── Profile gate — should we send today? ──────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
SHOULD_SEND=false
|
||||
|
||||
case "$DIGEST_PROFILE" in
|
||||
@@ -56,229 +123,206 @@ case "$DIGEST_PROFILE" in
|
||||
TODAY_NAME=$(date '+%A')
|
||||
if [[ "$TODAY_NAME" == "$DIGEST_DAY" ]]; then
|
||||
SHOULD_SEND=true
|
||||
log "Profile: weekly — today is $DIGEST_DAY, will send"
|
||||
log "Profile: weekly — today is $DIGEST_DAY — will send"
|
||||
else
|
||||
info "Profile: weekly — today is $TODAY_NAME, digest day is $DIGEST_DAY — skipping"
|
||||
log "Profile: weekly — today is $TODAY_NAME, digest day is $DIGEST_DAY — skipping"
|
||||
exit 0
|
||||
fi
|
||||
;;
|
||||
smart)
|
||||
log "Profile: smart — will evaluate findings before deciding"
|
||||
SHOULD_SEND=false # determined after gathering data
|
||||
log "Profile: smart — evaluating findings before deciding"
|
||||
SHOULD_SEND=false
|
||||
;;
|
||||
*)
|
||||
warn "Unknown DIGEST_PROFILE: $DIGEST_PROFILE — defaulting to weekly behavior"
|
||||
warn "Unknown DIGEST_PROFILE: $DIGEST_PROFILE — defaulting to weekly"
|
||||
TODAY_NAME=$(date '+%A')
|
||||
[[ "$TODAY_NAME" == "$DIGEST_DAY" ]] && SHOULD_SEND=true
|
||||
;;
|
||||
esac
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# DATA GATHERING
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_DIGEST Gathering System Data ━━━"
|
||||
# ==============================================================================================
|
||||
# ── Data Gathering ────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
FINDINGS=() # notable but not critical
|
||||
ISSUES=() # need attention
|
||||
DIGEST_LINES=() # full report lines
|
||||
|
||||
FINDINGS=() # things worth noting
|
||||
ISSUES=() # things that need attention
|
||||
DIGEST_LINES=() # full report lines
|
||||
|
||||
# ── Failover State ──────────────────────────────────────────────────────────────────────────
|
||||
# ── Failover State ────────────────────────────────────────────────────────────────────────────
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
FAILOVER_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
DIGEST_LINES+=("$ICON_FAILOVER Failover: $FAILOVER_STATE (last change: ${FAILOVER_CHANGE:-unknown})")
|
||||
|
||||
if [[ "$FAILOVER_STATE" != "NORMAL" && -n "$FAILOVER_STATE" ]]; then
|
||||
ISSUES+=("Failover state: $FAILOVER_STATE")
|
||||
[[ "$DIGEST_SMART_ON_FAILOVER" == true ]] && SHOULD_SEND=true
|
||||
if [[ -n "$FAILOVER_STATE" ]]; then
|
||||
DIGEST_LINES+=("$ICON_FAILOVER Failover: $FAILOVER_STATE")
|
||||
if [[ "$FAILOVER_STATE" != "NORMAL" ]]; then
|
||||
ISSUES+=("Failover state: $FAILOVER_STATE")
|
||||
[[ "$DIGEST_SMART_ON_FAILOVER" == true ]] && SHOULD_SEND=true
|
||||
fi
|
||||
fi
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_FAILOVER Failover: state file not found")
|
||||
fi
|
||||
|
||||
# ── Transcode Ramdisk ───────────────────────────────────────────────────────────────────────
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
|
||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
|
||||
RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail | tail -1 | tr -d ' ')
|
||||
RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}")
|
||||
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
||||
DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET")
|
||||
|
||||
# Read weekly transcode stats from daily log if available
|
||||
TRANSCODE_DAILY_LOG="/boot/config/transcode_daily.db"
|
||||
if [[ -f "$TRANSCODE_DAILY_LOG" ]]; then
|
||||
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d')
|
||||
|
||||
# Peak ramdisk usage this week
|
||||
WEEK_PEAK=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||||
'$1 >= cutoff {if ($2 > max) max=$2} END {printf "%.2f", max+0}' \
|
||||
"$TRANSCODE_DAILY_LOG")
|
||||
|
||||
# Total flips this week
|
||||
WEEK_FLIPS=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||||
'$1 >= cutoff {sum+=$3} END {print sum+0}' \
|
||||
"$TRANSCODE_DAILY_LOG")
|
||||
|
||||
# Total files cleaned this week
|
||||
WEEK_FILES=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||||
'$1 >= cutoff {sum+=$6} END {print sum+0}' \
|
||||
"$TRANSCODE_DAILY_LOG")
|
||||
|
||||
# Ram vs SSD session ratio
|
||||
WEEK_RAM=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||||
'$1 >= cutoff {sum+=$4} END {print sum+0}' \
|
||||
"$TRANSCODE_DAILY_LOG")
|
||||
WEEK_SSD=$(awk -F'|' -v cutoff="$WEEK_CUTOFF" \
|
||||
'$1 >= cutoff {sum+=$5} END {print sum+0}' \
|
||||
"$TRANSCODE_DAILY_LOG")
|
||||
|
||||
DIGEST_LINES+=("$ICON_RAM Transcode week: peak ${WEEK_PEAK}GB | flips: ${WEEK_FLIPS} | cleaned: ${WEEK_FILES} files")
|
||||
DIGEST_LINES+=("$ICON_RAM Session storage: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD")
|
||||
|
||||
# Warn if peak is getting close to threshold
|
||||
PEAK_INT=$(printf "%.0f" "$WEEK_PEAK")
|
||||
WARN_INT=$(printf "%.0f" "$RAMDISK_WARN_GB")
|
||||
if [[ "$PEAK_INT" -ge "$WARN_INT" ]]; then
|
||||
DIGEST_LINES+=("$ICON_WARN Transcode peak ${WEEK_PEAK}GB near threshold ${RAMDISK_WARN_GB}GB — consider increasing RAMDISK_SIZE")
|
||||
fi
|
||||
fi
|
||||
# ── Container Skip List ───────────────────────────────────────────────────────────────────────
|
||||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||||
SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ')
|
||||
DIGEST_LINES+=("$ICON_NOT_RUNNING Skip list: $SKIP_COUNT containers — $SKIP_LIST")
|
||||
ISSUES+=("Containers on skip list (manual intervention needed): $SKIP_LIST")
|
||||
SHOULD_SEND=true
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_RAM Transcodes: ramdisk not mounted")
|
||||
ISSUES+=("Ramdisk not mounted")
|
||||
DIGEST_LINES+=("$ICON_RUNNING Skip list: empty ✅")
|
||||
fi
|
||||
|
||||
# ── Container Watchdog Strikes ──────────────────────────────────────────────────────────────
|
||||
# ── Container Watchdog Strikes ────────────────────────────────────────────────────────────────
|
||||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | wc -l)
|
||||
ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -c "." || echo 0)
|
||||
if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then
|
||||
STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
|
||||
DIGEST_LINES+=("$ICON_WATCHDOG Container strikes: $ACTIVE_STRIKES active — $STRIKE_LIST")
|
||||
FINDINGS+=("Container watchdog: $ACTIVE_STRIKES active strikes")
|
||||
[[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_WATCHDOG Container watchdog: no active strikes")
|
||||
DIGEST_LINES+=("$ICON_WATCHDOG Container watchdog: no active strikes ✅")
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── System Watchdog Strikes ─────────────────────────────────────────────────────────────────
|
||||
# ── System Watchdog Strikes ───────────────────────────────────────────────────────────────────
|
||||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||||
SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | wc -l)
|
||||
SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -c "." || echo 0)
|
||||
if [[ "$SYS_STRIKES" -gt 0 ]]; then
|
||||
SYS_STRIKE_LIST=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
|
||||
DIGEST_LINES+=("$ICON_REBOOT_SMART System strikes: $SYS_STRIKES active — $SYS_STRIKE_LIST")
|
||||
FINDINGS+=("System watchdog: $SYS_STRIKES active strikes")
|
||||
[[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_REBOOT_SMART System watchdog: no active strikes")
|
||||
DIGEST_LINES+=("$ICON_REBOOT_SMART System watchdog: no active strikes ✅")
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Container Skip List ─────────────────────────────────────────────────────────────────────
|
||||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||||
SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ')
|
||||
DIGEST_LINES+=("$ICON_NOT_RUNNING Skip list: $SKIP_COUNT containers — $SKIP_LIST")
|
||||
ISSUES+=("Containers on skip list: $SKIP_LIST")
|
||||
SHOULD_SEND=true
|
||||
# ── Transcode Ramdisk ─────────────────────────────────────────────────────────────────────────
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ')
|
||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", ${RAMDISK_USED_KB:-0} / 1048576}")
|
||||
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
||||
DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used → $SYMLINK_TARGET")
|
||||
|
||||
# Weekly transcode stats from TRANSCODE_DAILY_LOG
|
||||
if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]]; then
|
||||
WEEK_CUTOFF=$(date -d "7 days ago" '+%Y-%m-%d')
|
||||
WEEK_PEAK=$(awk -F'|' -v c="$WEEK_CUTOFF" \
|
||||
'$1>=c{if($2>max)max=$2} END{printf "%.2f",max+0}' "$TRANSCODE_DAILY_LOG")
|
||||
WEEK_FLIPS=$(awk -F'|' -v c="$WEEK_CUTOFF" \
|
||||
'$1>=c{sum+=$3} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||||
WEEK_RAM=$(awk -F'|' -v c="$WEEK_CUTOFF" \
|
||||
'$1>=c{sum+=$4} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||||
WEEK_SSD=$(awk -F'|' -v c="$WEEK_CUTOFF" \
|
||||
'$1>=c{sum+=$5} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||||
WEEK_FILES=$(awk -F'|' -v c="$WEEK_CUTOFF" \
|
||||
'$1>=c{sum+=$6} END{print sum+0}' "$TRANSCODE_DAILY_LOG")
|
||||
|
||||
DIGEST_LINES+=("$ICON_RAM Transcode week: peak ${WEEK_PEAK}GB | flips: $WEEK_FLIPS | sessions: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD | cleaned: ${WEEK_FILES} files")
|
||||
|
||||
PEAK_INT=$(printf "%.0f" "$WEEK_PEAK")
|
||||
WARN_INT=$(printf "%.0f" "${RAMDISK_WARN_GB:-6.8}")
|
||||
if [[ "$PEAK_INT" -ge "$WARN_INT" ]]; then
|
||||
DIGEST_LINES+=("$ICON_WARN Peak ${WEEK_PEAK}GB near threshold ${RAMDISK_WARN_GB}GB — consider increasing HOST*_RAMDISK_SIZE")
|
||||
FINDINGS+=("Transcode ramdisk near threshold: ${WEEK_PEAK}GB / ${RAMDISK_WARN_GB}GB")
|
||||
fi
|
||||
fi
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_RUNNING Skip list: empty — all containers healthy")
|
||||
DIGEST_LINES+=("$ICON_RAM Transcodes: ramdisk not mounted")
|
||||
ISSUES+=("Ramdisk not mounted at $RAMDISK_PATH")
|
||||
SHOULD_SEND=true
|
||||
fi
|
||||
|
||||
# ── Bandwidth (yesterday's total) ───────────────────────────────────────────────────────────
|
||||
if [[ -f "$BANDWIDTH_LOG" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
|
||||
# ── Bandwidth ─────────────────────────────────────────────────────────────────────────────────
|
||||
# Updated for new log format: date|time|profile|duration|status|bytes|warn_flag
|
||||
if [[ -f "${BANDWIDTH_LOG:-}" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
|
||||
YESTERDAY=$(date -d "yesterday" '+%Y-%m-%d')
|
||||
YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$3} END{print sum+0}' "$BANDWIDTH_LOG")
|
||||
YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", $YESTERDAY_BYTES / 1073741824}")
|
||||
OVER_WARN=$(awk "BEGIN {print ($YESTERDAY_BYTES > $BANDWIDTH_WARN_GB * 1073741824) ? 1 : 0}")
|
||||
YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$6} END{print sum+0}' \
|
||||
"$BANDWIDTH_LOG")
|
||||
YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", ${YESTERDAY_BYTES:-0} / 1073741824}")
|
||||
YESTERDAY_LARGE=$(awk -F'|' -v d="$YESTERDAY" '$1==d && $7=="LARGE"' \
|
||||
"$BANDWIDTH_LOG" | wc -l)
|
||||
|
||||
if [[ "$OVER_WARN" == "1" ]]; then
|
||||
DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday's transfers: ${YESTERDAY_GB}GB ← exceeded ${BANDWIDTH_WARN_GB}GB threshold")
|
||||
if [[ "${YESTERDAY_LARGE:-0}" -gt 0 ]]; then
|
||||
DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday: ${YESTERDAY_GB}GB — $YESTERDAY_LARGE large transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB")
|
||||
FINDINGS+=("High bandwidth day: ${YESTERDAY_GB}GB transferred")
|
||||
[[ "$DIGEST_SMART_ON_BANDWIDTH" == true ]] && SHOULD_SEND=true
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday's transfers: ${YESTERDAY_GB}GB")
|
||||
DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday: ${YESTERDAY_GB}GB")
|
||||
fi
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_BANDWIDTH Bandwidth: no data yet")
|
||||
fi
|
||||
|
||||
# ── SSL Certificates ────────────────────────────────────────────────────────────────────────
|
||||
# ── SSL Certificates ──────────────────────────────────────────────────────────────────────────
|
||||
if [[ ${#CERT_MONITOR_DOMAINS[@]} -gt 0 ]] && command -v openssl >/dev/null 2>&1; then
|
||||
CERT_ISSUES=()
|
||||
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
||||
[[ -z "$domain" ]] && continue
|
||||
expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \
|
||||
expiry_str=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \
|
||||
-connect "${domain}:443" -servername "$domain" \
|
||||
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||||
|
||||
if [[ -n "$expiry_str" ]]; then
|
||||
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
|
||||
days_remaining=$(( (expiry_epoch - $(date +%s)) / 86400 ))
|
||||
|
||||
if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then
|
||||
if [[ "$days_remaining" -le "${CERT_CRIT_DAYS:-7}" ]]; then
|
||||
CERT_ISSUES+=("$domain: ${days_remaining}d CRITICAL")
|
||||
SHOULD_SEND=true
|
||||
elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then
|
||||
CERT_ISSUES+=("$domain: ${days_remaining}d WARNING")
|
||||
elif [[ "$days_remaining" -le "${CERT_WARN_DAYS:-30}" ]]; then
|
||||
CERT_ISSUES+=("$domain: ${days_remaining}d warning")
|
||||
[[ "$DIGEST_SMART_ON_CERT_WARN" == true ]] && SHOULD_SEND=true
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${#CERT_ISSUES[@]} -gt 0 ]]; then
|
||||
DIGEST_LINES+=("$ICON_CERT Certificates: ${CERT_ISSUES[*]}")
|
||||
FINDINGS+=("Cert issues: ${CERT_ISSUES[*]}")
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_CERT Certificates: all healthy")
|
||||
DIGEST_LINES+=("$ICON_CERT Certificates: all healthy ✅")
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Smart profile final decision ────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# ── Smart profile — exit silently if nothing to report ────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
if [[ "$DIGEST_PROFILE" == "smart" && "$SHOULD_SEND" == false ]]; then
|
||||
info "Profile: smart — no findings worth reporting — skipping notification"
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━"
|
||||
echo "$ICON_SUCCESS Everything looks healthy — no digest sent (smart profile)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
log "Profile: smart — no findings worth reporting — silent exit"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_DIGEST Build and Send Digest ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Build and Send Digest ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_DIGEST Health Digest — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
echo "━━━ $ICON_DIGEST Health Digest ━━━"
|
||||
|
||||
DIGEST_DATE=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
DIGEST_HOST=$(hostname)
|
||||
|
||||
# Build notification message
|
||||
NOTIFY_MSG="Health Digest — $DIGEST_HOST — $DIGEST_DATE"
|
||||
if [[ ${#ISSUES[@]} -gt 0 ]]; then
|
||||
NOTIFY_MSG+=" | Issues: ${ISSUES[*]}"
|
||||
fi
|
||||
if [[ ${#FINDINGS[@]} -gt 0 ]]; then
|
||||
NOTIFY_MSG+=" | Findings: ${FINDINGS[*]}"
|
||||
fi
|
||||
|
||||
# Print full digest to console
|
||||
for line in "${DIGEST_LINES[@]}"; do
|
||||
echo " $line"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━"
|
||||
echo "$ICON_DIGEST Profile: $DIGEST_PROFILE"
|
||||
echo "$ICON_TIME Generated: $DIGEST_DATE"
|
||||
echo "$ICON_ERROR Issues: ${#ISSUES[@]}"
|
||||
echo "$ICON_WARN Findings: ${#FINDINGS[@]}"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_DIGEST Profile: $DIGEST_PROFILE"
|
||||
echo "$ICON_ERROR Issues: ${#ISSUES[@]}"
|
||||
echo "$ICON_WARN Findings: ${#FINDINGS[@]}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Build notification message
|
||||
NOTIFY_MSG="Health Digest — $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
[[ ${#ISSUES[@]} -gt 0 ]] && NOTIFY_MSG+=" | Issues: ${ISSUES[*]}"
|
||||
[[ ${#FINDINGS[@]} -gt 0 ]] && NOTIFY_MSG+=" | Findings: ${FINDINGS[*]}"
|
||||
[[ ${#ISSUES[@]} -eq 0 && ${#FINDINGS[@]} -eq 0 ]] && NOTIFY_MSG+=" | All systems healthy"
|
||||
|
||||
NOTIFY_SEV="normal"
|
||||
[[ ${#ISSUES[@]} -gt 0 ]] && NOTIFY_SEV="warning"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — digest generated but not sent"
|
||||
elif [[ "$SHOULD_SEND" == true ]]; then
|
||||
notify "$NOTIFY_MSG" "Health Digest" "$([[ ${#ISSUES[@]} -gt 0 ]] && echo "warning" || echo "normal")"
|
||||
success "Digest sent"
|
||||
notify "$NOTIFY_MSG" "Health Digest" "$NOTIFY_SEV"
|
||||
log "Digest sent"
|
||||
fi
|
||||
+135
-102
@@ -1,93 +1,133 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- ZFS Memory Snapshot ----------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= ZFS Memory Snapshot ========================================
|
||||
# ==============================================================================================
|
||||
# Weekly ZFS pool health and memory diagnostic report.
|
||||
# Combines ZFS pool status, ARC statistics, memory summary, Docker memory usage
|
||||
# and kernel pressure into a single report. Informational only — no action taken.
|
||||
# system_watchdog.sh handles threshold-based intervention.
|
||||
#
|
||||
# ── WHAT IT REPORTS ───────────────────────────────────────────────────────────────────────────
|
||||
# ZFS pool health — status, state, errors per pool (excluding ignored pools)
|
||||
# ARC statistics — current size, max, utilization %, metadata pressure
|
||||
# Memory status — total/free/available RAM vs thresholds
|
||||
# Docker memory — top ZFS_REPORT_DOCKER_TOP containers by memory usage
|
||||
# Kernel pressure — vmstat snapshot (3 samples)
|
||||
#
|
||||
# ── OUTPUT ────────────────────────────────────────────────────────────────────────────────────
|
||||
# Output goes to both console and ZFS_REPORT_LOG for later review.
|
||||
# In dry-run mode — console only, nothing written to log.
|
||||
# Notifies if any warning thresholds are exceeded.
|
||||
# Silent when all healthy — only problems produce output.
|
||||
#
|
||||
# Pools listed in ZFS_REPORT_IGNORE_POOLS are excluded from health reporting.
|
||||
# Useful for pools expected to run at high usage (docker, cache etc.)
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS → ZFS_REPORT_IGNORE_POOLS.
|
||||
# Each server ignores its own single-disk ZFS array pools — not the peer's.
|
||||
#
|
||||
# All configuration in Master.conf under ZFS Memory Snapshot section.
|
||||
# Supports --dry-run (preview only, no log write) and --status.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# acquire_lock — zpool + docker stats are slow, prevent duplicates
|
||||
# detect_hosts() — correct pool ignore list per host
|
||||
# validate_unraid_cmd — notify validated before use
|
||||
# DOCKER_TIMEOUT — docker stats protected against hung daemon
|
||||
# ZFS not available — skips pool and ARC sections gracefully
|
||||
# Docker not available — skips container section gracefully
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_ZFS_REPORT_IGNORE_POOLS — pools excluded from health reporting
|
||||
# Aliased by detect_hosts() — script uses ZFS_REPORT_IGNORE_POOLS
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# ZFS_REPORT_LOG — log file path for weekly reports
|
||||
# ZFS_REPORT_ARC_WARN_PCT — warn if ARC using more than this % of max
|
||||
# ZFS_REPORT_FREE_WARN_GB — warn if less than this GB free RAM
|
||||
# ZFS_REPORT_AVAIL_WARN_GB — warn if less than this GB available RAM
|
||||
# ZFS_REPORT_DOCKER_TOP — how many top Docker containers to show
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# zfs_memory_snapshot.sh — normal report (writes to log)
|
||||
# zfs_memory_snapshot.sh --dry-run — console only, no log write
|
||||
# zfs_memory_snapshot.sh --log — verbose output
|
||||
# zfs_memory_snapshot.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
# Monitor/report script — output is the point
|
||||
SILENT_MODE=false
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
validate_unraid_cmd \
|
||||
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
||||
"" "" \
|
||||
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
acquire_lock
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_ZFS_REPORT_IGNORE_POOLS
|
||||
detect_hosts
|
||||
|
||||
# Build ignore pool lookup map — O(1) check per pool
|
||||
declare -A IGNORE_POOL_MAP
|
||||
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]:-}"; do
|
||||
[[ -n "$pool" ]] && IGNORE_POOL_MAP["$pool"]=1
|
||||
done
|
||||
|
||||
log "Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
log "Ignoring pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
|
||||
# Tee output to log file unless dry run
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
mkdir -p "$(dirname "$ZFS_REPORT_LOG")"
|
||||
exec > >(tee -a "$ZFS_REPORT_LOG") 2>&1
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — output will not be written to log"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
# Build ignore list for quick lookup
|
||||
declare -A IGNORE_POOL_MAP
|
||||
for pool in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do
|
||||
[[ -n "$pool" ]] && IGNORE_POOL_MAP["$pool"]=1
|
||||
done
|
||||
|
||||
if [[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]]; then
|
||||
info "Ignoring pools: ${ZFS_REPORT_IGNORE_POOLS[*]}"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_ZFS Log file: $ZFS_REPORT_LOG"
|
||||
echo "$ICON_ZFS ARC warn: ${ZFS_REPORT_ARC_WARN_PCT}%"
|
||||
echo "$ICON_MEM Free RAM warn: ${ZFS_REPORT_FREE_WARN_GB}GB"
|
||||
echo "$ICON_MEM Avail RAM warn: ${ZFS_REPORT_AVAIL_WARN_GB}GB"
|
||||
echo "$ICON_CONTAINERS Docker top: $ZFS_REPORT_DOCKER_TOP"
|
||||
echo "$ICON_ZFS Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_ZFS Log file: $ZFS_REPORT_LOG"
|
||||
echo "$ICON_ZFS ARC warn: ${ZFS_REPORT_ARC_WARN_PCT}%"
|
||||
echo "$ICON_MEM Free RAM warn: ${ZFS_REPORT_FREE_WARN_GB}GB"
|
||||
echo "$ICON_MEM Avail warn: ${ZFS_REPORT_AVAIL_WARN_GB}GB"
|
||||
echo "$ICON_CONTAINERS Docker top: $ZFS_REPORT_DOCKER_TOP"
|
||||
echo "$ICON_ZFS Ignore pools: ${ZFS_REPORT_IGNORE_POOLS[*]:-none}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — output will not be written to log"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Tracking
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Report ━━━
|
||||
# ==============================================================================================
|
||||
WARNINGS=()
|
||||
START=$(date +%s)
|
||||
DATE=$(date +"%Y-%m-%d %H:%M:%S")
|
||||
DATE=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " $ICON_ZFS ZFS WEEKLY HEALTH REPORT — $DATE"
|
||||
echo " $ICON_HOST Host: $(hostname)"
|
||||
echo " $ICON_HOST $MY_ID — $LOCAL_SERVER_NAME"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_ZFS ZFS Pool Health ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── ZFS Pool Health ───────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_ZFS ZFS Pool Health ━━━"
|
||||
|
||||
@@ -95,13 +135,11 @@ if ! command -v zpool >/dev/null 2>&1; then
|
||||
warn "ZFS not available on this system — skipping pool checks"
|
||||
else
|
||||
# Pool status — filtered to key lines, ignoring specified pools
|
||||
info "Pool status:"
|
||||
CURRENT_POOL=""
|
||||
while IFS= read -r line; do
|
||||
# Extract pool name from "pool: poolname" lines
|
||||
if [[ "$line" =~ ^[[:space:]]*pool:[[:space:]]*(.+) ]]; then
|
||||
CURRENT_POOL="${BASH_REMATCH[1]// /}"
|
||||
fi
|
||||
# Skip lines belonging to ignored pools
|
||||
[[ -n "${IGNORE_POOL_MAP[$CURRENT_POOL]:-}" ]] && continue
|
||||
echo " $line"
|
||||
done < <(zpool status 2>/dev/null | grep -E "pool:|state:|status:|errors:|scan:")
|
||||
@@ -109,41 +147,36 @@ else
|
||||
echo ""
|
||||
|
||||
# Pool list — filter out ignored pools
|
||||
info "Pool overview:"
|
||||
zpool list 2>/dev/null | while IFS= read -r line; do
|
||||
# Always show header line
|
||||
if [[ "$line" == NAME* ]]; then
|
||||
echo " $line"
|
||||
continue
|
||||
fi
|
||||
# Extract pool name (first field)
|
||||
pool_name=$(echo "$line" | awk '{print $1}')
|
||||
[[ -n "${IGNORE_POOL_MAP[$pool_name]:-}" ]] && continue
|
||||
echo " $line"
|
||||
done
|
||||
|
||||
# Check for unhealthy pools — excluding ignored ones
|
||||
UNHEALTHY=$(zpool list -H -o name,health 2>/dev/null | while IFS=$'\t' read -r name health; do
|
||||
[[ -n "${IGNORE_POOL_MAP[$name]:-}" ]] && continue
|
||||
[[ "$health" != "ONLINE" ]] && echo "$name: $health"
|
||||
done)
|
||||
# Check for unhealthy non-ignored pools
|
||||
UNHEALTHY=$(zpool list -H -o name,health 2>/dev/null | \
|
||||
while IFS=$'\t' read -r name health; do
|
||||
[[ -n "${IGNORE_POOL_MAP[$name]:-}" ]] && continue
|
||||
[[ "$health" != "ONLINE" ]] && echo "$name: $health"
|
||||
done)
|
||||
|
||||
if [[ -n "$UNHEALTHY" ]]; then
|
||||
error "One or more ZFS pools are NOT ONLINE: $UNHEALTHY"
|
||||
WARNINGS+=("ZFS pool unhealthy: $UNHEALTHY")
|
||||
else
|
||||
success "All monitored ZFS pools are ONLINE"
|
||||
log "All monitored ZFS pools are ONLINE ✅"
|
||||
fi
|
||||
|
||||
# Show ignored pools
|
||||
if [[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]]; then
|
||||
info "Ignored pools (not reported): ${ZFS_REPORT_IGNORE_POOLS[*]}"
|
||||
log "Ignored pools: ${ZFS_REPORT_IGNORE_POOLS[*]}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_ZFS ARC Statistics ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── ARC Statistics ────────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_ZFS ARC Statistics ━━━"
|
||||
|
||||
@@ -170,14 +203,16 @@ else
|
||||
warn "ARC utilization ${ARC_PCT}% — above ${ZFS_REPORT_ARC_WARN_PCT}% threshold"
|
||||
WARNINGS+=("ARC high: ${ARC_PCT}%")
|
||||
else
|
||||
success "ARC utilization ${ARC_PCT}% — within threshold (${ZFS_REPORT_ARC_WARN_PCT}%)"
|
||||
log "ARC utilization ${ARC_PCT}% — within threshold ✅"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
info "Metadata pressure:"
|
||||
META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||||
META_MFU_GHOST=$(awk '/^mfu_ghost_metadata / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||||
META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||||
META_MRU_GHOST=$(awk '/^mru_ghost_metadata / {print $3}' \
|
||||
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||||
META_MFU_GHOST=$(awk '/^mfu_ghost_metadata / {print $3}' \
|
||||
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||||
META_MISSES=$(awk '/^demand_metadata_misses / {print $3}' \
|
||||
/proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0)
|
||||
|
||||
MRU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MRU_GHOST / 1073741824}")
|
||||
MFU_GB=$(awk "BEGIN {printf \"%.2f\", $META_MFU_GHOST / 1073741824}")
|
||||
@@ -187,17 +222,15 @@ else
|
||||
echo " $ICON_ZFS Metadata Misses: ${META_MISSES}"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_MEM Memory Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── Memory Status ─────────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_MEM Memory Status ━━━"
|
||||
|
||||
FREE_HUMAN=$(free -h | awk '/Mem:/ {print $4}')
|
||||
FREE_HUMAN=$(free -h | awk '/Mem:/ {print $4}')
|
||||
AVAIL_HUMAN=$(free -h | awk '/Mem:/ {print $7}')
|
||||
TOTAL_HUMAN=$(free -h | awk '/Mem:/ {print $2}')
|
||||
FREE_GB=$(free -g | awk '/Mem:/ {print $4}')
|
||||
AVAIL_GB=$(free -g | awk '/Mem:/ {print $7}')
|
||||
FREE_GB=$(free -g | awk '/Mem:/ {print $4}')
|
||||
AVAIL_GB=$(free -g | awk '/Mem:/ {print $7}')
|
||||
|
||||
echo " $ICON_MEM Total RAM: $TOTAL_HUMAN"
|
||||
echo " $ICON_MEM Free RAM: $FREE_HUMAN"
|
||||
@@ -207,42 +240,38 @@ if [[ "$FREE_GB" -lt "$ZFS_REPORT_FREE_WARN_GB" ]]; then
|
||||
warn "Free RAM ${FREE_HUMAN} — below ${ZFS_REPORT_FREE_WARN_GB}GB threshold"
|
||||
WARNINGS+=("Low free RAM: ${FREE_HUMAN}")
|
||||
else
|
||||
success "Free RAM ${FREE_HUMAN} — within threshold (${ZFS_REPORT_FREE_WARN_GB}GB)"
|
||||
log "Free RAM ${FREE_HUMAN} — within threshold ✅"
|
||||
fi
|
||||
|
||||
if [[ "$AVAIL_GB" -lt "$ZFS_REPORT_AVAIL_WARN_GB" ]]; then
|
||||
warn "Available RAM ${AVAIL_HUMAN} — below ${ZFS_REPORT_AVAIL_WARN_GB}GB threshold"
|
||||
WARNINGS+=("Low available RAM: ${AVAIL_HUMAN}")
|
||||
else
|
||||
success "Available RAM ${AVAIL_HUMAN} — within threshold (${ZFS_REPORT_AVAIL_WARN_GB}GB)"
|
||||
log "Available RAM ${AVAIL_HUMAN} — within threshold ✅"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CONTAINERS Docker Memory ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── Docker Memory ─────────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Top $ZFS_REPORT_DOCKER_TOP Docker Memory Users ━━━"
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
warn "Docker not available — skipping container memory section"
|
||||
else
|
||||
docker stats --no-stream \
|
||||
timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
|
||||
--format "table {{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}" \
|
||||
2>/dev/null | head -n $(( ZFS_REPORT_DOCKER_TOP + 1 )) | while IFS= read -r line; do
|
||||
echo " $line"
|
||||
done
|
||||
2>/dev/null | head -n $(( ZFS_REPORT_DOCKER_TOP + 1 )) | \
|
||||
while IFS= read -r line; do
|
||||
echo " $line"
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Kernel Pressure ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── Kernel Pressure ───────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Kernel Pressure ━━━"
|
||||
|
||||
if ! command -v vmstat >/dev/null 2>&1; then
|
||||
warn "vmstat not available — skipping kernel pressure section"
|
||||
else
|
||||
info "vmstat snapshot (3 samples):"
|
||||
vmstat 1 3 2>/dev/null | while IFS= read -r line; do
|
||||
echo " $line"
|
||||
done
|
||||
@@ -250,23 +279,27 @@ fi
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── Summary ───────────────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ZFS REPORT SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_ZFS Log: $ZFS_REPORT_LOG"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo "$ICON_ZFS Log: $ZFS_REPORT_LOG"
|
||||
[[ ${#ZFS_REPORT_IGNORE_POOLS[@]} -gt 0 ]] && \
|
||||
echo "$ICON_ZFS Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]}"
|
||||
log "Ignored: ${ZFS_REPORT_IGNORE_POOLS[*]}"
|
||||
echo ""
|
||||
|
||||
if [[ ${#WARNINGS[@]} -eq 0 ]]; then
|
||||
success "Report complete — no warnings"
|
||||
log "$ICON_DONE All checks within thresholds ✅"
|
||||
else
|
||||
echo "$ICON_WARN Warnings: ${#WARNINGS[@]}"
|
||||
for w in "${WARNINGS[@]}"; do
|
||||
echo " $ICON_WARN $w"
|
||||
done
|
||||
notify "ZFS weekly report on $(hostname) — ${#WARNINGS[@]} warning(s): ${WARNINGS[*]}" "ZFS Report" "warning"
|
||||
notify "ZFS weekly report on $(hostname) — ${#WARNINGS[@]} warning(s): ${WARNINGS[*]}" \
|
||||
"ZFS Report" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#WARNINGS[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user