did way to much,,,,,, mostly added monitors, but almost every file was edited in some way
This commit is contained in:
@@ -0,0 +1,187 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Backup Verify ----------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Verifies the rsync mirror is healthy by comparing random file samples between
|
||||
# local and remote servers using MD5 checksums.
|
||||
#
|
||||
# Randomly samples BACKUP_VERIFY_SAMPLE files per share, computes checksums locally,
|
||||
# then computes the same checksums on the remote via SSH and compares results.
|
||||
#
|
||||
# Results per file:
|
||||
# MATCH — checksums identical, file is correctly mirrored
|
||||
# MISMATCH — file exists on both but checksums differ — sync may have failed
|
||||
# MISSING — file exists locally but not on remote — not yet synced or deleted
|
||||
#
|
||||
# Silent when all files match. Notifies on any mismatch or missing file.
|
||||
# Uses existing SSH keys — no additional configuration needed beyond share list.
|
||||
#
|
||||
# All configuration in Master.conf under Backup Verify section.
|
||||
# Supports --dry-run to show what would be checked without running checksums.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
detect_hosts
|
||||
resolve_remote_ip
|
||||
|
||||
# Use BACKUP_VERIFY_SHARES if defined, fall back to DAILY_SYNC_SHARES
|
||||
if [[ ${#BACKUP_VERIFY_SHARES[@]} -gt 0 ]]; then
|
||||
VERIFY_SHARES=("${BACKUP_VERIFY_SHARES[@]}")
|
||||
info "Using BACKUP_VERIFY_SHARES (${#VERIFY_SHARES[@]} shares)"
|
||||
else
|
||||
VERIFY_SHARES=("${DAILY_SYNC_SHARES[@]}")
|
||||
info "BACKUP_VERIFY_SHARES not set — using DAILY_SYNC_SHARES (${#VERIFY_SHARES[@]} shares)"
|
||||
fi
|
||||
|
||||
if [[ ${#VERIFY_SHARES[@]} -eq 0 ]]; then
|
||||
warn "No shares configured — nothing to verify"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_VERIFY Shares: ${#VERIFY_SHARES[@]}"
|
||||
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share"
|
||||
echo "$ICON_VERIFY Min size: $BACKUP_VERIFY_MIN_SIZE"
|
||||
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing sample selection only, no checksums computed"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_VERIFY Backup Verification ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_VERIFY Backup Verification — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME ($REMOTE_SERVER)"
|
||||
echo "$ICON_VERIFY Sample: $BACKUP_VERIFY_SAMPLE files per share (min size: $BACKUP_VERIFY_MIN_SIZE)"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
TOTAL_CHECKED=0
|
||||
TOTAL_MATCH=0
|
||||
TOTAL_MISMATCH=0
|
||||
TOTAL_MISSING=0
|
||||
SHARES_WITH_ISSUES=()
|
||||
|
||||
for share in "${VERIFY_SHARES[@]}"; do
|
||||
SHARE_NAME=$(basename "$share")
|
||||
echo "━━━ $ICON_VERIFY $SHARE_NAME ━━━"
|
||||
|
||||
if [[ ! -d "$share" ]]; then
|
||||
warn "$SHARE_NAME not found locally — skipping"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
# Find files above minimum size and randomly sample
|
||||
SAMPLE_FILES=$(find "$share" -type f -size +"$BACKUP_VERIFY_MIN_SIZE" 2>/dev/null | \
|
||||
shuf | head -n "$BACKUP_VERIFY_SAMPLE")
|
||||
|
||||
SAMPLE_COUNT=$(echo "$SAMPLE_FILES" | grep -c "." 2>/dev/null || echo 0)
|
||||
|
||||
if [[ "$SAMPLE_COUNT" -eq 0 ]]; then
|
||||
info "No files found above $BACKUP_VERIFY_MIN_SIZE — skipping"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
info "Sampled $SAMPLE_COUNT files"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$SAMPLE_FILES" | while IFS= read -r f; do
|
||||
warn "DRY RUN — would check: $(basename "$f")"
|
||||
done
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
SHARE_MISMATCH=0
|
||||
SHARE_MISSING=0
|
||||
SHARE_MATCH=0
|
||||
|
||||
while IFS= read -r local_file; do
|
||||
[[ -z "$local_file" ]] && continue
|
||||
|
||||
# Compute local checksum
|
||||
local_md5=$(md5sum "$local_file" 2>/dev/null | awk '{print $1}')
|
||||
|
||||
if [[ -z "$local_md5" ]]; then
|
||||
warn "Could not checksum: $local_file — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Compute remote checksum via SSH
|
||||
remote_md5=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \
|
||||
"md5sum '$local_file' 2>/dev/null | awk '{print \$1}'" 2>/dev/null)
|
||||
|
||||
((TOTAL_CHECKED++))
|
||||
|
||||
if [[ -z "$remote_md5" ]]; then
|
||||
warn "$ICON_ERROR MISSING: $(basename "$local_file")"
|
||||
((SHARE_MISSING++))
|
||||
((TOTAL_MISSING++))
|
||||
elif [[ "$local_md5" == "$remote_md5" ]]; then
|
||||
log "MATCH: $(basename "$local_file")"
|
||||
((SHARE_MATCH++))
|
||||
((TOTAL_MATCH++))
|
||||
else
|
||||
error "$ICON_ERROR MISMATCH: $(basename "$local_file")"
|
||||
((SHARE_MISMATCH++))
|
||||
((TOTAL_MISMATCH++))
|
||||
fi
|
||||
|
||||
done <<< "$SAMPLE_FILES"
|
||||
|
||||
echo " $ICON_SUCCESS Match: $SHARE_MATCH $ICON_WARN Missing: $SHARE_MISSING $ICON_ERROR Mismatch: $SHARE_MISMATCH"
|
||||
|
||||
if [[ "$SHARE_MISMATCH" -gt 0 || "$SHARE_MISSING" -gt 0 ]]; then
|
||||
SHARES_WITH_ISSUES+=("$SHARE_NAME")
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo "━━━━━ $ICON_SUMMARY BACKUP VERIFY SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Remote: $REMOTE_SERVER_NAME"
|
||||
echo "$ICON_VERIFY Checked: $TOTAL_CHECKED files"
|
||||
echo "$ICON_SUCCESS Match: $TOTAL_MATCH"
|
||||
echo "$ICON_WARN Missing: $TOTAL_MISSING"
|
||||
echo "$ICON_ERROR Mismatch: $TOTAL_MISMATCH"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no checksums computed"
|
||||
elif [[ "$TOTAL_MISMATCH" -gt 0 || "$TOTAL_MISSING" -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ISSUES FOUND — ${#SHARES_WITH_ISSUES[@]} share(s) need attention"
|
||||
notify "Backup verify failed on $(hostname) → $REMOTE_SERVER_NAME — mismatches: $TOTAL_MISMATCH missing: $TOTAL_MISSING — shares: ${SHARES_WITH_ISSUES[*]}" "Backup Verify" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL FILES MATCH"
|
||||
notify "Backup verify passed on $(hostname) → $REMOTE_SERVER_NAME — $TOTAL_CHECKED files checked across ${#VERIFY_SHARES[@]} shares" "Backup Verify" "normal"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -0,0 +1,152 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Bandwidth Monitor ------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Logs daily rsync transfer totals and generates weekly summary reports.
|
||||
# Designed for minimal flash drive impact — one bounded write per day.
|
||||
#
|
||||
# Two modes:
|
||||
# --log-transfer "profile" bytes duration — called by rsync.sh after each sync
|
||||
# appends one line, trims old entries
|
||||
# --report — generates weekly summary from log
|
||||
# (no args) — generates summary report
|
||||
#
|
||||
# Log format (one line per transfer):
|
||||
# YYYY-MM-DD|profile|bytes|duration_seconds
|
||||
#
|
||||
# Log file stays bounded to BANDWIDTH_LOG_RETENTION days — old entries trimmed on write.
|
||||
# Minimal writes: one append per rsync run, one trim per append.
|
||||
#
|
||||
# All configuration in Master.conf under Bandwidth Monitor section.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# Check for log-transfer mode
|
||||
LOG_TRANSFER_MODE=false
|
||||
REPORT_MODE=false
|
||||
TRANSFER_PROFILE=""
|
||||
TRANSFER_BYTES=0
|
||||
TRANSFER_DURATION=0
|
||||
|
||||
for arg in "${PARSED_ARGS[@]}"; do
|
||||
case "$arg" in
|
||||
--log-transfer) LOG_TRANSFER_MODE=true ;;
|
||||
--report) REPORT_MODE=true ;;
|
||||
*)
|
||||
if [[ "$LOG_TRANSFER_MODE" == true && -z "$TRANSFER_PROFILE" ]]; then
|
||||
TRANSFER_PROFILE="$arg"
|
||||
elif [[ "$LOG_TRANSFER_MODE" == true && "$TRANSFER_BYTES" -eq 0 ]]; then
|
||||
TRANSFER_BYTES="$arg"
|
||||
elif [[ "$LOG_TRANSFER_MODE" == true ]]; then
|
||||
TRANSFER_DURATION="$arg"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Ensure log file exists
|
||||
touch "$BANDWIDTH_LOG" 2>/dev/null || {
|
||||
error "Cannot create bandwidth log: $BANDWIDTH_LOG"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# LOG TRANSFER MODE
|
||||
# Called by rsync.sh after each successful sync — appends one line and trims old entries
|
||||
# Usage: bandwidth_monitor.sh --log-transfer "profile" bytes duration
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$LOG_TRANSFER_MODE" == true ]]; then
|
||||
TODAY=$(date '+%Y-%m-%d')
|
||||
echo "${TODAY}|${TRANSFER_PROFILE}|${TRANSFER_BYTES}|${TRANSFER_DURATION}" >> "$BANDWIDTH_LOG"
|
||||
log "Logged transfer: $TRANSFER_PROFILE — $TRANSFER_BYTES bytes in ${TRANSFER_DURATION}s"
|
||||
|
||||
# Trim entries older than retention period — keeps file bounded
|
||||
CUTOFF=$(date -d "${BANDWIDTH_LOG_RETENTION} days ago" '+%Y-%m-%d')
|
||||
TEMP_FILE="${BANDWIDTH_LOG}.tmp"
|
||||
awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' "$BANDWIDTH_LOG" > "$TEMP_FILE"
|
||||
mv "$TEMP_FILE" "$BANDWIDTH_LOG"
|
||||
log "Log trimmed — keeping entries from $CUTOFF onwards"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REPORT MODE — generate summary from log
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_BANDWIDTH Bandwidth Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
|
||||
if [[ ! -s "$BANDWIDTH_LOG" ]]; then
|
||||
warn "No bandwidth data yet — log is empty"
|
||||
warn "Data accumulates as rsync jobs run"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
START=$(date +%s)
|
||||
|
||||
# Calculate date range in log
|
||||
OLDEST=$(awk -F'|' 'NR==1{print $1}' "$BANDWIDTH_LOG")
|
||||
NEWEST=$(awk -F'|' 'END{print $1}' "$BANDWIDTH_LOG")
|
||||
ENTRY_COUNT=$(wc -l < "$BANDWIDTH_LOG")
|
||||
|
||||
info "Log covers: $OLDEST → $NEWEST ($ENTRY_COUNT entries)"
|
||||
echo ""
|
||||
|
||||
# Total bytes transferred
|
||||
TOTAL_BYTES=$(awk -F'|' '{sum += $3} END {print sum+0}' "$BANDWIDTH_LOG")
|
||||
TOTAL_GB=$(awk "BEGIN {printf \"%.2f\", $TOTAL_BYTES / 1073741824}")
|
||||
|
||||
# Per-profile breakdown
|
||||
echo "━━━ $ICON_BANDWIDTH Per-Profile Totals ━━━"
|
||||
awk -F'|' '{
|
||||
bytes[$2] += $3
|
||||
runs[$2]++
|
||||
duration[$2] += $4
|
||||
}
|
||||
END {
|
||||
for (profile in bytes) {
|
||||
gb = bytes[profile] / 1073741824
|
||||
printf " %-20s %6.2f GB (%d runs)\n", profile, gb, runs[profile]
|
||||
}
|
||||
}' "$BANDWIDTH_LOG" | sort -k3 -rn
|
||||
|
||||
echo ""
|
||||
|
||||
# Daily totals for the last 7 days
|
||||
echo "━━━ $ICON_BANDWIDTH Last 7 Days ━━━"
|
||||
for i in 6 5 4 3 2 1 0; do
|
||||
day=$(date -d "$i days ago" '+%Y-%m-%d')
|
||||
day_bytes=$(awk -F'|' -v d="$day" '$1==d{sum+=$3} END{print sum+0}' "$BANDWIDTH_LOG")
|
||||
day_gb=$(awk "BEGIN {printf \"%.2f\", $day_bytes / 1073741824}")
|
||||
|
||||
# Flag days that exceeded warning threshold
|
||||
over_warn=$(awk "BEGIN {print ($day_bytes > $BANDWIDTH_WARN_GB * 1073741824) ? 1 : 0}")
|
||||
if [[ "$over_warn" == "1" ]]; then
|
||||
echo " $ICON_WARN $day ${day_gb} GB ← exceeded ${BANDWIDTH_WARN_GB}GB threshold"
|
||||
else
|
||||
echo " $ICON_TIME $day ${day_gb} GB"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_SUMMARY Totals ━━━"
|
||||
echo " $ICON_BANDWIDTH Total transferred: ${TOTAL_GB} GB"
|
||||
echo " $ICON_TIME Log period: $OLDEST → $NEWEST"
|
||||
echo " $ICON_GEAR Retention: ${BANDWIDTH_LOG_RETENTION} days"
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY BANDWIDTH SUMMARY ━━━━━"
|
||||
echo "$ICON_BANDWIDTH Total: ${TOTAL_GB} GB"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
notify "Bandwidth report on $(hostname) — ${TOTAL_GB}GB transferred (${OLDEST} to ${NEWEST})" "Bandwidth Monitor" "normal"
|
||||
@@ -0,0 +1,182 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Certificate Monitor ----------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Monitors SSL certificate expiry for all configured domains by connecting directly
|
||||
# via openssl — no dependency on NPM or any other service. Reads the actual certificate
|
||||
# the server is presenting to the outside world.
|
||||
#
|
||||
# This approach catches real-world cert issues that API-based checks miss:
|
||||
# - Cert renewed but server not reloaded
|
||||
# - Wrong cert being served
|
||||
# - Cert chain issues
|
||||
#
|
||||
# Each domain and subdomain is a separate entry — they have independent certs.
|
||||
# Silent when all certs are healthy. Notifies when any approach warning threshold.
|
||||
# Notifications batched per severity — one message for warnings, one for criticals.
|
||||
#
|
||||
# All configuration in Master.conf under Certificate Monitor section.
|
||||
# Supports --dry-run to check certs and show results without sending notifications.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if ! command -v openssl >/dev/null 2>&1; then
|
||||
error "openssl not found — required for certificate checks"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "openssl available"
|
||||
|
||||
if [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then
|
||||
warn "CERT_MONITOR_DOMAINS is empty in Master.conf — add your domains to enable monitoring"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
info "$ICON_CERT Domains to check: ${#CERT_MONITOR_DOMAINS[@]}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_CERT Domains: ${CERT_MONITOR_DOMAINS[*]}"
|
||||
echo "$ICON_WARN Warn at: ${CERT_WARN_DAYS} days remaining"
|
||||
echo "$ICON_ERROR Crit at: ${CERT_CRIT_DAYS} days remaining"
|
||||
echo "$ICON_TIME Timeout: ${CERT_TIMEOUT}s per domain"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — results shown but no notifications sent"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CERT CHECK FUNCTION
|
||||
# Connects to domain:443 via openssl, extracts expiry date, calculates days remaining.
|
||||
# Returns 0=healthy 1=warning 2=critical 3=failed
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_cert() {
|
||||
local domain="$1"
|
||||
local port="${2:-443}"
|
||||
|
||||
local expiry_str
|
||||
expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \
|
||||
-connect "${domain}:${port}" \
|
||||
-servername "$domain" \
|
||||
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||||
|
||||
if [[ -z "$expiry_str" ]]; then
|
||||
error "$ICON_CERT $domain — could not retrieve certificate"
|
||||
return 3
|
||||
fi
|
||||
|
||||
local expiry_epoch
|
||||
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
|
||||
|
||||
if [[ -z "$expiry_epoch" ]]; then
|
||||
error "$ICON_CERT $domain — could not parse expiry date: $expiry_str"
|
||||
return 3
|
||||
fi
|
||||
|
||||
local now days_remaining expiry_display
|
||||
now=$(date +%s)
|
||||
days_remaining=$(( (expiry_epoch - now) / 86400 ))
|
||||
expiry_display=$(date -d "$expiry_str" '+%Y-%m-%d' 2>/dev/null)
|
||||
|
||||
if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then
|
||||
error "$ICON_CERT $domain — CRITICAL: ${days_remaining} days remaining (expires $expiry_display)"
|
||||
return 2
|
||||
elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then
|
||||
warn "$ICON_CERT $domain — WARNING: ${days_remaining} days remaining (expires $expiry_display)"
|
||||
return 1
|
||||
else
|
||||
success "$ICON_CERT $domain — OK: ${days_remaining} days remaining (expires $expiry_display)"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CERT Certificate Monitor ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CERT Certificate Monitor — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_WARN Warn threshold: ${CERT_WARN_DAYS} days"
|
||||
echo "$ICON_ERROR Crit threshold: ${CERT_CRIT_DAYS} days"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
HEALTHY=()
|
||||
WARNING=()
|
||||
CRITICAL=()
|
||||
FAILED=()
|
||||
declare -A DOMAIN_STATUS
|
||||
|
||||
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
||||
[[ -z "$domain" ]] && continue
|
||||
echo "━━━ $ICON_CERT $domain ━━━"
|
||||
check_cert "$domain"
|
||||
result=$?
|
||||
case $result in
|
||||
0) HEALTHY+=("$domain"); DOMAIN_STATUS["$domain"]="OK" ;;
|
||||
1) WARNING+=("$domain"); DOMAIN_STATUS["$domain"]="WARN" ;;
|
||||
2) CRITICAL+=("$domain"); DOMAIN_STATUS["$domain"]="CRIT" ;;
|
||||
3) FAILED+=("$domain"); DOMAIN_STATUS["$domain"]="FAIL" ;;
|
||||
esac
|
||||
echo ""
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
[[ ${#CRITICAL[@]} -gt 0 ]] && \
|
||||
notify "Certificate CRITICAL on $(hostname) — expiring within ${CERT_CRIT_DAYS} days: ${CRITICAL[*]}" "Certificate Monitor" "warning"
|
||||
[[ ${#WARNING[@]} -gt 0 ]] && \
|
||||
notify "Certificate WARNING on $(hostname) — expiring within ${CERT_WARN_DAYS} days: ${WARNING[*]}" "Certificate Monitor" "warning"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && \
|
||||
notify "Certificate check FAILED on $(hostname) — could not reach: ${FAILED[*]}" "Certificate Monitor" "warning"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo "━━━━━ $ICON_SUMMARY CERTIFICATE MONITOR SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo ""
|
||||
echo " $ICON_SUCCESS Healthy: ${#HEALTHY[@]} $ICON_WARN Warning: ${#WARNING[@]} $ICON_ERROR Critical: ${#CRITICAL[@]} Failed: ${#FAILED[@]}"
|
||||
echo ""
|
||||
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
||||
[[ -z "$domain" ]] && continue
|
||||
case "${DOMAIN_STATUS[$domain]:-UNKN}" in
|
||||
OK) echo " $ICON_SUCCESS $domain" ;;
|
||||
WARN) echo " $ICON_WARN $domain" ;;
|
||||
CRIT) echo " $ICON_ERROR $domain" ;;
|
||||
FAIL) echo " $ICON_ERROR $domain (unreachable)" ;;
|
||||
esac
|
||||
done
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no notifications sent"
|
||||
elif [[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: ACTION REQUIRED"
|
||||
elif [[ ${#WARNING[@]} -gt 0 ]]; then
|
||||
echo "$ICON_WARN Status: WARNINGS — renewal recommended"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL CERTS HEALTHY"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#CRITICAL[@]} -gt 0 || ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -0,0 +1,249 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- SMART Health Monitor ---------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Checks SMART health attributes for all drives on the system.
|
||||
# Reads data live from each drive via smartctl — no persistent writes.
|
||||
#
|
||||
# Monitored attributes:
|
||||
# Reallocated_Sector_Ct — bad sectors remapped — any > 0 is concerning
|
||||
# Current_Pending_Sector — sectors waiting for reallocation — any > 0 is concerning
|
||||
# Offline_Uncorrectable — sectors that could not be corrected — any > 0 is critical
|
||||
# Temperature_Celsius — drive temperature vs SMART_TEMP_WARN / SMART_TEMP_CRIT
|
||||
# Power_On_Hours — informational — drive age estimation
|
||||
# SMART overall status — pass/fail per drive
|
||||
#
|
||||
# Discovers drives automatically — no configuration needed for drive list.
|
||||
# SMART_IGNORE_DRIVES allows skipping specific drives (e.g. USB flash drives).
|
||||
#
|
||||
# All configuration in Master.conf under SMART Health section.
|
||||
# Supports --dry-run to show which drives would be checked without running smartctl.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
if ! command -v smartctl >/dev/null 2>&1; then
|
||||
error "smartctl not found — install smartmontools"
|
||||
notify "SMART health check failed on $(hostname) — smartmontools not installed" "SMART Health" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "smartctl available"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_SMART Temp warn: ${SMART_TEMP_WARN}°C"
|
||||
echo "$ICON_SMART Temp crit: ${SMART_TEMP_CRIT}°C"
|
||||
echo "$ICON_SMART Ignore drives: ${SMART_IGNORE_DRIVES[*]:-none}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo ""
|
||||
|
||||
# Show drives that would be checked
|
||||
echo "━━━ Discovered Drives ━━━"
|
||||
for drive in /dev/sd? /dev/nvme?; do
|
||||
[[ ! -e "$drive" ]] && continue
|
||||
drive_name=$(basename "$drive")
|
||||
ignored=false
|
||||
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
|
||||
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
|
||||
done
|
||||
if [[ "$ignored" == true ]]; then
|
||||
echo " $ICON_WARN $drive — ignored"
|
||||
else
|
||||
echo " $ICON_SMART $drive — would check"
|
||||
fi
|
||||
done
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — showing drive list only, no SMART data read"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# HELPER — extract SMART attribute value
|
||||
# Usage: get_smart_attr "/dev/sda" "Reallocated_Sector_Ct"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
get_smart_attr() {
|
||||
local drive="$1" attr="$2"
|
||||
smartctl -A "$drive" 2>/dev/null | \
|
||||
awk -v attr="$attr" '$2 == attr {print $10}'
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SMART SMART Health Check ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SMART SMART Health Check — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
DRIVES_OK=()
|
||||
DRIVES_WARN=()
|
||||
DRIVES_CRIT=()
|
||||
DRIVES_SKIP=()
|
||||
|
||||
for drive in /dev/sd? /dev/nvme?; do
|
||||
[[ ! -e "$drive" ]] && continue
|
||||
drive_name=$(basename "$drive")
|
||||
|
||||
# Check ignore list
|
||||
ignored=false
|
||||
for ignore in "${SMART_IGNORE_DRIVES[@]}"; do
|
||||
[[ "$drive_name" == "$ignore" ]] && ignored=true && break
|
||||
done
|
||||
|
||||
if [[ "$ignored" == true ]]; then
|
||||
info "$drive_name — ignored (in SMART_IGNORE_DRIVES)"
|
||||
DRIVES_SKIP+=("$drive_name")
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "━━━ $ICON_SMART $drive_name ━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would check $drive_name"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
# Check if drive supports SMART
|
||||
if ! smartctl -i "$drive" 2>/dev/null | grep -q "SMART support is: Enabled"; then
|
||||
warn "$drive_name — SMART not enabled or not supported"
|
||||
DRIVES_SKIP+=("$drive_name")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
# Overall SMART status
|
||||
SMART_STATUS=$(smartctl -H "$drive" 2>/dev/null | grep "overall-health" | awk '{print $NF}')
|
||||
if [[ "$SMART_STATUS" == "PASSED" ]]; then
|
||||
success "Overall status: PASSED"
|
||||
else
|
||||
error "Overall status: $SMART_STATUS"
|
||||
fi
|
||||
|
||||
# Key attributes
|
||||
DRIVE_WARN=false
|
||||
DRIVE_CRIT=false
|
||||
|
||||
# Reallocated sectors
|
||||
REALLOC=$(get_smart_attr "$drive" "Reallocated_Sector_Ct")
|
||||
if [[ -n "$REALLOC" ]]; then
|
||||
if [[ "$REALLOC" -gt 0 ]]; then
|
||||
warn "$ICON_SMART Reallocated sectors: $REALLOC — drive showing wear"
|
||||
DRIVE_WARN=true
|
||||
else
|
||||
success "$ICON_SMART Reallocated sectors: $REALLOC"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Pending sectors
|
||||
PENDING=$(get_smart_attr "$drive" "Current_Pending_Sector")
|
||||
if [[ -n "$PENDING" ]]; then
|
||||
if [[ "$PENDING" -gt 0 ]]; then
|
||||
warn "$ICON_SMART Pending sectors: $PENDING — sectors awaiting reallocation"
|
||||
DRIVE_WARN=true
|
||||
else
|
||||
success "$ICON_SMART Pending sectors: $PENDING"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Uncorrectable sectors
|
||||
UNCORR=$(get_smart_attr "$drive" "Offline_Uncorrectable")
|
||||
if [[ -n "$UNCORR" ]]; then
|
||||
if [[ "$UNCORR" -gt 0 ]]; then
|
||||
error "$ICON_SMART Uncorrectable sectors: $UNCORR — CRITICAL"
|
||||
DRIVE_CRIT=true
|
||||
else
|
||||
success "$ICON_SMART Uncorrectable sectors: $UNCORR"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Temperature
|
||||
TEMP=$(get_smart_attr "$drive" "Temperature_Celsius")
|
||||
# NVMe uses different attribute name
|
||||
[[ -z "$TEMP" ]] && TEMP=$(smartctl -A "$drive" 2>/dev/null | \
|
||||
awk '/Temperature/{print $2}' | head -1)
|
||||
|
||||
if [[ -n "$TEMP" ]]; then
|
||||
if [[ "$TEMP" -ge "$SMART_TEMP_CRIT" ]]; then
|
||||
error "$ICON_SMART Temperature: ${TEMP}°C — CRITICAL (threshold: ${SMART_TEMP_CRIT}°C)"
|
||||
DRIVE_CRIT=true
|
||||
elif [[ "$TEMP" -ge "$SMART_TEMP_WARN" ]]; then
|
||||
warn "$ICON_SMART Temperature: ${TEMP}°C — warning (threshold: ${SMART_TEMP_WARN}°C)"
|
||||
DRIVE_WARN=true
|
||||
else
|
||||
success "$ICON_SMART Temperature: ${TEMP}°C"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Power on hours — informational
|
||||
POH=$(get_smart_attr "$drive" "Power_On_Hours")
|
||||
if [[ -n "$POH" ]]; then
|
||||
POH_DAYS=$(( POH / 24 ))
|
||||
info "$ICON_SMART Power on hours: $POH (${POH_DAYS} days)"
|
||||
fi
|
||||
|
||||
# Classify drive
|
||||
if [[ "$DRIVE_CRIT" == true ]]; then
|
||||
DRIVES_CRIT+=("$drive_name")
|
||||
elif [[ "$DRIVE_WARN" == true ]]; then
|
||||
DRIVES_WARN+=("$drive_name")
|
||||
else
|
||||
DRIVES_OK+=("$drive_name")
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo "━━━━━ $ICON_SUMMARY SMART HEALTH SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo ""
|
||||
echo " $ICON_SUCCESS Healthy: ${#DRIVES_OK[@]} $ICON_WARN Warning: ${#DRIVES_WARN[@]} $ICON_ERROR Critical: ${#DRIVES_CRIT[@]} skipped: ${#DRIVES_SKIP[@]}"
|
||||
echo ""
|
||||
|
||||
[[ ${#DRIVES_OK[@]} -gt 0 ]] && echo " $ICON_SUCCESS ${DRIVES_OK[*]}"
|
||||
[[ ${#DRIVES_WARN[@]} -gt 0 ]] && echo " $ICON_WARN ${DRIVES_WARN[*]}"
|
||||
[[ ${#DRIVES_CRIT[@]} -gt 0 ]] && echo " $ICON_ERROR ${DRIVES_CRIT[*]}"
|
||||
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN"
|
||||
elif [[ ${#DRIVES_CRIT[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: CRITICAL — ${DRIVES_CRIT[*]}"
|
||||
notify "SMART CRITICAL on $(hostname) — drives need immediate attention: ${DRIVES_CRIT[*]}" "SMART Health" "warning"
|
||||
elif [[ ${#DRIVES_WARN[@]} -gt 0 ]]; then
|
||||
echo "$ICON_WARN Status: WARNING — ${DRIVES_WARN[*]}"
|
||||
notify "SMART WARNING on $(hostname) — drives showing wear: ${DRIVES_WARN[*]}" "SMART Health" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DRIVES HEALTHY"
|
||||
notify "SMART health check passed on $(hostname) — ${#DRIVES_OK[@]} drives healthy" "SMART Health" "normal"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -0,0 +1,245 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Health Digest ----------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Aggregates system health data from across the ecosystem into a single digest report.
|
||||
# Reads existing state files — no new writes to flash drive.
|
||||
#
|
||||
# Three profiles controlled by DIGEST_PROFILE in Master.conf:
|
||||
# always — sends every run regardless of findings (schedule daily for daily digest)
|
||||
# smart — sends only if something worth reporting was found (intelligent filtering)
|
||||
# weekly — sends once per week on DIGEST_DAY regardless of schedule frequency
|
||||
#
|
||||
# The cron schedule stays the same regardless of profile — just change DIGEST_PROFILE
|
||||
# in Master.conf to switch behavior. Run daily, profile controls when it actually notifies.
|
||||
#
|
||||
# Data sources (reads only — no writes):
|
||||
# /tmp/transcode_state.db — ramdisk symlink and usage
|
||||
# /tmp/container_watchdog_state.db — active container strikes
|
||||
# /tmp/system_watchdog_state.db — active system strikes
|
||||
# /boot/config/failover_state.db — current failover state
|
||||
# /boot/config/system_watchdog_failed.db — container skip list
|
||||
# /boot/config/bandwidth_history.db — recent transfer totals
|
||||
# SSL certs via openssl (live check) — days remaining per domain
|
||||
#
|
||||
# All configuration in Master.conf under Health Digest section.
|
||||
# Supports --dry-run to generate report without sending notification.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
success "Health Digest — profile: $DIGEST_PROFILE"
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — report generated but no notification sent"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Profile check — should we send today?
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
SHOULD_SEND=false
|
||||
|
||||
case "$DIGEST_PROFILE" in
|
||||
always)
|
||||
SHOULD_SEND=true
|
||||
log "Profile: always — will send"
|
||||
;;
|
||||
weekly)
|
||||
TODAY_NAME=$(date '+%A')
|
||||
if [[ "$TODAY_NAME" == "$DIGEST_DAY" ]]; then
|
||||
SHOULD_SEND=true
|
||||
log "Profile: weekly — today is $DIGEST_DAY, will send"
|
||||
else
|
||||
info "Profile: weekly — today is $TODAY_NAME, digest day is $DIGEST_DAY — skipping"
|
||||
exit 0
|
||||
fi
|
||||
;;
|
||||
smart)
|
||||
log "Profile: smart — will evaluate findings before deciding"
|
||||
SHOULD_SEND=false # determined after gathering data
|
||||
;;
|
||||
*)
|
||||
warn "Unknown DIGEST_PROFILE: $DIGEST_PROFILE — defaulting to weekly behavior"
|
||||
TODAY_NAME=$(date '+%A')
|
||||
[[ "$TODAY_NAME" == "$DIGEST_DAY" ]] && SHOULD_SEND=true
|
||||
;;
|
||||
esac
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# DATA GATHERING
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_DIGEST Gathering System Data ━━━"
|
||||
|
||||
FINDINGS=() # things worth noting
|
||||
ISSUES=() # things that need attention
|
||||
DIGEST_LINES=() # full report lines
|
||||
|
||||
# ── Failover State ──────────────────────────────────────────────────────────────────────────
|
||||
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
|
||||
FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
FAILOVER_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
|
||||
DIGEST_LINES+=("$ICON_FAILOVER Failover: $FAILOVER_STATE (last change: ${FAILOVER_CHANGE:-unknown})")
|
||||
|
||||
if [[ "$FAILOVER_STATE" != "NORMAL" && -n "$FAILOVER_STATE" ]]; then
|
||||
ISSUES+=("Failover state: $FAILOVER_STATE")
|
||||
[[ "$DIGEST_SMART_ON_FAILOVER" == true ]] && SHOULD_SEND=true
|
||||
fi
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_FAILOVER Failover: state file not found")
|
||||
fi
|
||||
|
||||
# ── Transcode Ramdisk ───────────────────────────────────────────────────────────────────────
|
||||
if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then
|
||||
RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used | tail -1 | tr -d ' ')
|
||||
RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}")
|
||||
RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail | tail -1 | tr -d ' ')
|
||||
RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}")
|
||||
SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null || echo "unknown")
|
||||
DIGEST_LINES+=("$ICON_RAM Transcodes: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET")
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_RAM Transcodes: ramdisk not mounted")
|
||||
ISSUES+=("Ramdisk not mounted")
|
||||
fi
|
||||
|
||||
# ── Container Watchdog Strikes ──────────────────────────────────────────────────────────────
|
||||
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
|
||||
ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | wc -l)
|
||||
if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then
|
||||
STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
|
||||
DIGEST_LINES+=("$ICON_WATCHDOG Container strikes: $ACTIVE_STRIKES active — $STRIKE_LIST")
|
||||
FINDINGS+=("Container watchdog: $ACTIVE_STRIKES active strikes")
|
||||
[[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_WATCHDOG Container watchdog: no active strikes")
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── System Watchdog Strikes ─────────────────────────────────────────────────────────────────
|
||||
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
|
||||
SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | wc -l)
|
||||
if [[ "$SYS_STRIKES" -gt 0 ]]; then
|
||||
SYS_STRIKE_LIST=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
|
||||
DIGEST_LINES+=("$ICON_REBOOT_SMART System strikes: $SYS_STRIKES active — $SYS_STRIKE_LIST")
|
||||
FINDINGS+=("System watchdog: $SYS_STRIKES active strikes")
|
||||
[[ "$DIGEST_SMART_ON_WATCHDOG" == true ]] && SHOULD_SEND=true
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_REBOOT_SMART System watchdog: no active strikes")
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Container Skip List ─────────────────────────────────────────────────────────────────────
|
||||
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
|
||||
SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ')
|
||||
DIGEST_LINES+=("$ICON_NOT_RUNNING Skip list: $SKIP_COUNT containers — $SKIP_LIST")
|
||||
ISSUES+=("Containers on skip list: $SKIP_LIST")
|
||||
SHOULD_SEND=true
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_RUNNING Skip list: empty — all containers healthy")
|
||||
fi
|
||||
|
||||
# ── Bandwidth (yesterday's total) ───────────────────────────────────────────────────────────
|
||||
if [[ -f "$BANDWIDTH_LOG" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then
|
||||
YESTERDAY=$(date -d "yesterday" '+%Y-%m-%d')
|
||||
YESTERDAY_BYTES=$(awk -F'|' -v d="$YESTERDAY" '$1==d{sum+=$3} END{print sum+0}' "$BANDWIDTH_LOG")
|
||||
YESTERDAY_GB=$(awk "BEGIN {printf \"%.2f\", $YESTERDAY_BYTES / 1073741824}")
|
||||
OVER_WARN=$(awk "BEGIN {print ($YESTERDAY_BYTES > $BANDWIDTH_WARN_GB * 1073741824) ? 1 : 0}")
|
||||
|
||||
if [[ "$OVER_WARN" == "1" ]]; then
|
||||
DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday's transfers: ${YESTERDAY_GB}GB ← exceeded ${BANDWIDTH_WARN_GB}GB threshold")
|
||||
FINDINGS+=("High bandwidth day: ${YESTERDAY_GB}GB transferred")
|
||||
[[ "$DIGEST_SMART_ON_BANDWIDTH" == true ]] && SHOULD_SEND=true
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_BANDWIDTH Yesterday's transfers: ${YESTERDAY_GB}GB")
|
||||
fi
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_BANDWIDTH Bandwidth: no data yet")
|
||||
fi
|
||||
|
||||
# ── SSL Certificates ────────────────────────────────────────────────────────────────────────
|
||||
if [[ ${#CERT_MONITOR_DOMAINS[@]} -gt 0 ]] && command -v openssl >/dev/null 2>&1; then
|
||||
CERT_ISSUES=()
|
||||
for domain in "${CERT_MONITOR_DOMAINS[@]}"; do
|
||||
[[ -z "$domain" ]] && continue
|
||||
expiry_str=$(echo | timeout "$CERT_TIMEOUT" openssl s_client \
|
||||
-connect "${domain}:443" -servername "$domain" \
|
||||
2>/dev/null | openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)
|
||||
|
||||
if [[ -n "$expiry_str" ]]; then
|
||||
expiry_epoch=$(date -d "$expiry_str" +%s 2>/dev/null)
|
||||
days_remaining=$(( (expiry_epoch - $(date +%s)) / 86400 ))
|
||||
|
||||
if [[ "$days_remaining" -le "$CERT_CRIT_DAYS" ]]; then
|
||||
CERT_ISSUES+=("$domain: ${days_remaining}d CRITICAL")
|
||||
SHOULD_SEND=true
|
||||
elif [[ "$days_remaining" -le "$CERT_WARN_DAYS" ]]; then
|
||||
CERT_ISSUES+=("$domain: ${days_remaining}d WARNING")
|
||||
[[ "$DIGEST_SMART_ON_CERT_WARN" == true ]] && SHOULD_SEND=true
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ${#CERT_ISSUES[@]} -gt 0 ]]; then
|
||||
DIGEST_LINES+=("$ICON_CERT Certificates: ${CERT_ISSUES[*]}")
|
||||
FINDINGS+=("Cert issues: ${CERT_ISSUES[*]}")
|
||||
else
|
||||
DIGEST_LINES+=("$ICON_CERT Certificates: all healthy")
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Smart profile final decision ────────────────────────────────────────────────────────────
|
||||
if [[ "$DIGEST_PROFILE" == "smart" && "$SHOULD_SEND" == false ]]; then
|
||||
info "Profile: smart — no findings worth reporting — skipping notification"
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━"
|
||||
echo "$ICON_SUCCESS Everything looks healthy — no digest sent (smart profile)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_DIGEST Build and Send Digest ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_DIGEST Health Digest ━━━"
|
||||
|
||||
DIGEST_DATE=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
DIGEST_HOST=$(hostname)
|
||||
|
||||
# Build notification message
|
||||
NOTIFY_MSG="Health Digest — $DIGEST_HOST — $DIGEST_DATE"
|
||||
if [[ ${#ISSUES[@]} -gt 0 ]]; then
|
||||
NOTIFY_MSG+=" | Issues: ${ISSUES[*]}"
|
||||
fi
|
||||
if [[ ${#FINDINGS[@]} -gt 0 ]]; then
|
||||
NOTIFY_MSG+=" | Findings: ${FINDINGS[*]}"
|
||||
fi
|
||||
|
||||
# Print full digest to console
|
||||
for line in "${DIGEST_LINES[@]}"; do
|
||||
echo " $line"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DIGEST SUMMARY ━━━━━"
|
||||
echo "$ICON_DIGEST Profile: $DIGEST_PROFILE"
|
||||
echo "$ICON_TIME Generated: $DIGEST_DATE"
|
||||
echo "$ICON_ERROR Issues: ${#ISSUES[@]}"
|
||||
echo "$ICON_WARN Findings: ${#FINDINGS[@]}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — digest generated but not sent"
|
||||
elif [[ "$SHOULD_SEND" == true ]]; then
|
||||
notify "$NOTIFY_MSG" "Health Digest" "$([[ ${#ISSUES[@]} -gt 0 ]] && echo "warning" || echo "normal")"
|
||||
success "Digest sent"
|
||||
fi
|
||||
Reference in New Issue
Block a user