#!/bin/bash # ============================================================================================== # ================================= Bandwidth Monitor ========================================== # ============================================================================================== # Logs rsync transfer history and generates weekly summary reports. # Designed for minimal flash drive impact — one bounded write per rsync run. # # ── TWO MODES ───────────────────────────────────────────────────────────────────────────────── # # --log-transfer "profile" duration_seconds status # Called automatically by rsync.sh after each sync completes. # Appends one line to the log and trims entries older than BANDWIDTH_LOG_RETENTION. # Flags syncs exceeding BANDWIDTH_WARN_GB in the log for weekly report highlighting. # # --report (or no args) # Generates a summary from the accumulated log. # Shows per-profile breakdown, last 7 days, and overall totals. # This is a monitor script — SILENT_MODE=false — output is the point. # # ── LOG FORMAT ──────────────────────────────────────────────────────────────────────────────── # One line per transfer — version-proof, never needs rsync output parsing: # YYYY-MM-DD|HH:MM|profile|duration_seconds|status|bytes_transferred # # Log file stays bounded to BANDWIDTH_LOG_RETENTION days — trimmed on every write. # Minimal flash drive impact: one append + one trim per rsync run. # # ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── # acquire_lock "wait" — prevents log corruption from concurrent rsync completions # validate_unraid_cmd — notify script validated before use # Atomic log write — temp file + mv prevents partial writes on trim # Log existence check — creates log directory if needed, exits cleanly if unwritable # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # BANDWIDTH_LOG — log file path # BANDWIDTH_LOG_RETENTION — days before old entries are purged (default 90) # BANDWIDTH_WARN_GB — flag syncs larger than this in report (default 50) # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # bandwidth_monitor.sh — generate report # bandwidth_monitor.sh --report — generate report (explicit) # bandwidth_monitor.sh --log-transfer profile secs ok — log a transfer (called by rsync.sh) # bandwidth_monitor.sh --status — show config and exit # bandwidth_monitor.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" # Monitor script — output is the point SILENT_MODE=false parse_args "$@" # ── Parse mode from PARSED_ARGS ─────────────────────────────────────────────────────────────── LOG_TRANSFER_MODE=false TRANSFER_PROFILE="" TRANSFER_DURATION=0 TRANSFER_STATUS="success" TRANSFER_BYTES=0 for arg in "${PARSED_ARGS[@]}"; do case "$arg" in --log-transfer) LOG_TRANSFER_MODE=true ;; --report) LOG_TRANSFER_MODE=false ;; *) if [[ "$LOG_TRANSFER_MODE" == true ]]; then if [[ -z "$TRANSFER_PROFILE" ]]; then TRANSFER_PROFILE="$arg" elif [[ "$TRANSFER_DURATION" -eq 0 && "$arg" =~ ^[0-9]+$ ]]; then TRANSFER_DURATION="$arg" elif [[ "$arg" == "success" || "$arg" == "failed" ]]; then TRANSFER_STATUS="$arg" elif [[ "$arg" =~ ^[0-9]+$ ]]; then TRANSFER_BYTES="$arg" fi fi ;; esac done # ── Ensure log file exists and is writable ──────────────────────────────────────────────────── mkdir -p "$(dirname "$BANDWIDTH_LOG")" touch "$BANDWIDTH_LOG" 2>/dev/null || { error "Cannot write to bandwidth log: $BANDWIDTH_LOG" exit 1 } # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi validate_unraid_cmd \ "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ "" "" \ "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" # detect_hosts() sets MY_ID for report header detect_hosts # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_BANDWIDTH Log file: $BANDWIDTH_LOG" echo "$ICON_BANDWIDTH Retention: ${BANDWIDTH_LOG_RETENTION} days" echo "$ICON_BANDWIDTH Warn GB: ${BANDWIDTH_WARN_GB}GB" local entry_count=0 [[ -f "$BANDWIDTH_LOG" ]] && entry_count=$(wc -l < "$BANDWIDTH_LOG") echo "$ICON_BANDWIDTH Log entries: $entry_count" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ━━━ Log Transfer Mode ━━━ # ============================================================================================== # Called by rsync.sh after each sync — appends one line and trims old entries. # Uses "wait" lock — if two rsync jobs finish simultaneously, wait and write in order. # Usage: bandwidth_monitor.sh --log-transfer "profile" duration_seconds status [bytes] if [[ "$LOG_TRANSFER_MODE" == true ]]; then [[ -z "$TRANSFER_PROFILE" ]] && { error "No profile specified for --log-transfer"; exit 1; } acquire_lock "wait" TODAY=$(date '+%Y-%m-%d') NOW=$(date '+%H:%M') DURATION_FMT=$(format_duration "$TRANSFER_DURATION") # Check if transfer exceeds warn threshold WARN_FLAG="" if [[ -n "$TRANSFER_BYTES" && "$TRANSFER_BYTES" -gt 0 ]]; then WARN_BYTES=$(awk "BEGIN {printf \"%d\", $BANDWIDTH_WARN_GB * 1073741824}") [[ "$TRANSFER_BYTES" -gt "$WARN_BYTES" ]] && WARN_FLAG="LARGE" fi # Append entry — format: date|time|profile|duration|status|bytes|warn_flag echo "${TODAY}|${NOW}|${TRANSFER_PROFILE}|${TRANSFER_DURATION}|${TRANSFER_STATUS}|${TRANSFER_BYTES}|${WARN_FLAG}" \ >> "$BANDWIDTH_LOG" log "$ICON_BANDWIDTH Logged: $TRANSFER_PROFILE — ${DURATION_FMT} — $TRANSFER_STATUS${WARN_FLAG:+ [$WARN_FLAG]}" # Trim entries older than retention — atomic write via temp file CUTOFF=$(date -d "${BANDWIDTH_LOG_RETENTION} days ago" '+%Y-%m-%d') TEMP_FILE="${BANDWIDTH_LOG}.tmp" awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' "$BANDWIDTH_LOG" > "$TEMP_FILE" && \ mv "$TEMP_FILE" "$BANDWIDTH_LOG" log "$ICON_BANDWIDTH Log trimmed — retaining entries from $CUTOFF" exit 0 fi # ============================================================================================== # ━━━ Report Mode ━━━ # ============================================================================================== acquire_lock "wait" echo "" echo "━━━ $ICON_BANDWIDTH Bandwidth Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" echo "" if [[ ! -s "$BANDWIDTH_LOG" ]]; then warn "No bandwidth data yet — log is empty" warn "Data accumulates as rsync jobs complete via rsync.sh" exit 0 fi START=$(date +%s) # ── Log overview ────────────────────────────────────────────────────────────────────────────── OLDEST=$(awk -F'|' 'NR==1{print $1}' "$BANDWIDTH_LOG") NEWEST=$(awk -F'|' 'END{print $1}' "$BANDWIDTH_LOG") ENTRY_COUNT=$(wc -l < "$BANDWIDTH_LOG") SUCCESS_COUNT=$(awk -F'|' '$5=="success"' "$BANDWIDTH_LOG" | wc -l) FAILED_COUNT=$(awk -F'|' '$5=="failed"' "$BANDWIDTH_LOG" | wc -l) LARGE_COUNT=$(awk -F'|' '$7=="LARGE"' "$BANDWIDTH_LOG" | wc -l) info "Log covers: $OLDEST → $NEWEST ($ENTRY_COUNT runs)" echo "" # ── Per-profile breakdown ───────────────────────────────────────────────────────────────────── echo "━━━ $ICON_BANDWIDTH Per-Profile Summary ━━━" awk -F'|' '{ runs[$3]++ duration[$3] += $4 if ($5 == "failed") fails[$3]++ if ($7 == "LARGE") large[$3]++ } END { for (profile in runs) { avg = (runs[profile] > 0) ? duration[profile] / runs[profile] : 0 mins = int(avg / 60) secs = int(avg % 60) fail_count = (profile in fails) ? fails[profile] : 0 large_count = (profile in large) ? large[profile] : 0 large_str = (large_count > 0) ? " ⚠️ " large_count " large" : "" printf " %-22s %3d runs avg %dm%ds failed: %d%s\n", \ profile, runs[profile], mins, secs, fail_count, large_str } }' "$BANDWIDTH_LOG" | sort echo "" # ── Last 7 days ─────────────────────────────────────────────────────────────────────────────── echo "━━━ $ICON_BANDWIDTH Last 7 Days ━━━" for i in 6 5 4 3 2 1 0; do day=$(date -d "$i days ago" '+%Y-%m-%d') day_name=$(date -d "$i days ago" '+%a') day_runs=$(awk -F'|' -v d="$day" '$1==d' "$BANDWIDTH_LOG" | wc -l) day_failed=$(awk -F'|' -v d="$day" '$1==d && $5=="failed"' "$BANDWIDTH_LOG" | wc -l) day_large=$(awk -F'|' -v d="$day" '$1==d && $7=="LARGE"' "$BANDWIDTH_LOG" | wc -l) day_duration=$(awk -F'|' -v d="$day" '$1==d{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG") day_duration_fmt=$(format_duration "$day_duration") if [[ "$day_runs" -eq 0 ]]; then echo " $ICON_TIME $day ($day_name) — no syncs" elif [[ "$day_failed" -gt 0 ]]; then echo " $ICON_WARN $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / $ICON_ERROR $day_failed failed" elif [[ "$day_large" -gt 0 ]]; then echo " $ICON_WARN $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / ⚠️ $day_large large" else echo " $ICON_DONE $day ($day_name) — $day_runs runs / ${day_duration_fmt} total" fi done echo "" # ── Large transfers ─────────────────────────────────────────────────────────────────────────── if [[ "$LARGE_COUNT" -gt 0 ]]; then echo "━━━ $ICON_WARN Large Transfers (>${BANDWIDTH_WARN_GB}GB) ━━━" awk -F'|' '$7=="LARGE" { bytes=$6+0 gb=bytes/1073741824 printf " %s %s %-20s %.1fGB\n", $1, $2, $3, gb }' "$BANDWIDTH_LOG" | tail -10 echo "" fi # ── Totals ──────────────────────────────────────────────────────────────────────────────────── TOTAL_DURATION=$(awk -F'|' '{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG") TOTAL_DURATION_FMT=$(format_duration "$TOTAL_DURATION") END=$(date +%s) echo "━━━━━ $ICON_SUMMARY BANDWIDTH SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_BANDWIDTH Runs: $ENTRY_COUNT ($SUCCESS_COUNT success / $FAILED_COUNT failed)" [[ "$LARGE_COUNT" -gt 0 ]] && \ warn "Large: $LARGE_COUNT transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB" echo "$ICON_TIME Total: $TOTAL_DURATION_FMT" echo "$ICON_TIME Period: $OLDEST → $NEWEST" echo "$ICON_GEAR Retention: ${BANDWIDTH_LOG_RETENTION} days" echo "$ICON_TIME Generated: $(format_duration $(( END - START )))" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Only notify if there are failures or large transfers worth flagging if [[ "$FAILED_COUNT" -gt 0 ]]; then notify "Bandwidth report on $(hostname) — $FAILED_COUNT failed sync(s) in ${BANDWIDTH_LOG_RETENTION} day window" \ "Bandwidth Monitor" "warning" elif [[ "$LARGE_COUNT" -gt 0 ]]; then notify "Bandwidth report on $(hostname) — $LARGE_COUNT large transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB" \ "Bandwidth Monitor" "normal" fi