#!/bin/bash # ============================================================================================== # ============================= Bandwidth Monitor ============================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # rsync transfer history logging and weekly summary reporting. Log mode is called # automatically by rsync.sh after each sync — no manual scheduling needed for # logging. Report mode scheduled weekly (Sunday 11am). # # Log mode (--log-transfer): appends one line per sync to BANDWIDTH_LOG, trims # entries older than BANDWIDTH_LOG_RETENTION days. One bounded write per rsync run. # # Report mode (default): reads the accumulated log and generates a summary — # per-profile breakdown, run count, total transferred, average duration, failures, # last 7 days activity timeline, any transfers or days exceeding BANDWIDTH_WARN_GB. # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Version-Proof Log Format # Earlier designs parsed rsync's human-readable output for bytes transferred. # rsync changes its output format between versions — those parsers break silently. # The log line (YYYY-MM-DD|HH:MM|profile|duration|status|bytes) captures bytes # from rsync --stats via awk using version-stable field names. Survives any rsync # update with no changes. # # Minimal Flash Drive Impact # unRAID boots from USB flash. One append + one trim per rsync run. The log file # is bounded to BANDWIDTH_LOG_RETENTION days and never grows unbounded. # Atomic write: tmp file + mv prevents partial writes during trim. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Lock With Wait # acquire_lock "wait" — multiple rsync profiles may complete close together. # Waiting (not skipping) prevents log entries from being lost. # # Atomic Log Write # Trim uses tmp file + mv — partial writes on log trim cannot corrupt the log. # # Log Directory Guard # Creates the log directory if it doesn't exist. Exits cleanly if unwritable. # # Notification Validated # platform_require_cmd confirms the notify script is present before use. # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # master.conf # # BANDWIDTH_LOG # Log file path. Stored on /boot/ to survive reboots. # (default: /boot/config/bandwidth_history.db) # # BANDWIDTH_LOG_RETENTION # Days before old entries are purged. File stays bounded. (default: 90) # # BANDWIDTH_WARN_GB # Flag transfers or daily totals exceeding this in the report. (default: 50) # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # bandwidth_monitor.sh # Generate transfer history report from accumulated log. # # bandwidth_monitor.sh --report # Generate report (explicit form). # # bandwidth_monitor.sh --log-transfer profile secs status bytes # Log a completed rsync transfer. Called by rsync.sh — do not call manually. # # bandwidth_monitor.sh --status # Show log path, retention, warn threshold, and log statistics. Then exit. # # bandwidth_monitor.sh --log # Verbose output during report generation. # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" # ── Parse mode from PARSED_ARGS ─────────────────────────────────────────────────────────────── LOG_TRANSFER_MODE=false TRANSFER_PROFILE="" TRANSFER_DURATION=0 TRANSFER_STATUS="success" TRANSFER_BYTES=0 for arg in "${PARSED_ARGS[@]}"; do case "$arg" in --log-transfer) LOG_TRANSFER_MODE=true ;; --report) LOG_TRANSFER_MODE=false ;; *) if [[ "$LOG_TRANSFER_MODE" == true ]]; then if [[ -z "$TRANSFER_PROFILE" ]]; then TRANSFER_PROFILE="$arg" elif [[ "$TRANSFER_DURATION" -eq 0 && "$arg" =~ ^[0-9]+$ ]]; then TRANSFER_DURATION="$arg" elif [[ "$arg" == "success" || "$arg" == "failed" ]]; then TRANSFER_STATUS="$arg" elif [[ "$arg" =~ ^[0-9]+$ ]]; then TRANSFER_BYTES="$arg" fi fi ;; esac done # ── Ensure log file exists and is writable ──────────────────────────────────────────────────── mkdir -p "$(dirname "$BANDWIDTH_LOG")" touch "$BANDWIDTH_LOG" 2>/dev/null || { error "Cannot write to bandwidth log: $BANDWIDTH_LOG" exit 1 } # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi # detect_hosts() sets MY_ID for report header detect_hosts log "$ICON_GEAR Config: log=${BANDWIDTH_LOG} retention=${BANDWIDTH_LOG_RETENTION}d warn=${BANDWIDTH_WARN_GB}GB" # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_BANDWIDTH Log file: $BANDWIDTH_LOG" echo "$ICON_BANDWIDTH Retention: ${BANDWIDTH_LOG_RETENTION} days" echo "$ICON_BANDWIDTH Warn GB: ${BANDWIDTH_WARN_GB}GB" entry_count=0 [[ -f "$BANDWIDTH_LOG" ]] && entry_count=$(wc -l < "$BANDWIDTH_LOG") echo "$ICON_BANDWIDTH Log entries: $entry_count" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ━━━ Log Transfer Mode ━━━ # ============================================================================================== # Called by rsync.sh after each sync — appends one line and trims old entries. # Uses "wait" lock — if two rsync jobs finish simultaneously, wait and write in order. # Usage: bandwidth_monitor.sh --log-transfer "profile" duration_seconds status [bytes] if [[ "$LOG_TRANSFER_MODE" == true ]]; then [[ -z "$TRANSFER_PROFILE" ]] && { error "No profile specified for --log-transfer"; exit 1; } acquire_lock "wait" TODAY=$(date '+%Y-%m-%d') NOW=$(date '+%H:%M') DURATION_FMT=$(format_duration "$TRANSFER_DURATION") # Check if transfer exceeds warn threshold WARN_FLAG="" if [[ -n "$TRANSFER_BYTES" && "$TRANSFER_BYTES" -gt 0 ]]; then WARN_BYTES=$(awk "BEGIN {printf \"%d\", $BANDWIDTH_WARN_GB * 1073741824}") [[ "$TRANSFER_BYTES" -gt "$WARN_BYTES" ]] && WARN_FLAG="LARGE" fi # Append entry — format: date|time|profile|duration|status|bytes|warn_flag echo "${TODAY}|${NOW}|${TRANSFER_PROFILE}|${TRANSFER_DURATION}|${TRANSFER_STATUS}|${TRANSFER_BYTES}|${WARN_FLAG}" \ >> "$BANDWIDTH_LOG" echo "$ICON_BANDWIDTH Logged: $TRANSFER_PROFILE — ${DURATION_FMT} — $TRANSFER_STATUS${WARN_FLAG:+ [$WARN_FLAG]}" # Trim entries older than retention — atomic write via temp file CUTOFF=$(date -d "${BANDWIDTH_LOG_RETENTION} days ago" '+%Y-%m-%d') TEMP_FILE="${BANDWIDTH_LOG}.tmp" awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' "$BANDWIDTH_LOG" > "$TEMP_FILE" && \ mv "$TEMP_FILE" "$BANDWIDTH_LOG" log "$ICON_BANDWIDTH Log trimmed — retaining entries from $CUTOFF" exit 0 fi # ============================================================================================== # ━━━ Report Mode ━━━ # ============================================================================================== acquire_lock "wait" echo "" echo "━━━ $ICON_BANDWIDTH Bandwidth Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)" echo "" if [[ ! -s "$BANDWIDTH_LOG" ]]; then warn "No bandwidth data yet — log is empty" warn "Data accumulates as rsync jobs complete via rsync.sh" exit 0 fi START=$(date +%s) # ── Log overview ────────────────────────────────────────────────────────────────────────────── OLDEST=$(awk -F'|' 'NR==1{print $1}' "$BANDWIDTH_LOG") NEWEST=$(awk -F'|' 'END{print $1}' "$BANDWIDTH_LOG") ENTRY_COUNT=$(wc -l < "$BANDWIDTH_LOG") SUCCESS_COUNT=$(awk -F'|' '$5=="success"' "$BANDWIDTH_LOG" | wc -l) FAILED_COUNT=$(awk -F'|' '$5=="failed"' "$BANDWIDTH_LOG" | wc -l) LARGE_COUNT=$(awk -F'|' '$7=="LARGE"' "$BANDWIDTH_LOG" | wc -l) info "Log covers: $OLDEST → $NEWEST ($ENTRY_COUNT runs)" echo "" # ── Per-profile breakdown ───────────────────────────────────────────────────────────────────── echo "━━━ $ICON_BANDWIDTH Per-Profile Summary ━━━" awk -F'|' '{ runs[$3]++ duration[$3] += $4 if ($5 == "failed") fails[$3]++ if ($7 == "LARGE") large[$3]++ } END { for (profile in runs) { avg = (runs[profile] > 0) ? duration[profile] / runs[profile] : 0 mins = int(avg / 60) secs = int(avg % 60) fail_count = (profile in fails) ? fails[profile] : 0 large_count = (profile in large) ? large[profile] : 0 large_str = (large_count > 0) ? " ⚠️ " large_count " large" : "" printf " %-22s %3d runs avg %dm%ds failed: %d%s\n", \ profile, runs[profile], mins, secs, fail_count, large_str } }' "$BANDWIDTH_LOG" | sort echo "" # ── Last 7 days ─────────────────────────────────────────────────────────────────────────────── echo "━━━ $ICON_BANDWIDTH Last 7 Days ━━━" for i in 6 5 4 3 2 1 0; do day=$(date -d "$i days ago" '+%Y-%m-%d') day_name=$(date -d "$i days ago" '+%a') day_runs=$(awk -F'|' -v d="$day" '$1==d' "$BANDWIDTH_LOG" | wc -l) day_failed=$(awk -F'|' -v d="$day" '$1==d && $5=="failed"' "$BANDWIDTH_LOG" | wc -l) day_large=$(awk -F'|' -v d="$day" '$1==d && $7=="LARGE"' "$BANDWIDTH_LOG" | wc -l) day_duration=$(awk -F'|' -v d="$day" '$1==d{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG") day_duration_fmt=$(format_duration "$day_duration") if [[ "$day_runs" -eq 0 ]]; then echo " $ICON_TIME $day ($day_name) — no syncs" elif [[ "$day_failed" -gt 0 ]]; then echo " $ICON_WARN $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / $ICON_ERROR $day_failed failed" elif [[ "$day_large" -gt 0 ]]; then echo " $ICON_WARN $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / ⚠️ $day_large large" else echo " $ICON_DONE $day ($day_name) — $day_runs runs / ${day_duration_fmt} total" fi done echo "" # ── Large transfers ─────────────────────────────────────────────────────────────────────────── if [[ "$LARGE_COUNT" -gt 0 ]]; then echo "━━━ $ICON_WARN Large Transfers (>${BANDWIDTH_WARN_GB}GB) ━━━" awk -F'|' '$7=="LARGE" { bytes=$6+0 gb=bytes/1073741824 printf " %s %s %-20s %.1fGB\n", $1, $2, $3, gb }' "$BANDWIDTH_LOG" | tail -10 echo "" fi # ── Totals ──────────────────────────────────────────────────────────────────────────────────── TOTAL_DURATION=$(awk -F'|' '{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG") TOTAL_DURATION_FMT=$(format_duration "$TOTAL_DURATION") END=$(date +%s) echo "━━━━━ $ICON_SUMMARY BANDWIDTH SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_BANDWIDTH Runs: $ENTRY_COUNT ($SUCCESS_COUNT success / $FAILED_COUNT failed)" [[ "$LARGE_COUNT" -gt 0 ]] && \ warn "Large: $LARGE_COUNT transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB" echo "$ICON_TIME Total: $TOTAL_DURATION_FMT" echo "$ICON_TIME Period: $OLDEST → $NEWEST" echo "$ICON_GEAR Retention: ${BANDWIDTH_LOG_RETENTION} days" echo "$ICON_TIME Generated: $(format_duration $(( END - START )))" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Only notify if there are failures or large transfers worth flagging if [[ "$FAILED_COUNT" -gt 0 ]]; then notify "Bandwidth report on $(hostname) — $FAILED_COUNT failed sync(s) in ${BANDWIDTH_LOG_RETENTION} day window" \ "Bandwidth Monitor" "warning" elif [[ "$LARGE_COUNT" -gt 0 ]]; then notify "Bandwidth report on $(hostname) — $LARGE_COUNT large transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB" \ "Bandwidth Monitor" "normal" fi