Varaverk/Monitors/bandwidth_monitor.sh

#!/bin/bash
# ==============================================================================================
# ================================= Bandwidth Monitor ==========================================
# ==============================================================================================
# Logs rsync transfer history and generates weekly summary reports.
# Designed for minimal flash drive impact — one bounded write per rsync run.
#
# ── TWO MODES ─────────────────────────────────────────────────────────────────────────────────
#
#   --log-transfer "profile" duration_seconds status
#     Called automatically by rsync.sh after each sync completes.
#     Appends one line to the log and trims entries older than BANDWIDTH_LOG_RETENTION.
#     Flags syncs exceeding BANDWIDTH_WARN_GB in the log for weekly report highlighting.
#
#   --report (or no args)
#     Generates a summary from the accumulated log.
#     Shows per-profile breakdown, last 7 days, and overall totals.
#     This is a monitor script — SILENT_MODE=false — output is the point.
#
# ── LOG FORMAT ────────────────────────────────────────────────────────────────────────────────
# One line per transfer — version-proof, never needs rsync output parsing:
#   YYYY-MM-DD|HH:MM|profile|duration_seconds|status|bytes_transferred
#
# Log file stays bounded to BANDWIDTH_LOG_RETENTION days — trimmed on every write.
# Minimal flash drive impact: one append + one trim per rsync run.
#
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
#   acquire_lock "wait"  — prevents log corruption from concurrent rsync completions
#   validate_unraid_cmd  — notify script validated before use
#   Atomic log write     — temp file + mv prevents partial writes on trim
#   Log existence check  — creates log directory if needed, exits cleanly if unwritable
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
#   BANDWIDTH_LOG            — log file path
#   BANDWIDTH_LOG_RETENTION  — days before old entries are purged (default 90)
#   BANDWIDTH_WARN_GB        — flag syncs larger than this in report (default 50)
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
#   bandwidth_monitor.sh                                 — generate report
#   bandwidth_monitor.sh --report                        — generate report (explicit)
#   bandwidth_monitor.sh --log-transfer profile secs ok  — log a transfer (called by rsync.sh)
#   bandwidth_monitor.sh --status                        — show config and exit
#   bandwidth_monitor.sh --log                           — verbose output
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../load_config.sh"

# Monitor script — output is the point
SILENT_MODE=false

parse_args "$@"

# ── Parse mode from PARSED_ARGS ───────────────────────────────────────────────────────────────
LOG_TRANSFER_MODE=false
TRANSFER_PROFILE=""
TRANSFER_DURATION=0
TRANSFER_STATUS="success"
TRANSFER_BYTES=0

for arg in "${PARSED_ARGS[@]}"; do
    case "$arg" in
        --log-transfer) LOG_TRANSFER_MODE=true ;;
        --report)       LOG_TRANSFER_MODE=false ;;
        *)
            if [[ "$LOG_TRANSFER_MODE" == true ]]; then
                if [[ -z "$TRANSFER_PROFILE" ]]; then
                    TRANSFER_PROFILE="$arg"
                elif [[ "$TRANSFER_DURATION" -eq 0 && "$arg" =~ ^[0-9]+$ ]]; then
                    TRANSFER_DURATION="$arg"
                elif [[ "$arg" == "success" || "$arg" == "failed" ]]; then
                    TRANSFER_STATUS="$arg"
                elif [[ "$arg" =~ ^[0-9]+$ ]]; then
                    TRANSFER_BYTES="$arg"
                fi
            fi
            ;;
    esac
done

# ── Ensure log file exists and is writable ────────────────────────────────────────────────────
mkdir -p "$(dirname "$BANDWIDTH_LOG")"
touch "$BANDWIDTH_LOG" 2>/dev/null || {
    error "Cannot write to bandwidth log: $BANDWIDTH_LOG"
    exit 1
}

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

validate_unraid_cmd \
    "/usr/local/emhttp/plugins/dynamix/scripts/notify" \
    "" "" \
    "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"

# detect_hosts() sets MY_ID for report header
detect_hosts

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
    echo "$ICON_HOST       Identity:   $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_BANDWIDTH  Log file:   $BANDWIDTH_LOG"
    echo "$ICON_BANDWIDTH  Retention:  ${BANDWIDTH_LOG_RETENTION} days"
    echo "$ICON_BANDWIDTH  Warn GB:    ${BANDWIDTH_WARN_GB}GB"
    entry_count=0
    [[ -f "$BANDWIDTH_LOG" ]] && entry_count=$(wc -l < "$BANDWIDTH_LOG")
    echo "$ICON_BANDWIDTH  Log entries: $entry_count"
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Log Transfer Mode ━━━
# ==============================================================================================
# Called by rsync.sh after each sync — appends one line and trims old entries.
# Uses "wait" lock — if two rsync jobs finish simultaneously, wait and write in order.
# Usage: bandwidth_monitor.sh --log-transfer "profile" duration_seconds status [bytes]

if [[ "$LOG_TRANSFER_MODE" == true ]]; then
    [[ -z "$TRANSFER_PROFILE" ]] && { error "No profile specified for --log-transfer"; exit 1; }

    acquire_lock "wait"

    TODAY=$(date '+%Y-%m-%d')
    NOW=$(date '+%H:%M')
    DURATION_FMT=$(format_duration "$TRANSFER_DURATION")

    # Check if transfer exceeds warn threshold
    WARN_FLAG=""
    if [[ -n "$TRANSFER_BYTES" && "$TRANSFER_BYTES" -gt 0 ]]; then
        WARN_BYTES=$(awk "BEGIN {printf \"%d\", $BANDWIDTH_WARN_GB * 1073741824}")
        [[ "$TRANSFER_BYTES" -gt "$WARN_BYTES" ]] && WARN_FLAG="LARGE"
    fi

    # Append entry — format: date|time|profile|duration|status|bytes|warn_flag
    echo "${TODAY}|${NOW}|${TRANSFER_PROFILE}|${TRANSFER_DURATION}|${TRANSFER_STATUS}|${TRANSFER_BYTES}|${WARN_FLAG}" \
        >> "$BANDWIDTH_LOG"
    log "$ICON_BANDWIDTH Logged: $TRANSFER_PROFILE — ${DURATION_FMT} — $TRANSFER_STATUS${WARN_FLAG:+ [$WARN_FLAG]}"

    # Trim entries older than retention — atomic write via temp file
    CUTOFF=$(date -d "${BANDWIDTH_LOG_RETENTION} days ago" '+%Y-%m-%d')
    TEMP_FILE="${BANDWIDTH_LOG}.tmp"
    awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' "$BANDWIDTH_LOG" > "$TEMP_FILE" && \
        mv "$TEMP_FILE" "$BANDWIDTH_LOG"
    log "$ICON_BANDWIDTH Log trimmed — retaining entries from $CUTOFF"
    exit 0
fi

# ==============================================================================================
# ━━━ Report Mode ━━━
# ==============================================================================================
acquire_lock "wait"

echo ""
echo "━━━ $ICON_BANDWIDTH Bandwidth Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
echo ""

if [[ ! -s "$BANDWIDTH_LOG" ]]; then
    warn "No bandwidth data yet — log is empty"
    warn "Data accumulates as rsync jobs complete via rsync.sh"
    exit 0
fi

START=$(date +%s)

# ── Log overview ──────────────────────────────────────────────────────────────────────────────
OLDEST=$(awk -F'|' 'NR==1{print $1}' "$BANDWIDTH_LOG")
NEWEST=$(awk -F'|' 'END{print $1}' "$BANDWIDTH_LOG")
ENTRY_COUNT=$(wc -l < "$BANDWIDTH_LOG")
SUCCESS_COUNT=$(awk -F'|' '$5=="success"' "$BANDWIDTH_LOG" | wc -l)
FAILED_COUNT=$(awk -F'|' '$5=="failed"' "$BANDWIDTH_LOG" | wc -l)
LARGE_COUNT=$(awk -F'|' '$7=="LARGE"' "$BANDWIDTH_LOG" | wc -l)

info "Log covers: $OLDEST → $NEWEST ($ENTRY_COUNT runs)"
echo ""

# ── Per-profile breakdown ─────────────────────────────────────────────────────────────────────
echo "━━━ $ICON_BANDWIDTH Per-Profile Summary ━━━"
awk -F'|' '{
    runs[$3]++
    duration[$3] += $4
    if ($5 == "failed") fails[$3]++
    if ($7 == "LARGE") large[$3]++
}
END {
    for (profile in runs) {
        avg = (runs[profile] > 0) ? duration[profile] / runs[profile] : 0
        mins = int(avg / 60)
        secs = int(avg % 60)
        fail_count = (profile in fails) ? fails[profile] : 0
        large_count = (profile in large) ? large[profile] : 0
        large_str = (large_count > 0) ? "  ⚠️ " large_count " large" : ""
        printf "  %-22s %3d runs  avg %dm%ds  failed: %d%s\n", \
            profile, runs[profile], mins, secs, fail_count, large_str
    }
}' "$BANDWIDTH_LOG" | sort
echo ""

# ── Last 7 days ───────────────────────────────────────────────────────────────────────────────
echo "━━━ $ICON_BANDWIDTH Last 7 Days ━━━"
for i in 6 5 4 3 2 1 0; do
    day=$(date -d "$i days ago" '+%Y-%m-%d')
    day_name=$(date -d "$i days ago" '+%a')
    day_runs=$(awk -F'|' -v d="$day" '$1==d' "$BANDWIDTH_LOG" | wc -l)
    day_failed=$(awk -F'|' -v d="$day" '$1==d && $5=="failed"' "$BANDWIDTH_LOG" | wc -l)
    day_large=$(awk -F'|' -v d="$day" '$1==d && $7=="LARGE"' "$BANDWIDTH_LOG" | wc -l)
    day_duration=$(awk -F'|' -v d="$day" '$1==d{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG")
    day_duration_fmt=$(format_duration "$day_duration")

    if [[ "$day_runs" -eq 0 ]]; then
        echo "  $ICON_TIME  $day ($day_name) — no syncs"
    elif [[ "$day_failed" -gt 0 ]]; then
        echo "  $ICON_WARN  $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / $ICON_ERROR $day_failed failed"
    elif [[ "$day_large" -gt 0 ]]; then
        echo "  $ICON_WARN  $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / ⚠️  $day_large large"
    else
        echo "  $ICON_DONE  $day ($day_name) — $day_runs runs / ${day_duration_fmt} total"
    fi
done
echo ""

# ── Large transfers ───────────────────────────────────────────────────────────────────────────
if [[ "$LARGE_COUNT" -gt 0 ]]; then
    echo "━━━ $ICON_WARN Large Transfers (>${BANDWIDTH_WARN_GB}GB) ━━━"
    awk -F'|' '$7=="LARGE" {
        bytes=$6+0
        gb=bytes/1073741824
        printf "  %s %s  %-20s  %.1fGB\n", $1, $2, $3, gb
    }' "$BANDWIDTH_LOG" | tail -10
    echo ""
fi

# ── Totals ────────────────────────────────────────────────────────────────────────────────────
TOTAL_DURATION=$(awk -F'|' '{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG")
TOTAL_DURATION_FMT=$(format_duration "$TOTAL_DURATION")

END=$(date +%s)

echo "━━━━━ $ICON_SUMMARY BANDWIDTH SUMMARY ━━━━━"
echo "$ICON_HOST       Identity:  $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_BANDWIDTH  Runs:      $ENTRY_COUNT ($SUCCESS_COUNT success / $FAILED_COUNT failed)"
[[ "$LARGE_COUNT" -gt 0 ]] && \
    warn "Large:     $LARGE_COUNT transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB"
echo "$ICON_TIME       Total:     $TOTAL_DURATION_FMT"
echo "$ICON_TIME       Period:    $OLDEST → $NEWEST"
echo "$ICON_GEAR       Retention: ${BANDWIDTH_LOG_RETENTION} days"
echo "$ICON_TIME       Generated: $(format_duration $(( END - START )))"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

# Only notify if there are failures or large transfers worth flagging
if [[ "$FAILED_COUNT" -gt 0 ]]; then
    notify "Bandwidth report on $(hostname) — $FAILED_COUNT failed sync(s) in ${BANDWIDTH_LOG_RETENTION} day window" \
        "Bandwidth Monitor" "warning"
elif [[ "$LARGE_COUNT" -gt 0 ]]; then
    notify "Bandwidth report on $(hostname) — $LARGE_COUNT large transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB" \
        "Bandwidth Monitor" "normal"
fi