Gmer4Lfe
95151c2278
Move all watchdog scripts to a dedicated Watchdogs/ folder: Docker_Essentials/docker_watchdog.sh → Watchdogs/ unRAID_Essentials/system_watchdog.sh → Watchdogs/ unRAID_Essentials/resource_watchdog.sh → Watchdogs/ Orchestrators/watchdog_orchestrator.sh → Watchdogs/ Tools/watchdog_skip_list_manager.sh → Watchdogs/ Rename host config files: master_host1.conf → host1.conf master_host2.conf → host2.conf Update all references across the ecosystem: master.conf: WATCHDOG_ORCHESTRATOR_SCRIPTS paths → Watchdogs/ load_config.sh: host*.conf glob + all comments git_pull_execute.sh: sparse checkout glob + all comments Partnership/ssh_setup.sh: HOST_CONF path construction user_script_plug-in.sh: all script paths + per-host conf path common.sh, README.md, README-User_Script_Plug-in.md: comment refs All Partnership, Fallback, Monitors, Transcodes, Tools scripts: comment refs
308 lines
14 KiB
Bash
308 lines
14 KiB
Bash
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ============================= Bandwidth Monitor ==============================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# rsync transfer history logging and weekly summary reporting. Log mode is called
|
|
# automatically by rsync.sh after each sync — no manual scheduling needed for
|
|
# logging. Report mode scheduled weekly (Sunday 11am).
|
|
#
|
|
# Log mode (--log-transfer): appends one line per sync to BANDWIDTH_LOG, trims
|
|
# entries older than BANDWIDTH_LOG_RETENTION days. One bounded write per rsync run.
|
|
#
|
|
# Report mode (default): reads the accumulated log and generates a summary —
|
|
# per-profile breakdown, run count, total transferred, average duration, failures,
|
|
# last 7 days activity timeline, any transfers or days exceeding BANDWIDTH_WARN_GB.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Version-Proof Log Format
|
|
# Earlier designs parsed rsync's human-readable output for bytes transferred.
|
|
# rsync changes its output format between versions — those parsers break silently.
|
|
# The log line (YYYY-MM-DD|HH:MM|profile|duration|status|bytes) captures bytes
|
|
# from rsync --stats via awk using version-stable field names. Survives any rsync
|
|
# update with no changes.
|
|
#
|
|
# Minimal Flash Drive Impact
|
|
# unRAID boots from USB flash. One append + one trim per rsync run. The log file
|
|
# is bounded to BANDWIDTH_LOG_RETENTION days and never grows unbounded.
|
|
# Atomic write: tmp file + mv prevents partial writes during trim.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Lock With Wait
|
|
# acquire_lock "wait" — multiple rsync profiles may complete close together.
|
|
# Waiting (not skipping) prevents log entries from being lost.
|
|
#
|
|
# Atomic Log Write
|
|
# Trim uses tmp file + mv — partial writes on log trim cannot corrupt the log.
|
|
#
|
|
# Log Directory Guard
|
|
# Creates the log directory if it doesn't exist. Exits cleanly if unwritable.
|
|
#
|
|
# Notification Validated
|
|
# validate_unraid_cmd confirms the notify script is present before use.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# BANDWIDTH_LOG
|
|
# Log file path. Stored on /boot/ to survive reboots.
|
|
# (default: /boot/config/bandwidth_history.db)
|
|
#
|
|
# BANDWIDTH_LOG_RETENTION
|
|
# Days before old entries are purged. File stays bounded. (default: 90)
|
|
#
|
|
# BANDWIDTH_WARN_GB
|
|
# Flag transfers or daily totals exceeding this in the report. (default: 50)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# bandwidth_monitor.sh
|
|
# Generate transfer history report from accumulated log.
|
|
#
|
|
# bandwidth_monitor.sh --report
|
|
# Generate report (explicit form).
|
|
#
|
|
# bandwidth_monitor.sh --log-transfer profile secs status bytes
|
|
# Log a completed rsync transfer. Called by rsync.sh — do not call manually.
|
|
#
|
|
# bandwidth_monitor.sh --status
|
|
# Show log path, retention, warn threshold, and log statistics. Then exit.
|
|
#
|
|
# bandwidth_monitor.sh --log
|
|
# Verbose output during report generation.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ── Parse mode from PARSED_ARGS ───────────────────────────────────────────────────────────────
|
|
LOG_TRANSFER_MODE=false
|
|
TRANSFER_PROFILE=""
|
|
TRANSFER_DURATION=0
|
|
TRANSFER_STATUS="success"
|
|
TRANSFER_BYTES=0
|
|
|
|
for arg in "${PARSED_ARGS[@]}"; do
|
|
case "$arg" in
|
|
--log-transfer) LOG_TRANSFER_MODE=true ;;
|
|
--report) LOG_TRANSFER_MODE=false ;;
|
|
*)
|
|
if [[ "$LOG_TRANSFER_MODE" == true ]]; then
|
|
if [[ -z "$TRANSFER_PROFILE" ]]; then
|
|
TRANSFER_PROFILE="$arg"
|
|
elif [[ "$TRANSFER_DURATION" -eq 0 && "$arg" =~ ^[0-9]+$ ]]; then
|
|
TRANSFER_DURATION="$arg"
|
|
elif [[ "$arg" == "success" || "$arg" == "failed" ]]; then
|
|
TRANSFER_STATUS="$arg"
|
|
elif [[ "$arg" =~ ^[0-9]+$ ]]; then
|
|
TRANSFER_BYTES="$arg"
|
|
fi
|
|
fi
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# ── Ensure log file exists and is writable ────────────────────────────────────────────────────
|
|
mkdir -p "$(dirname "$BANDWIDTH_LOG")"
|
|
touch "$BANDWIDTH_LOG" 2>/dev/null || {
|
|
error "Cannot write to bandwidth log: $BANDWIDTH_LOG"
|
|
exit 1
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
validate_unraid_cmd \
|
|
"/usr/local/emhttp/plugins/dynamix/scripts/notify" \
|
|
"" "" \
|
|
"unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
|
|
|
# detect_hosts() sets MY_ID for report header
|
|
detect_hosts
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_BANDWIDTH Log file: $BANDWIDTH_LOG"
|
|
echo "$ICON_BANDWIDTH Retention: ${BANDWIDTH_LOG_RETENTION} days"
|
|
echo "$ICON_BANDWIDTH Warn GB: ${BANDWIDTH_WARN_GB}GB"
|
|
entry_count=0
|
|
[[ -f "$BANDWIDTH_LOG" ]] && entry_count=$(wc -l < "$BANDWIDTH_LOG")
|
|
echo "$ICON_BANDWIDTH Log entries: $entry_count"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Log Transfer Mode ━━━
|
|
# ==============================================================================================
|
|
# Called by rsync.sh after each sync — appends one line and trims old entries.
|
|
# Uses "wait" lock — if two rsync jobs finish simultaneously, wait and write in order.
|
|
# Usage: bandwidth_monitor.sh --log-transfer "profile" duration_seconds status [bytes]
|
|
|
|
if [[ "$LOG_TRANSFER_MODE" == true ]]; then
|
|
[[ -z "$TRANSFER_PROFILE" ]] && { error "No profile specified for --log-transfer"; exit 1; }
|
|
|
|
acquire_lock "wait"
|
|
|
|
TODAY=$(date '+%Y-%m-%d')
|
|
NOW=$(date '+%H:%M')
|
|
DURATION_FMT=$(format_duration "$TRANSFER_DURATION")
|
|
|
|
# Check if transfer exceeds warn threshold
|
|
WARN_FLAG=""
|
|
if [[ -n "$TRANSFER_BYTES" && "$TRANSFER_BYTES" -gt 0 ]]; then
|
|
WARN_BYTES=$(awk "BEGIN {printf \"%d\", $BANDWIDTH_WARN_GB * 1073741824}")
|
|
[[ "$TRANSFER_BYTES" -gt "$WARN_BYTES" ]] && WARN_FLAG="LARGE"
|
|
fi
|
|
|
|
# Append entry — format: date|time|profile|duration|status|bytes|warn_flag
|
|
echo "${TODAY}|${NOW}|${TRANSFER_PROFILE}|${TRANSFER_DURATION}|${TRANSFER_STATUS}|${TRANSFER_BYTES}|${WARN_FLAG}" \
|
|
>> "$BANDWIDTH_LOG"
|
|
echo "$ICON_BANDWIDTH Logged: $TRANSFER_PROFILE — ${DURATION_FMT} — $TRANSFER_STATUS${WARN_FLAG:+ [$WARN_FLAG]}"
|
|
|
|
# Trim entries older than retention — atomic write via temp file
|
|
CUTOFF=$(date -d "${BANDWIDTH_LOG_RETENTION} days ago" '+%Y-%m-%d')
|
|
TEMP_FILE="${BANDWIDTH_LOG}.tmp"
|
|
awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' "$BANDWIDTH_LOG" > "$TEMP_FILE" && \
|
|
mv "$TEMP_FILE" "$BANDWIDTH_LOG"
|
|
log "$ICON_BANDWIDTH Log trimmed — retaining entries from $CUTOFF"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Report Mode ━━━
|
|
# ==============================================================================================
|
|
acquire_lock "wait"
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_BANDWIDTH Bandwidth Report — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_HOST $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo ""
|
|
|
|
if [[ ! -s "$BANDWIDTH_LOG" ]]; then
|
|
warn "No bandwidth data yet — log is empty"
|
|
warn "Data accumulates as rsync jobs complete via rsync.sh"
|
|
exit 0
|
|
fi
|
|
|
|
START=$(date +%s)
|
|
|
|
# ── Log overview ──────────────────────────────────────────────────────────────────────────────
|
|
OLDEST=$(awk -F'|' 'NR==1{print $1}' "$BANDWIDTH_LOG")
|
|
NEWEST=$(awk -F'|' 'END{print $1}' "$BANDWIDTH_LOG")
|
|
ENTRY_COUNT=$(wc -l < "$BANDWIDTH_LOG")
|
|
SUCCESS_COUNT=$(awk -F'|' '$5=="success"' "$BANDWIDTH_LOG" | wc -l)
|
|
FAILED_COUNT=$(awk -F'|' '$5=="failed"' "$BANDWIDTH_LOG" | wc -l)
|
|
LARGE_COUNT=$(awk -F'|' '$7=="LARGE"' "$BANDWIDTH_LOG" | wc -l)
|
|
|
|
info "Log covers: $OLDEST → $NEWEST ($ENTRY_COUNT runs)"
|
|
echo ""
|
|
|
|
# ── Per-profile breakdown ─────────────────────────────────────────────────────────────────────
|
|
echo "━━━ $ICON_BANDWIDTH Per-Profile Summary ━━━"
|
|
awk -F'|' '{
|
|
runs[$3]++
|
|
duration[$3] += $4
|
|
if ($5 == "failed") fails[$3]++
|
|
if ($7 == "LARGE") large[$3]++
|
|
}
|
|
END {
|
|
for (profile in runs) {
|
|
avg = (runs[profile] > 0) ? duration[profile] / runs[profile] : 0
|
|
mins = int(avg / 60)
|
|
secs = int(avg % 60)
|
|
fail_count = (profile in fails) ? fails[profile] : 0
|
|
large_count = (profile in large) ? large[profile] : 0
|
|
large_str = (large_count > 0) ? " ⚠️ " large_count " large" : ""
|
|
printf " %-22s %3d runs avg %dm%ds failed: %d%s\n", \
|
|
profile, runs[profile], mins, secs, fail_count, large_str
|
|
}
|
|
}' "$BANDWIDTH_LOG" | sort
|
|
echo ""
|
|
|
|
# ── Last 7 days ───────────────────────────────────────────────────────────────────────────────
|
|
echo "━━━ $ICON_BANDWIDTH Last 7 Days ━━━"
|
|
for i in 6 5 4 3 2 1 0; do
|
|
day=$(date -d "$i days ago" '+%Y-%m-%d')
|
|
day_name=$(date -d "$i days ago" '+%a')
|
|
day_runs=$(awk -F'|' -v d="$day" '$1==d' "$BANDWIDTH_LOG" | wc -l)
|
|
day_failed=$(awk -F'|' -v d="$day" '$1==d && $5=="failed"' "$BANDWIDTH_LOG" | wc -l)
|
|
day_large=$(awk -F'|' -v d="$day" '$1==d && $7=="LARGE"' "$BANDWIDTH_LOG" | wc -l)
|
|
day_duration=$(awk -F'|' -v d="$day" '$1==d{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG")
|
|
day_duration_fmt=$(format_duration "$day_duration")
|
|
|
|
if [[ "$day_runs" -eq 0 ]]; then
|
|
echo " $ICON_TIME $day ($day_name) — no syncs"
|
|
elif [[ "$day_failed" -gt 0 ]]; then
|
|
echo " $ICON_WARN $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / $ICON_ERROR $day_failed failed"
|
|
elif [[ "$day_large" -gt 0 ]]; then
|
|
echo " $ICON_WARN $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / ⚠️ $day_large large"
|
|
else
|
|
echo " $ICON_DONE $day ($day_name) — $day_runs runs / ${day_duration_fmt} total"
|
|
fi
|
|
done
|
|
echo ""
|
|
|
|
# ── Large transfers ───────────────────────────────────────────────────────────────────────────
|
|
if [[ "$LARGE_COUNT" -gt 0 ]]; then
|
|
echo "━━━ $ICON_WARN Large Transfers (>${BANDWIDTH_WARN_GB}GB) ━━━"
|
|
awk -F'|' '$7=="LARGE" {
|
|
bytes=$6+0
|
|
gb=bytes/1073741824
|
|
printf " %s %s %-20s %.1fGB\n", $1, $2, $3, gb
|
|
}' "$BANDWIDTH_LOG" | tail -10
|
|
echo ""
|
|
fi
|
|
|
|
# ── Totals ────────────────────────────────────────────────────────────────────────────────────
|
|
TOTAL_DURATION=$(awk -F'|' '{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG")
|
|
TOTAL_DURATION_FMT=$(format_duration "$TOTAL_DURATION")
|
|
|
|
END=$(date +%s)
|
|
|
|
echo "━━━━━ $ICON_SUMMARY BANDWIDTH SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_BANDWIDTH Runs: $ENTRY_COUNT ($SUCCESS_COUNT success / $FAILED_COUNT failed)"
|
|
[[ "$LARGE_COUNT" -gt 0 ]] && \
|
|
warn "Large: $LARGE_COUNT transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB"
|
|
echo "$ICON_TIME Total: $TOTAL_DURATION_FMT"
|
|
echo "$ICON_TIME Period: $OLDEST → $NEWEST"
|
|
echo "$ICON_GEAR Retention: ${BANDWIDTH_LOG_RETENTION} days"
|
|
echo "$ICON_TIME Generated: $(format_duration $(( END - START )))"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
# Only notify if there are failures or large transfers worth flagging
|
|
if [[ "$FAILED_COUNT" -gt 0 ]]; then
|
|
notify "Bandwidth report on $(hostname) — $FAILED_COUNT failed sync(s) in ${BANDWIDTH_LOG_RETENTION} day window" \
|
|
"Bandwidth Monitor" "warning"
|
|
elif [[ "$LARGE_COUNT" -gt 0 ]]; then
|
|
notify "Bandwidth report on $(hostname) — $LARGE_COUNT large transfer(s) exceeded ${BANDWIDTH_WARN_GB}GB" \
|
|
"Bandwidth Monitor" "normal"
|
|
fi |