diff --git a/Master.conf b/Master.conf index fcb9bd9..82e146d 100644 --- a/Master.conf +++ b/Master.conf @@ -304,7 +304,6 @@ declare -A PROFILE_SKIP_DISK_CHECK=( # FAILOVER — remote down, internet up — start remote's containers locally (additive) # Own normal containers keep running — failover containers added on top # NO_INTERNET — internet down — stop public-facing containers, wait for recovery -# DARK — remote down + internet down — same actions as NO_INTERNET # # Handback sequence when remote returns after FAILOVER: # 1. Strike confirmation — FAILOVER_HANDBACK_STRIKES consecutive remote-up checks @@ -670,7 +669,7 @@ MEDIA_MAINTENANCE_JOBS=( # Usage thresholds in GB RAMDISK_WARN_GB=6.8 # flip symlink to SSD at or above this usage - RAMDISK_LOW_GB=5.5 # flip symlink back to ramdisk when usage drops here + RAMDISK_LOW_GB=5.0 # flip symlink back to ramdisk when usage drops here RAMDISK_SSD_MIN_GB=20 # minimum free GB on SSD required before allowing flip to SSD # Cleanup age thresholds — files must be older than these AND not open by any process @@ -698,10 +697,8 @@ MEDIA_MAINTENANCE_JOBS=( # Each domain and subdomain is a separate entry — they have independent certs. # Add your public-facing domains — uncomment and replace with your actual domains. CERT_MONITOR_DOMAINS=( - # "yourdomain.com" - # "auth.yourdomain.com" - # "emby.yourdomain.com" - # "nextcloud.yourdomain.com" + "Gmer4Lfe.com" + "Gmer4Lfe.us" ) CERT_WARN_DAYS=30 # notify warning when cert expires within this many days CERT_CRIT_DAYS=7 # notify critical when cert expires within this many days @@ -765,7 +762,7 @@ SMART_IGNORE_DRIVES=( # Shows active streams, library counts, transcode vs direct play ratio. # Requires an API key from Emby Settings → API Keys in the Emby WebUI. EMBY_URL="http://localhost:8096" - EMBY_API_KEY="" # paste your Emby API key here + EMBY_API_KEY="0c27448d93a7431f9ac63569f7655829" # paste your Emby API key here EMBY_REPORT_DAYS=7 # number of days to include in the report period EMBY_REPORT_TOP_N=10 # number of top content items to show in report @@ -835,8 +832,8 @@ SMART_IGNORE_DRIVES=( # false = reboot anyway regardless of this condition (aggressive) # Philosophy: a graceful reboot before crash is always better than a hard crash mid-operation SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true # unhealthy pool + reboot risks data loss - SYS_WATCHDOG_ABORT_ON_PARITY=true # aborting parity check beats crashing mid-check - SYS_WATCHDOG_ABORT_ON_MOVER=true # aborting mover beats crashing mid-move + SYS_WATCHDOG_ABORT_ON_PARITY=false # aborting parity check beats crashing mid-check + SYS_WATCHDOG_ABORT_ON_MOVER=false # aborting mover beats crashing mid-move # ============================================================================================== # ──────────────────────── End Of User Variables ─────────────────────────────────────────────── diff --git a/Monitors/bandwidth_monitor.sh b/Monitors/bandwidth_monitor.sh index 7e72097..8608ef1 100644 --- a/Monitors/bandwidth_monitor.sh +++ b/Monitors/bandwidth_monitor.sh @@ -2,20 +2,19 @@ # ----------------------------------------------------------------------------------------------- # --------------------------------- Bandwidth Monitor ------------------------------------------ # ----------------------------------------------------------------------------------------------- -# Logs daily rsync transfer totals and generates weekly summary reports. -# Designed for minimal flash drive impact — one bounded write per day. +# Logs rsync transfer history and generates weekly summary reports. +# Designed for minimal flash drive impact — one bounded write per rsync run. # # Two modes: -# --log-transfer "profile" bytes duration — called by rsync.sh after each sync -# appends one line, trims old entries -# --report — generates weekly summary from log -# (no args) — generates summary report +# --log-transfer "profile" duration status — called by rsync.sh after each sync +# appends one line, trims old entries +# --report (or no args) — generates summary from log # -# Log format (one line per transfer): -# YYYY-MM-DD|profile|bytes|duration_seconds +# Log format — one line per transfer, version-proof, never needs rsync output parsing: +# YYYY-MM-DD|HH:MM|profile|duration_seconds|status # -# Log file stays bounded to BANDWIDTH_LOG_RETENTION days — old entries trimmed on write. -# Minimal writes: one append per rsync run, one trim per append. +# Log file stays bounded to BANDWIDTH_LOG_RETENTION days — old entries trimmed on every write. +# Minimal flash drive impact: one append + one trim per rsync run. # # All configuration in Master.conf under Bandwidth Monitor section. # ----------------------------------------------------------------------------------------------- @@ -27,51 +26,61 @@ source "$SCRIPT_DIR/../common.sh" parse_args "$@" -# Check for log-transfer mode +# ----------------------------------------------------------------------------------------------- +# Parse mode from PARSED_ARGS +# ----------------------------------------------------------------------------------------------- LOG_TRANSFER_MODE=false -REPORT_MODE=false TRANSFER_PROFILE="" -TRANSFER_BYTES=0 TRANSFER_DURATION=0 +TRANSFER_STATUS="success" for arg in "${PARSED_ARGS[@]}"; do case "$arg" in --log-transfer) LOG_TRANSFER_MODE=true ;; - --report) REPORT_MODE=true ;; + --report) LOG_TRANSFER_MODE=false ;; *) - if [[ "$LOG_TRANSFER_MODE" == true && -z "$TRANSFER_PROFILE" ]]; then - TRANSFER_PROFILE="$arg" - elif [[ "$LOG_TRANSFER_MODE" == true && "$TRANSFER_BYTES" -eq 0 ]]; then - TRANSFER_BYTES="$arg" - elif [[ "$LOG_TRANSFER_MODE" == true ]]; then - TRANSFER_DURATION="$arg" + if [[ "$LOG_TRANSFER_MODE" == true ]]; then + if [[ -z "$TRANSFER_PROFILE" ]]; then + TRANSFER_PROFILE="$arg" + elif [[ "$TRANSFER_DURATION" -eq 0 ]]; then + TRANSFER_DURATION="$arg" + else + TRANSFER_STATUS="$arg" + fi fi ;; esac done -# Ensure log file exists +# Ensure log directory and file exist +mkdir -p "$(dirname "$BANDWIDTH_LOG")" touch "$BANDWIDTH_LOG" 2>/dev/null || { - error "Cannot create bandwidth log: $BANDWIDTH_LOG" + error "Cannot write to bandwidth log: $BANDWIDTH_LOG" exit 1 } # ----------------------------------------------------------------------------------------------- # LOG TRANSFER MODE -# Called by rsync.sh after each successful sync — appends one line and trims old entries -# Usage: bandwidth_monitor.sh --log-transfer "profile" bytes duration +# Called by rsync.sh after each sync — appends one line and trims old entries. +# Usage: bandwidth_monitor.sh --log-transfer "profile" duration_seconds status # ----------------------------------------------------------------------------------------------- if [[ "$LOG_TRANSFER_MODE" == true ]]; then + [[ -z "$TRANSFER_PROFILE" ]] && error "No profile specified for --log-transfer" && exit 1 + TODAY=$(date '+%Y-%m-%d') - echo "${TODAY}|${TRANSFER_PROFILE}|${TRANSFER_BYTES}|${TRANSFER_DURATION}" >> "$BANDWIDTH_LOG" - log "Logged transfer: $TRANSFER_PROFILE — $TRANSFER_BYTES bytes in ${TRANSFER_DURATION}s" + NOW=$(date '+%H:%M') + DURATION_FMT=$(format_duration "$TRANSFER_DURATION") + + # Append entry + echo "${TODAY}|${NOW}|${TRANSFER_PROFILE}|${TRANSFER_DURATION}|${TRANSFER_STATUS}" >> "$BANDWIDTH_LOG" + log "$ICON_BANDWIDTH Logged: $TRANSFER_PROFILE — ${DURATION_FMT} — $TRANSFER_STATUS" # Trim entries older than retention period — keeps file bounded CUTOFF=$(date -d "${BANDWIDTH_LOG_RETENTION} days ago" '+%Y-%m-%d') TEMP_FILE="${BANDWIDTH_LOG}.tmp" - awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' "$BANDWIDTH_LOG" > "$TEMP_FILE" - mv "$TEMP_FILE" "$BANDWIDTH_LOG" - log "Log trimmed — keeping entries from $CUTOFF onwards" + awk -F'|' -v cutoff="$CUTOFF" '$1 >= cutoff' "$BANDWIDTH_LOG" > "$TEMP_FILE" && \ + mv "$TEMP_FILE" "$BANDWIDTH_LOG" + log "$ICON_BANDWIDTH Log trimmed — retaining entries from $CUTOFF onwards" exit 0 fi @@ -84,69 +93,76 @@ echo "" if [[ ! -s "$BANDWIDTH_LOG" ]]; then warn "No bandwidth data yet — log is empty" - warn "Data accumulates as rsync jobs run" + warn "Data accumulates as rsync jobs complete" exit 0 fi START=$(date +%s) -# Calculate date range in log +# Date range OLDEST=$(awk -F'|' 'NR==1{print $1}' "$BANDWIDTH_LOG") NEWEST=$(awk -F'|' 'END{print $1}' "$BANDWIDTH_LOG") ENTRY_COUNT=$(wc -l < "$BANDWIDTH_LOG") +SUCCESS_COUNT=$(awk -F'|' '$5=="success"' "$BANDWIDTH_LOG" | wc -l) +FAILED_COUNT=$(awk -F'|' '$5=="failed"' "$BANDWIDTH_LOG" | wc -l) -info "Log covers: $OLDEST → $NEWEST ($ENTRY_COUNT entries)" +info "Log covers: $OLDEST → $NEWEST ($ENTRY_COUNT runs)" echo "" -# Total bytes transferred -TOTAL_BYTES=$(awk -F'|' '{sum += $3} END {print sum+0}' "$BANDWIDTH_LOG") -TOTAL_GB=$(awk "BEGIN {printf \"%.2f\", $TOTAL_BYTES / 1073741824}") - -# Per-profile breakdown -echo "━━━ $ICON_BANDWIDTH Per-Profile Totals ━━━" +# ── Per-profile breakdown ──────────────────────────────────────────────────────────────────── +echo "━━━ $ICON_BANDWIDTH Per-Profile Summary ━━━" awk -F'|' '{ - bytes[$2] += $3 - runs[$2]++ - duration[$2] += $4 + runs[$3]++ + duration[$3] += $4 + if ($5 == "failed") fails[$3]++ } END { - for (profile in bytes) { - gb = bytes[profile] / 1073741824 - printf " %-20s %6.2f GB (%d runs)\n", profile, gb, runs[profile] + for (profile in runs) { + avg = (runs[profile] > 0) ? duration[profile] / runs[profile] : 0 + mins = int(avg / 60) + secs = int(avg % 60) + fail_count = (profile in fails) ? fails[profile] : 0 + printf " %-20s %3d runs avg %dm%ds failed: %d\n", \ + profile, runs[profile], mins, secs, fail_count } -}' "$BANDWIDTH_LOG" | sort -k3 -rn +}' "$BANDWIDTH_LOG" | sort echo "" -# Daily totals for the last 7 days +# ── Last 7 days ────────────────────────────────────────────────────────────────────────────── echo "━━━ $ICON_BANDWIDTH Last 7 Days ━━━" for i in 6 5 4 3 2 1 0; do day=$(date -d "$i days ago" '+%Y-%m-%d') - day_bytes=$(awk -F'|' -v d="$day" '$1==d{sum+=$3} END{print sum+0}' "$BANDWIDTH_LOG") - day_gb=$(awk "BEGIN {printf \"%.2f\", $day_bytes / 1073741824}") + day_name=$(date -d "$i days ago" '+%a') + day_runs=$(awk -F'|' -v d="$day" '$1==d' "$BANDWIDTH_LOG" | wc -l) + day_success=$(awk -F'|' -v d="$day" '$1==d && $5=="success"' "$BANDWIDTH_LOG" | wc -l) + day_failed=$(awk -F'|' -v d="$day" '$1==d && $5=="failed"' "$BANDWIDTH_LOG" | wc -l) + day_duration=$(awk -F'|' -v d="$day" '$1==d{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG") + day_duration_fmt=$(format_duration "$day_duration") - # Flag days that exceeded warning threshold - over_warn=$(awk "BEGIN {print ($day_bytes > $BANDWIDTH_WARN_GB * 1073741824) ? 1 : 0}") - if [[ "$over_warn" == "1" ]]; then - echo " $ICON_WARN $day ${day_gb} GB ← exceeded ${BANDWIDTH_WARN_GB}GB threshold" + if [[ "$day_runs" -eq 0 ]]; then + echo " $ICON_TIME $day ($day_name) — no syncs" + elif [[ "$day_failed" -gt 0 ]]; then + echo " $ICON_WARN $day ($day_name) — $day_runs runs / ${day_duration_fmt} total / $ICON_ERROR $day_failed failed" else - echo " $ICON_TIME $day ${day_gb} GB" + echo " $ICON_DONE $day ($day_name) — $day_runs runs / ${day_duration_fmt} total" fi done echo "" -echo "━━━ $ICON_SUMMARY Totals ━━━" -echo " $ICON_BANDWIDTH Total transferred: ${TOTAL_GB} GB" -echo " $ICON_TIME Log period: $OLDEST → $NEWEST" -echo " $ICON_GEAR Retention: ${BANDWIDTH_LOG_RETENTION} days" + +# ── Totals ─────────────────────────────────────────────────────────────────────────────────── +TOTAL_DURATION=$(awk -F'|' '{sum+=$4} END{print sum+0}' "$BANDWIDTH_LOG") +TOTAL_DURATION_FMT=$(format_duration "$TOTAL_DURATION") END=$(date +%s) -echo "" echo "━━━━━ $ICON_SUMMARY BANDWIDTH SUMMARY ━━━━━" -echo "$ICON_BANDWIDTH Total: ${TOTAL_GB} GB" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -echo "$ICON_DONE Status: $ICON_SUCCESS DONE" +echo "$ICON_BANDWIDTH Total runs: $ENTRY_COUNT ($SUCCESS_COUNT success / $FAILED_COUNT failed)" +echo "$ICON_TIME Total time: $TOTAL_DURATION_FMT" +echo "$ICON_TIME Log period: $OLDEST → $NEWEST" +echo "$ICON_GEAR Retention: ${BANDWIDTH_LOG_RETENTION} days" +echo "$ICON_TIME Generated in: $(format_duration $((END - START)))" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -notify "Bandwidth report on $(hostname) — ${TOTAL_GB}GB transferred (${OLDEST} to ${NEWEST})" "Bandwidth Monitor" "normal" \ No newline at end of file +notify "Bandwidth report on $(hostname) — $ENTRY_COUNT rsync runs ($SUCCESS_COUNT success / $FAILED_COUNT failed) over ${BANDWIDTH_LOG_RETENTION} day window" "Bandwidth Monitor" "normal" \ No newline at end of file diff --git a/unRAID_Essentials/zfs_memory_snapshot.sh b/Monitors/zfs_memory_snapshot.sh similarity index 100% rename from unRAID_Essentials/zfs_memory_snapshot.sh rename to Monitors/zfs_memory_snapshot.sh diff --git a/Rsync/rsync.sh b/Rsync/rsync.sh index 9d17c9c..46c06ce 100644 --- a/Rsync/rsync.sh +++ b/Rsync/rsync.sh @@ -6,6 +6,9 @@ # Profile is inferred from the directory basename (lowercased). # If no profile match is found all settings fall through to global defaults in Master.conf. # +# After each successful sync, logs the transfer to bandwidth_monitor.sh for weekly reporting. +# Log entry: date | time | profile | duration | status +# # Usage: # rsync.sh /mnt/user/Movies — media share, uses global defaults # rsync.sh /mnt/user/appdata-Failover/Arrs_Stack — matched to [arrs_stack] profile @@ -20,7 +23,6 @@ source "$SCRIPT_DIR/../common.sh" # ----------------------------------------------------------------------------------------------- # Separate the positional directory argument from flag/key=value args. -# Flags and key=value pairs are passed to parse_args — directory is handled here. # ----------------------------------------------------------------------------------------------- DIRECTORY="" RAW_ARGS=() @@ -46,25 +48,24 @@ detect_hosts resolve_remote_ip # ----------------------------------------------------------------------------------------------- -# Profile is inferred from the directory basename (lowercased) -# e.g. /mnt/user/appdata-Failover/Arrs_Stack → arrs_stack +# Profile inference — basename of directory lowercased # ----------------------------------------------------------------------------------------------- echo "" PROFILE_NAME=$(basename "$DIRECTORY" | tr '[:upper:]' '[:lower:]') info "$ICON_GEAR Loading profile: $PROFILE_NAME" -# Scalar overrides — use profile value if defined, fall back to global default +# Scalar overrides BW_LIMIT=${PROFILE_BW_LIMIT[$PROFILE_NAME]:-$BW_LIMIT} RETRY_COUNT=${PROFILE_RETRY_COUNT[$PROFILE_NAME]:-$RETRY_COUNT} SLEEP=${PROFILE_SLEEP[$PROFILE_NAME]:-$SLEEP} CONTAINER_DELAY=${PROFILE_CONTAINER_DELAY[$PROFILE_NAME]:-$CONTAINER_DELAY} -# Array overrides — profile strings must be converted to bash arrays before use +# Array overrides read -r -a CRITICAL_CONTAINER_NAMES <<< "${PROFILE_CRITICAL_CONTAINER_NAMES[$PROFILE_NAME]:-}" read -r -a DELAYED_CONTAINERS <<< "${PROFILE_DELAYED_CONTAINERS[$PROFILE_NAME]:-}" read -r -a EXCLUDE_DIRS <<< "${PROFILE_EXCLUDE_DIRS[$PROFILE_NAME]:-}" -# Disk check toggle — true = skip per-disk check (ZFS pool), false = run check (array disks) +# Disk check toggle SKIP_DISK_CHECK=${PROFILE_SKIP_DISK_CHECK[$PROFILE_NAME]:-false} [[ "$SHOW_STATUS" == true ]] && show_status && exit 0 @@ -105,7 +106,7 @@ echo "" get_rsync_opts -# Append profile excludes to rsync options +# Append profile excludes for ex in "${EXCLUDE_DIRS[@]}"; do [[ -n "$ex" ]] && RSYNC_OPTS+=(--exclude="$ex") done @@ -139,6 +140,21 @@ echo "━━━ $ICON_START $ICON_CONTAINERS Containers ━━━" start_containers END=$(date +%s) +DURATION=$((END - START)) + +# ----------------------------------------------------------------------------------------------- +# Log transfer to bandwidth monitor — only on successful non-dry-run syncs +# Reliable format: date|time|profile|duration|status +# Does not parse rsync output — version-proof and always works +# ----------------------------------------------------------------------------------------------- +BANDWIDTH_MONITOR="$SCRIPT_DIR/../Monitor/bandwidth_monitor.sh" + +if [[ "$DRY_RUN" == false ]] && [[ -f "$BANDWIDTH_MONITOR" ]]; then + STATUS="success" + [[ "$RSYNC_SUCCESS" == false ]] && STATUS="failed" + bash "$BANDWIDTH_MONITOR" --log-transfer "$PROFILE_NAME" "$DURATION" "$STATUS" + log "$ICON_BANDWIDTH Transfer logged to bandwidth monitor" +fi # ----------------------------------------------------------------------------------------------- # ━━━ Summary ━━━ @@ -148,10 +164,11 @@ echo "━━━━━ $ICON_SUMMARY SUMMARY ━━━━━" echo "$ICON_RUN Directory: $DIRECTORY" echo "$ICON_GEAR Profile: $PROFILE_NAME" echo "$ICON_DISK Disk check: $([[ "$SKIP_DISK_CHECK" == "true" ]] && echo "skipped (ZFS pool)" || echo "passed")" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_TIME Duration: $(format_duration $DURATION)" + if [[ "$RSYNC_SUCCESS" == true ]]; then echo "$ICON_DONE Status: $ICON_SUCCESS DONE" - notify "Rsync complete — $DIRECTORY ($PROFILE_NAME) in $(format_duration $((END - START)))" "Rsync" "normal" + notify "Rsync complete — $DIRECTORY ($PROFILE_NAME) in $(format_duration $DURATION)" "Rsync" "normal" else echo "$ICON_ERROR Status: $ICON_ERROR FAILED after $RETRY_COUNT attempts" notify "Rsync failed — $DIRECTORY ($PROFILE_NAME) after $RETRY_COUNT attempts" "Rsync" "warning"