Gmer4Lfe
ea9f39a803
log() is gated on ENABLE_LOGGING — silent on normal runs. Healthy confirmations (All healthy, Network healthy, Storage healthy, WebGUI healthy) were invisible, making banners appear with nothing after them. Switched all four to plain echo so they show every cycle.
440 lines
20 KiB
Bash
Executable File
440 lines
20 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= Storage Watchdog ===========================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Pool and storage health monitoring — catches runaway data growth before it
|
|
# fills a pool. Runs as a single-pass script called by watchdog_orchestrator.sh
|
|
# every cycle. Sits between docker_watchdog.sh (container health) and
|
|
# system_watchdog.sh (last line of defense). Never reboots — detects, alerts,
|
|
# and optionally remediates.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Appdata Size Monitoring
|
|
# Two complementary checks run every cycle:
|
|
#
|
|
# Part 1 — Growth rate (zero-config catch-all):
|
|
# Reads per-container dir totals via du, compares to previous cycle baseline.
|
|
# Any container growing more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused
|
|
# *.log scan inside that container's dir. No per-container config required —
|
|
# new containers are covered automatically. Baseline built on first cycle after
|
|
# boot; growth detection active from cycle 2.
|
|
#
|
|
# Part 2 — Absolute log size:
|
|
# Finds *.log / *.log.* files over WATCHDOG_APPDATA_LOG_MAX_GB anywhere in
|
|
# WATCHDOG_APPDATA_PATHS. Catches logs already large but no longer actively
|
|
# growing. Independent strike counter per file.
|
|
#
|
|
# Strike System
|
|
# Reuses the same strike pattern as CPU/HTTP checks in docker_watchdog.sh:
|
|
#
|
|
# Strike 1 — warn + notify: condition first detected this run
|
|
# Strike 2 — warn + escalated notify: still present next cycle
|
|
# Strike 3 (WATCHDOG_APPDATA_STRIKE_LIMIT) — action cycle:
|
|
# WATCHDOG_APPDATA_TRUNCATE_LOGS=true → truncate *.log in-place, clear strikes
|
|
# WATCHDOG_APPDATA_TRUNCATE_LOGS=false → critical notify, hold strikes until resolved
|
|
# Condition resolves (growth stops / log drops below threshold) → strikes auto-clear
|
|
#
|
|
# Suppress Ceiling (WATCHDOG_APPDATA_SIZES)
|
|
# Containers in HOST*_WATCHDOG_APPDATA_SIZES suppress growth warnings while
|
|
# under their configured ceiling MB. Use ONLY when a container legitimately
|
|
# holds large stable data and would otherwise false-alarm (e.g. Tdarr cache).
|
|
# Zero-config growth detection covers everything else automatically.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Zero-config for new containers
|
|
# Growth rate detection requires no per-container configuration. Add a game
|
|
# server, spin up a new arr, install anything — it is monitored automatically
|
|
# from the second cycle after it appears. The suppress ceiling in conf is the
|
|
# exception, not the rule.
|
|
#
|
|
# Alert-only for data, truncate-only for logs
|
|
# Non-log growth (databases, game saves, caches) is detected and alerted but
|
|
# never touched. Only *.log / *.log.* files are candidates for truncation —
|
|
# and only when WATCHDOG_APPDATA_TRUNCATE_LOGS=true. Truncation zeroes the
|
|
# file in-place; the container keeps its open file handle, space is reclaimed
|
|
# immediately. Never deletes.
|
|
#
|
|
# Strike before acting
|
|
# One cycle of growth could be a legitimate library scan or game save burst.
|
|
# Three consecutive cycles of growth is a runaway. The strike system separates
|
|
# transient activity from sustained problems before any action fires.
|
|
#
|
|
# Silent when healthy
|
|
# Produces no output when all checks pass. Loud only when something needs
|
|
# attention.
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# WATCHDOG_CHECK_APPDATA
|
|
# Master toggle for all appdata checks (default: true)
|
|
#
|
|
# WATCHDOG_APPDATA_PATHS
|
|
# Array of paths to scan (e.g. "/mnt/docker-unraid/appdata")
|
|
#
|
|
# WATCHDOG_APPDATA_GROWTH_GB
|
|
# Per-cycle growth threshold in GB — flag containers growing more than this (default: 2)
|
|
#
|
|
# WATCHDOG_APPDATA_LOG_MAX_GB
|
|
# Absolute *.log file size alert threshold in GB (default: 2)
|
|
#
|
|
# WATCHDOG_APPDATA_TRUNCATE_LOGS
|
|
# Auto-truncate oversized *.log files on action cycle (default: false)
|
|
#
|
|
# WATCHDOG_APPDATA_STRIKE_LIMIT
|
|
# Consecutive cycles before action fires (default: 3)
|
|
#
|
|
# WATCHDOG_APPDATA_GROWTH_FILE
|
|
# Per-container size baseline — /tmp resets on reboot (correct: stale baseline
|
|
# after reboot would give false growth readings on first cycle)
|
|
#
|
|
# STORAGE_WATCHDOG_STATE_FILE
|
|
# Strike counts for this script — /tmp resets on reboot
|
|
#
|
|
# host*.conf (aliased by detect_hosts())
|
|
#
|
|
# HOST*_WATCHDOG_APPDATA_SIZES
|
|
# Per-container growth suppress ceilings in MB. Suppress growth alerts while
|
|
# a container's dir stays below this ceiling. Only needed when a container
|
|
# legitimately has large stable data. Growth detection covers everything else.
|
|
#
|
|
# ==============================================================================================
|
|
# STATE FILES
|
|
# ==============================================================================================
|
|
#
|
|
# STORAGE_WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot ✅)
|
|
# WATCHDOG_APPDATA_GROWTH_FILE — per-container size baseline (/tmp — resets on reboot ✅)
|
|
#
|
|
# /tmp files reset on reboot — correct. Pre-reboot strikes and growth baselines are
|
|
# meaningless after a reboot. Both rebuild cleanly from cycle 1.
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# storage_watchdog.sh
|
|
# Single-pass storage health check. Silent if all healthy.
|
|
#
|
|
# storage_watchdog.sh --dry-run
|
|
# Run all checks without truncating anything. Shows what would be actioned.
|
|
#
|
|
# storage_watchdog.sh --status
|
|
# Show configuration, active strikes, and growth baseline status. Then exit.
|
|
#
|
|
# storage_watchdog.sh --log
|
|
# Verbose output — every container checked, every size comparison, every decision.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
|
|
source "$SCRIPT_DIR/../../load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
acquire_lock
|
|
detect_hosts
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be truncated"
|
|
|
|
touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null
|
|
touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null
|
|
|
|
# ━━━ Strike helpers ━━━
|
|
get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; }
|
|
set_strikes() {
|
|
local key="$1" count="$2" file="$3"
|
|
if grep -q "^${key}:" "$file" 2>/dev/null; then
|
|
sed -i "s|^${key}:.*|${key}:${count}|" "$file"
|
|
else
|
|
echo "${key}:${count}" >> "$file"
|
|
fi
|
|
}
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY STORAGE WATCHDOG STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_GEAR Appdata check: ${WATCHDOG_CHECK_APPDATA:-true}"
|
|
echo "$ICON_GEAR Paths: ${WATCHDOG_APPDATA_PATHS[*]:-none}"
|
|
echo "$ICON_GEAR Growth thresh: ${WATCHDOG_APPDATA_GROWTH_GB:-2}GB/cycle"
|
|
echo "$ICON_GEAR Log max: ${WATCHDOG_APPDATA_LOG_MAX_GB:-2}GB"
|
|
echo "$ICON_GEAR Truncate logs: ${WATCHDOG_APPDATA_TRUNCATE_LOGS:-false}"
|
|
echo "$ICON_GEAR Strike limit: ${WATCHDOG_APPDATA_STRIKE_LIMIT:-3}"
|
|
echo ""
|
|
echo "── Active Strikes ──"
|
|
if [[ -s "$STORAGE_WATCHDOG_STATE_FILE" ]]; then
|
|
while IFS=':' read -r _sk _sv; do
|
|
_sv_clean="${_sv//[^0-9]/}"
|
|
[[ "${_sv_clean:-0}" -gt 0 ]] && echo " $_sk → $_sv_clean strikes"
|
|
done < "$STORAGE_WATCHDOG_STATE_FILE"
|
|
else
|
|
echo " none"
|
|
fi
|
|
echo ""
|
|
echo "── Growth Baseline ──"
|
|
if [[ -s "$WATCHDOG_APPDATA_GROWTH_FILE" ]]; then
|
|
_entries=$(wc -l < "$WATCHDOG_APPDATA_GROWTH_FILE")
|
|
_age=$(( $(date +%s) - $(stat -c %Y "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null || echo 0) ))
|
|
echo " Entries: $_entries containers Age: $(( _age / 60 ))m ago"
|
|
else
|
|
echo " No baseline yet (builds on first cycle after boot)"
|
|
fi
|
|
echo ""
|
|
echo "── Suppress Ceilings (this host) ──"
|
|
if [[ ${#WATCHDOG_APPDATA_SIZES[@]} -gt 0 ]]; then
|
|
for _c in "${!WATCHDOG_APPDATA_SIZES[@]}"; do
|
|
_ceil_gb=$(awk "BEGIN {printf \"%.0f\", ${WATCHDOG_APPDATA_SIZES[$_c]} / 1024}")
|
|
echo " $_c → ${_ceil_gb}GB"
|
|
done
|
|
else
|
|
echo " none configured"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Appdata Size Monitoring ━━━
|
|
# ==============================================================================================
|
|
[[ "$WATCHDOG_CHECK_APPDATA" != "true" ]] && exit 0
|
|
|
|
echo "━━━ $ICON_DISK Storage Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
|
|
WARNINGS=0
|
|
|
|
_STRIKE_LIMIT=${WATCHDOG_APPDATA_STRIKE_LIMIT:-3}
|
|
_GROWTH_MB=$(( ${WATCHDOG_APPDATA_GROWTH_GB:-2} * 1024 ))
|
|
_LOG_KB=$(( ${WATCHDOG_APPDATA_LOG_MAX_GB:-2} * 1024 * 1024 ))
|
|
|
|
# Tracks files handled by Part 1 to prevent duplicate alerts in Part 2
|
|
declare -A _HANDLED=()
|
|
|
|
for _appdata_path in "${WATCHDOG_APPDATA_PATHS[@]:-}"; do
|
|
[[ -z "$_appdata_path" || ! -d "$_appdata_path" ]] && continue
|
|
|
|
# ── Part 1: Growth rate scan ──────────────────────────────────────────────────────────────
|
|
declare -A _PREV=()
|
|
if [[ -f "$WATCHDOG_APPDATA_GROWTH_FILE" ]]; then
|
|
while IFS='|' read -r _cn _cs _; do
|
|
[[ -n "$_cn" ]] && _PREV["$_cn"]="$_cs"
|
|
done < "$WATCHDOG_APPDATA_GROWTH_FILE"
|
|
fi
|
|
|
|
_growth_tmp=$(mktemp 2>/dev/null) || _growth_tmp=""
|
|
_now=$(date +%s)
|
|
|
|
while IFS= read -r _du_line; do
|
|
_curr_mb=$(echo "$_du_line" | awk '{print $1}')
|
|
_cdir=$(echo "$_du_line" | awk '{print $2}')
|
|
_cname=$(basename "$_cdir")
|
|
[[ -z "$_cname" || "$_cname" == "*" ]] && continue
|
|
|
|
[[ -n "$_growth_tmp" ]] && echo "${_cname}|${_curr_mb}|${_now}" >> "$_growth_tmp"
|
|
|
|
_prev_mb="${_PREV[$_cname]:-}"
|
|
[[ -z "$_prev_mb" ]] && continue # First run after boot — building baseline
|
|
|
|
_growth_mb=$(( _curr_mb - _prev_mb ))
|
|
_safe=$(echo "$_cname" | tr -cd '[:alnum:]_')
|
|
|
|
# Condition resolved — growth stopped, clear strikes
|
|
if [[ "$_growth_mb" -le 0 ]]; then
|
|
_existing=$(get_strikes "appdata_growth_${_safe}" "$STORAGE_WATCHDOG_STATE_FILE")
|
|
_existing="${_existing//[^0-9]/}"
|
|
[[ "${_existing:-0}" -gt 0 ]] && \
|
|
set_strikes "appdata_growth_${_safe}" 0 "$STORAGE_WATCHDOG_STATE_FILE"
|
|
continue
|
|
fi
|
|
|
|
# Check suppress ceiling
|
|
_ceiling="${WATCHDOG_APPDATA_SIZES[$_cname]:-}"
|
|
if [[ -n "$_ceiling" && "$_curr_mb" -lt "$_ceiling" ]]; then
|
|
log "$_cname — growth suppressed (${_curr_mb}MB < ${_ceiling}MB ceiling)"
|
|
continue
|
|
fi
|
|
|
|
# Growth exceeds threshold — strike logic
|
|
if [[ "$_growth_mb" -ge "$_GROWTH_MB" ]]; then
|
|
_growth_gb=$(awk "BEGIN {printf \"%.1f\", $_growth_mb / 1024}")
|
|
_curr_gb=$(awk "BEGIN {printf \"%.1f\", $_curr_mb / 1024}")
|
|
|
|
_strikes=$(get_strikes "appdata_growth_${_safe}" "$STORAGE_WATCHDOG_STATE_FILE")
|
|
_strikes="${_strikes//[^0-9]/}"; _strikes="${_strikes:-0}"
|
|
|
|
[[ "$_strikes" -lt "$_STRIKE_LIMIT" ]] && {
|
|
_strikes=$(( _strikes + 1 ))
|
|
set_strikes "appdata_growth_${_safe}" "$_strikes" "$STORAGE_WATCHDOG_STATE_FILE"
|
|
}
|
|
|
|
warn "$_cname — grew ${_growth_gb}GB this cycle (total: ${_curr_gb}GB) [strike ${_strikes}/${_STRIKE_LIMIT}]"
|
|
(( WARNINGS++ ))
|
|
|
|
# Focused *.log scan inside the growing container's dir
|
|
_found_logs=()
|
|
while IFS= read -r _lf; do
|
|
[[ -n "$_lf" ]] && _found_logs+=("$_lf")
|
|
done < <(find "$_cdir" -maxdepth 3 -type f \
|
|
\( -name "*.log" -o -name "*.log.*" \) \
|
|
-size +${_LOG_KB}k -printf "%s %p\n" 2>/dev/null)
|
|
|
|
_log_summary=""
|
|
[[ ${#_found_logs[@]} -gt 0 ]] && _log_summary=$(printf '%s\n' "${_found_logs[@]}" | \
|
|
awk '{printf "%.1fGB %s | ", $1/1073741824, $2}' | head -c 200)
|
|
|
|
if [[ "$_strikes" -lt "$_STRIKE_LIMIT" ]]; then
|
|
if [[ -n "$_log_summary" ]]; then
|
|
notify "$_cname grew ${_growth_gb}GB on $(hostname) [strike ${_strikes}/${_STRIKE_LIMIT}] — logs: ${_log_summary}" \
|
|
"Storage Watchdog" "warning"
|
|
else
|
|
notify "$_cname grew ${_growth_gb}GB on $(hostname) [strike ${_strikes}/${_STRIKE_LIMIT}] — data growth, no log files" \
|
|
"Storage Watchdog" "warning"
|
|
fi
|
|
else
|
|
# Action cycle
|
|
error "$_cname — growth strike limit reached (${_STRIKE_LIMIT} consecutive cycles, ${_growth_gb}GB this cycle)"
|
|
if [[ "$WATCHDOG_APPDATA_TRUNCATE_LOGS" == "true" && ${#_found_logs[@]} -gt 0 ]]; then
|
|
for _lf_entry in "${_found_logs[@]}"; do
|
|
_lf_path=$(echo "$_lf_entry" | cut -d' ' -f2-)
|
|
_lf_gb=$(echo "$_lf_entry" | awk '{printf "%.1f", $1/1073741824}')
|
|
_HANDLED["$_lf_path"]=1
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would truncate $_lf_path (${_lf_gb}GB)"
|
|
else
|
|
if truncate -s 0 "$_lf_path" 2>/dev/null; then
|
|
success "Truncated runaway log: $_lf_path (was ${_lf_gb}GB)"
|
|
notify "Truncated runaway log on $(hostname): $_lf_path (was ${_lf_gb}GB)" \
|
|
"Storage Watchdog" "warning"
|
|
else
|
|
warn "Failed to truncate $_lf_path"
|
|
fi
|
|
fi
|
|
done
|
|
set_strikes "appdata_growth_${_safe}" 0 "$STORAGE_WATCHDOG_STATE_FILE"
|
|
else
|
|
if [[ -n "$_log_summary" ]]; then
|
|
notify "$_cname appdata runaway on $(hostname) — ${_growth_gb}GB growth for ${_STRIKE_LIMIT} cycles — logs: ${_log_summary}" \
|
|
"Storage Watchdog" "critical"
|
|
else
|
|
notify "$_cname appdata runaway on $(hostname) — ${_growth_gb}GB growth for ${_STRIKE_LIMIT} cycles — data growth, manual investigation needed" \
|
|
"Storage Watchdog" "critical"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Mark found logs as handled to suppress Part 2 duplicates this cycle
|
|
for _lf_entry in "${_found_logs[@]}"; do
|
|
_HANDLED["$(echo "$_lf_entry" | cut -d' ' -f2-)"]=1
|
|
done
|
|
fi
|
|
done < <(du -sm "$_appdata_path"/*/ 2>/dev/null)
|
|
|
|
# Atomically update growth baseline
|
|
[[ -n "$_growth_tmp" ]] && mv "$_growth_tmp" "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null
|
|
|
|
# ── Part 2: Absolute log size scan ────────────────────────────────────────────────────────
|
|
# Catches *.log files already large but no longer actively growing this cycle.
|
|
# Same strike logic. Skips files already handled by Part 1 above.
|
|
declare -A _LOG_SEEN=()
|
|
|
|
while IFS= read -r _hit; do
|
|
[[ -z "$_hit" ]] && continue
|
|
_fpath=$(echo "$_hit" | cut -d' ' -f2-)
|
|
[[ -n "${_HANDLED[$_fpath]:-}" ]] && continue
|
|
|
|
_fsize_bytes=$(echo "$_hit" | awk '{print $1}')
|
|
_fsize_gb=$(awk "BEGIN {printf \"%.1f\", $_fsize_bytes / 1073741824}")
|
|
_safe_fkey=$(echo "$_fpath" | tr -cd '[:alnum:]_' | cut -c1-120)
|
|
_LOG_SEEN["appdata_log_${_safe_fkey}"]=1
|
|
|
|
_strikes=$(get_strikes "appdata_log_${_safe_fkey}" "$STORAGE_WATCHDOG_STATE_FILE")
|
|
_strikes="${_strikes//[^0-9]/}"; _strikes="${_strikes:-0}"
|
|
|
|
[[ "$_strikes" -lt "$_STRIKE_LIMIT" ]] && {
|
|
_strikes=$(( _strikes + 1 ))
|
|
set_strikes "appdata_log_${_safe_fkey}" "$_strikes" "$STORAGE_WATCHDOG_STATE_FILE"
|
|
}
|
|
|
|
warn "Oversized log: $_fpath (${_fsize_gb}GB) [strike ${_strikes}/${_STRIKE_LIMIT}]"
|
|
(( WARNINGS++ ))
|
|
|
|
if [[ "$_strikes" -lt "$_STRIKE_LIMIT" ]]; then
|
|
notify "Oversized log on $(hostname): $_fpath (${_fsize_gb}GB) [strike ${_strikes}/${_STRIKE_LIMIT}]" \
|
|
"Storage Watchdog" "warning"
|
|
else
|
|
error "Oversized log persists for ${_STRIKE_LIMIT} cycles: $_fpath (${_fsize_gb}GB)"
|
|
if [[ "$WATCHDOG_APPDATA_TRUNCATE_LOGS" == "true" ]]; then
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would truncate $_fpath (${_fsize_gb}GB)"
|
|
else
|
|
if truncate -s 0 "$_fpath" 2>/dev/null; then
|
|
success "Truncated oversized log: $_fpath (was ${_fsize_gb}GB)"
|
|
notify "Truncated oversized log on $(hostname): $_fpath (was ${_fsize_gb}GB)" \
|
|
"Storage Watchdog" "warning"
|
|
set_strikes "appdata_log_${_safe_fkey}" 0 "$STORAGE_WATCHDOG_STATE_FILE"
|
|
unset "_LOG_SEEN[appdata_log_${_safe_fkey}]"
|
|
else
|
|
warn "Failed to truncate $_fpath"
|
|
notify "Oversized log on $(hostname): $_fpath (${_fsize_gb}GB) — ${_STRIKE_LIMIT} cycles, truncate failed" \
|
|
"Storage Watchdog" "critical"
|
|
fi
|
|
fi
|
|
else
|
|
notify "Oversized log on $(hostname): $_fpath (${_fsize_gb}GB) — ${_STRIKE_LIMIT} cycles, manual intervention needed" \
|
|
"Storage Watchdog" "critical"
|
|
fi
|
|
fi
|
|
done < <(find "$_appdata_path" -maxdepth 4 -type f \
|
|
\( -name "*.log" -o -name "*.log.*" \) \
|
|
-size +${_LOG_KB}k -printf "%s %p\n" 2>/dev/null)
|
|
|
|
# Auto-clear strikes for log files no longer oversized this cycle
|
|
while IFS=':' read -r _sk _sv; do
|
|
[[ "$_sk" != appdata_log_* ]] && continue
|
|
_sv_clean="${_sv//[^0-9]/}"
|
|
[[ "${_sv_clean:-0}" -eq 0 ]] && continue
|
|
[[ -n "${_LOG_SEEN[$_sk]:-}" ]] && continue
|
|
set_strikes "$_sk" 0 "$STORAGE_WATCHDOG_STATE_FILE"
|
|
done < "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null
|
|
|
|
unset _LOG_SEEN
|
|
|
|
done
|
|
|
|
unset _PREV _HANDLED
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
if [[ "$WARNINGS" -gt 0 ]]; then
|
|
echo ""
|
|
echo "━━━ $ICON_DISK Storage Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_WATCHDOG Warnings: $WARNINGS"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
else
|
|
echo "Storage healthy ✅ ($(date '+%H:%M:%S'))"
|
|
fi
|