FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
bdcdfea175dfb3603c053d326c8a92fb2cc902fd
Varaverk/Watchdogs/System/storage_watchdog.sh
T
Gmer4Lfe 08fc551d9f Add verbose log() coverage across all watchdog scripts 
docker_watchdog: config dump at startup (thresholds/limits), skip list shown when active, per-container healthy log for Tier1 required + mem/CPU monitored containers
stability_watchdog: config dump with all tier thresholds, log() on pass for rootfs/log/tmp/load/zombies/NIC checks (previously silent on clean)
resource_watchdog: config dump with all pressure thresholds and container lists, log normal pressure state with live RAM/load values
system_watchdog: per-script timing on each child script run
network_watchdog: config dump (internet URL, DDNS domain/container, NPM URL, strike limit)
storage_watchdog: config dump (growth threshold, log max, paths, suppress ceilings)
webgui_watchdog: log nginx worker and php-fpm worker counts on healthy check
2026-06-01 20:56:46 -04:00

444 lines
21 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
# ==============================================================================================
# ================================= Storage Watchdog ===========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Pool and storage health monitoring — catches runaway data growth before it
# fills a pool. Runs as a single-pass script called by system_watchdog.sh each
# cycle. Sits between docker_watchdog.sh (container health) and
# stability_watchdog.sh (last line of defense). Never reboots — detects, alerts,
# and optionally remediates.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Appdata Size Monitoring
# Two complementary checks run every cycle:
#
# Part 1 — Growth rate (zero-config catch-all):
# Reads per-container dir totals via du, compares to previous cycle baseline.
# Any container growing more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused
# *.log scan inside that container's dir. No per-container config required —
# new containers are covered automatically. Baseline built on first cycle after
# boot; growth detection active from cycle 2.
#
# Part 2 — Absolute log size:
# Finds *.log / *.log.* files over WATCHDOG_APPDATA_LOG_MAX_GB anywhere in
# WATCHDOG_APPDATA_PATHS. Catches logs already large but no longer actively
# growing. Independent strike counter per file.
#
# Strike System
# Reuses the same strike pattern as CPU/HTTP checks in docker_watchdog.sh:
#
# Strike 1 — warn + notify: condition first detected this run
# Strike 2 — warn + escalated notify: still present next cycle
# Strike 3 (WATCHDOG_APPDATA_STRIKE_LIMIT) — action cycle:
# WATCHDOG_APPDATA_TRUNCATE_LOGS=true → truncate *.log in-place, clear strikes
# WATCHDOG_APPDATA_TRUNCATE_LOGS=false → critical notify, hold strikes until resolved
# Condition resolves (growth stops / log drops below threshold) → strikes auto-clear
#
# Suppress Ceiling (WATCHDOG_APPDATA_SIZES)
# Containers in HOST*_WATCHDOG_APPDATA_SIZES suppress growth warnings while
# under their configured ceiling MB. Use ONLY when a container legitimately
# holds large stable data and would otherwise false-alarm (e.g. Tdarr cache).
# Zero-config growth detection covers everything else automatically.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Zero-config for new containers
# Growth rate detection requires no per-container configuration. Add a game
# server, spin up a new arr, install anything — it is monitored automatically
# from the second cycle after it appears. The suppress ceiling in conf is the
# exception, not the rule.
#
# Alert-only for data, truncate-only for logs
# Non-log growth (databases, game saves, caches) is detected and alerted but
# never touched. Only *.log / *.log.* files are candidates for truncation —
# and only when WATCHDOG_APPDATA_TRUNCATE_LOGS=true. Truncation zeroes the
# file in-place; the container keeps its open file handle, space is reclaimed
# immediately. Never deletes.
#
# Strike before acting
# One cycle of growth could be a legitimate library scan or game save burst.
# Three consecutive cycles of growth is a runaway. The strike system separates
# transient activity from sustained problems before any action fires.
#
# Silent when healthy
# Produces no output when all checks pass. Loud only when something needs
# attention.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
# WATCHDOG_CHECK_APPDATA
# Master toggle for all appdata checks (default: true)
#
# WATCHDOG_APPDATA_PATHS
# Array of paths to scan (e.g. "/mnt/docker-unraid/appdata")
#
# WATCHDOG_APPDATA_GROWTH_GB
# Per-cycle growth threshold in GB — flag containers growing more than this (default: 2)
#
# WATCHDOG_APPDATA_LOG_MAX_GB
# Absolute *.log file size alert threshold in GB (default: 2)
#
# WATCHDOG_APPDATA_TRUNCATE_LOGS
# Auto-truncate oversized *.log files on action cycle (default: false)
#
# WATCHDOG_APPDATA_STRIKE_LIMIT
# Consecutive cycles before action fires (default: 3)
#
# WATCHDOG_APPDATA_GROWTH_FILE
# Per-container size baseline — /tmp resets on reboot (correct: stale baseline
# after reboot would give false growth readings on first cycle)
#
# STORAGE_WATCHDOG_STATE_FILE
# Strike counts for this script — /tmp resets on reboot
#
# host*.conf (aliased by detect_hosts())
#
# HOST*_WATCHDOG_APPDATA_SIZES
# Per-container growth suppress ceilings in MB. Suppress growth alerts while
# a container's dir stays below this ceiling. Only needed when a container
# legitimately has large stable data. Growth detection covers everything else.
#
# ==============================================================================================
# STATE FILES
# ==============================================================================================
#
# STORAGE_WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot ✅)
# WATCHDOG_APPDATA_GROWTH_FILE — per-container size baseline (/tmp — resets on reboot ✅)
#
# /tmp files reset on reboot — correct. Pre-reboot strikes and growth baselines are
# meaningless after a reboot. Both rebuild cleanly from cycle 1.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# storage_watchdog.sh
# Single-pass storage health check. Silent if all healthy.
#
# storage_watchdog.sh --dry-run
# Run all checks without truncating anything. Shows what would be actioned.
#
# storage_watchdog.sh --status
# Show configuration, active strikes, and growth baseline status. Then exit.
#
# storage_watchdog.sh --log
# Verbose output — every container checked, every size comparison, every decision.
#
# ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../../load_config.sh"
parse_args "$@"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
acquire_lock
detect_hosts
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be truncated"
log "$ICON_GEAR Config: growth-thresh=${WATCHDOG_APPDATA_GROWTH_GB:-2}GB/cycle log-max=${WATCHDOG_APPDATA_LOG_MAX_GB:-2}GB truncate=${WATCHDOG_APPDATA_TRUNCATE_LOGS:-false} strikes=${WATCHDOG_APPDATA_STRIKE_LIMIT:-3}"
log "$ICON_DISK Paths: ${WATCHDOG_APPDATA_PATHS[*]:-none configured}"
[[ ${#WATCHDOG_APPDATA_SIZES[@]} -gt 0 ]] && log "$ICON_GEAR Suppress ceilings: $(for k in "${!WATCHDOG_APPDATA_SIZES[@]}"; do printf '%s=%sMB ' "$k" "${WATCHDOG_APPDATA_SIZES[$k]}"; done)"
touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null
touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null
# ━━━ Strike helpers ━━━
get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"; }
set_strikes() {
local key="$1" count="$2" file="$3"
if grep -q "^${key}:" "$file" 2>/dev/null; then
sed -i "s|^${key}:.*|${key}:${count}|" "$file"
else
echo "${key}:${count}" >> "$file"
fi
}
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "━━━━━ $ICON_SUMMARY STORAGE WATCHDOG STATUS ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_GEAR Appdata check: ${WATCHDOG_CHECK_APPDATA:-true}"
echo "$ICON_GEAR Paths: ${WATCHDOG_APPDATA_PATHS[*]:-none}"
echo "$ICON_GEAR Growth thresh: ${WATCHDOG_APPDATA_GROWTH_GB:-2}GB/cycle"
echo "$ICON_GEAR Log max: ${WATCHDOG_APPDATA_LOG_MAX_GB:-2}GB"
echo "$ICON_GEAR Truncate logs: ${WATCHDOG_APPDATA_TRUNCATE_LOGS:-false}"
echo "$ICON_GEAR Strike limit: ${WATCHDOG_APPDATA_STRIKE_LIMIT:-3}"
echo ""
echo "── Active Strikes ──"
if [[ -s "$STORAGE_WATCHDOG_STATE_FILE" ]]; then
while IFS=':' read -r _sk _sv; do
_sv_clean="${_sv//[^0-9]/}"
[[ "${_sv_clean:-0}" -gt 0 ]] && echo " $_sk → $_sv_clean strikes"
done < "$STORAGE_WATCHDOG_STATE_FILE"
else
echo " none"
fi
echo ""
echo "── Growth Baseline ──"
if [[ -s "$WATCHDOG_APPDATA_GROWTH_FILE" ]]; then
_entries=$(wc -l < "$WATCHDOG_APPDATA_GROWTH_FILE")
_age=$(( $(date +%s) - $(stat -c %Y "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null || echo 0) ))
echo " Entries: $_entries containers Age: $(( _age / 60 ))m ago"
else
echo " No baseline yet (builds on first cycle after boot)"
fi
echo ""
echo "── Suppress Ceilings (this host) ──"
if [[ ${#WATCHDOG_APPDATA_SIZES[@]} -gt 0 ]]; then
for _c in "${!WATCHDOG_APPDATA_SIZES[@]}"; do
_ceil_gb=$(awk "BEGIN {printf \"%.0f\", ${WATCHDOG_APPDATA_SIZES[$_c]} / 1024}")
echo " $_c → ${_ceil_gb}GB"
done
else
echo " none configured"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━"
exit 0
fi
# ==============================================================================================
# ━━━ Appdata Size Monitoring ━━━
# ==============================================================================================
[[ "$WATCHDOG_CHECK_APPDATA" != "true" ]] && exit 0
echo "━━━ $ICON_DISK Storage Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
WARNINGS=0
_STRIKE_LIMIT=${WATCHDOG_APPDATA_STRIKE_LIMIT:-3}
_GROWTH_MB=$(( ${WATCHDOG_APPDATA_GROWTH_GB:-2} * 1024 ))
_LOG_KB=$(( ${WATCHDOG_APPDATA_LOG_MAX_GB:-2} * 1024 * 1024 ))
# Tracks files handled by Part 1 to prevent duplicate alerts in Part 2
declare -A _HANDLED=()
for _appdata_path in "${WATCHDOG_APPDATA_PATHS[@]:-}"; do
[[ -z "$_appdata_path" || ! -d "$_appdata_path" ]] && continue
# ── Part 1: Growth rate scan ──────────────────────────────────────────────────────────────
declare -A _PREV=()
if [[ -f "$WATCHDOG_APPDATA_GROWTH_FILE" ]]; then
while IFS='|' read -r _cn _cs _; do
[[ -n "$_cn" ]] && _PREV["$_cn"]="$_cs"
done < "$WATCHDOG_APPDATA_GROWTH_FILE"
fi
_growth_tmp=$(mktemp 2>/dev/null) || _growth_tmp=""
_now=$(date +%s)
while IFS= read -r _du_line; do
_curr_mb=$(echo "$_du_line" | awk '{print $1}')
_cdir=$(echo "$_du_line" | awk '{print $2}')
_cname=$(basename "$_cdir")
[[ -z "$_cname" || "$_cname" == "*" ]] && continue
[[ -n "$_growth_tmp" ]] && echo "${_cname}|${_curr_mb}|${_now}" >> "$_growth_tmp"
_prev_mb="${_PREV[$_cname]:-}"
[[ -z "$_prev_mb" ]] && continue # First run after boot — building baseline
_growth_mb=$(( _curr_mb - _prev_mb ))
_safe=$(echo "$_cname" | tr -cd '[:alnum:]_')
# Condition resolved — growth stopped, clear strikes
if [[ "$_growth_mb" -le 0 ]]; then
_existing=$(get_strikes "appdata_growth_${_safe}" "$STORAGE_WATCHDOG_STATE_FILE")
_existing="${_existing//[^0-9]/}"
[[ "${_existing:-0}" -gt 0 ]] && \
set_strikes "appdata_growth_${_safe}" 0 "$STORAGE_WATCHDOG_STATE_FILE"
continue
fi
# Check suppress ceiling
_ceiling="${WATCHDOG_APPDATA_SIZES[$_cname]:-}"
if [[ -n "$_ceiling" && "$_curr_mb" -lt "$_ceiling" ]]; then
log "$_cname — growth suppressed (${_curr_mb}MB < ${_ceiling}MB ceiling)"
continue
fi
# Growth exceeds threshold — strike logic
if [[ "$_growth_mb" -ge "$_GROWTH_MB" ]]; then
_growth_gb=$(awk "BEGIN {printf \"%.1f\", $_growth_mb / 1024}")
_curr_gb=$(awk "BEGIN {printf \"%.1f\", $_curr_mb / 1024}")
_strikes=$(get_strikes "appdata_growth_${_safe}" "$STORAGE_WATCHDOG_STATE_FILE")
_strikes="${_strikes//[^0-9]/}"; _strikes="${_strikes:-0}"
[[ "$_strikes" -lt "$_STRIKE_LIMIT" ]] && {
_strikes=$(( _strikes + 1 ))
set_strikes "appdata_growth_${_safe}" "$_strikes" "$STORAGE_WATCHDOG_STATE_FILE"
}
warn "$_cname — grew ${_growth_gb}GB this cycle (total: ${_curr_gb}GB) [strike ${_strikes}/${_STRIKE_LIMIT}]"
(( WARNINGS++ ))
# Focused *.log scan inside the growing container's dir
_found_logs=()
while IFS= read -r _lf; do
[[ -n "$_lf" ]] && _found_logs+=("$_lf")
done < <(find "$_cdir" -maxdepth 3 -type f \
\( -name "*.log" -o -name "*.log.*" \) \
-size +${_LOG_KB}k -printf "%s %p\n" 2>/dev/null)
_log_summary=""
[[ ${#_found_logs[@]} -gt 0 ]] && _log_summary=$(printf '%s\n' "${_found_logs[@]}" | \
awk '{printf "%.1fGB %s | ", $1/1073741824, $2}' | head -c 200)
if [[ "$_strikes" -lt "$_STRIKE_LIMIT" ]]; then
if [[ -n "$_log_summary" ]]; then
notify "$_cname grew ${_growth_gb}GB on $(hostname) [strike ${_strikes}/${_STRIKE_LIMIT}] — logs: ${_log_summary}" \
"Storage Watchdog" "warning"
else
notify "$_cname grew ${_growth_gb}GB on $(hostname) [strike ${_strikes}/${_STRIKE_LIMIT}] — data growth, no log files" \
"Storage Watchdog" "warning"
fi
else
# Action cycle
error "$_cname — growth strike limit reached (${_STRIKE_LIMIT} consecutive cycles, ${_growth_gb}GB this cycle)"
if [[ "$WATCHDOG_APPDATA_TRUNCATE_LOGS" == "true" && ${#_found_logs[@]} -gt 0 ]]; then
for _lf_entry in "${_found_logs[@]}"; do
_lf_path=$(echo "$_lf_entry" | cut -d' ' -f2-)
_lf_gb=$(echo "$_lf_entry" | awk '{printf "%.1f", $1/1073741824}')
_HANDLED["$_lf_path"]=1
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would truncate $_lf_path (${_lf_gb}GB)"
else
if truncate -s 0 "$_lf_path" 2>/dev/null; then
success "Truncated runaway log: $_lf_path (was ${_lf_gb}GB)"
notify "Truncated runaway log on $(hostname): $_lf_path (was ${_lf_gb}GB)" \
"Storage Watchdog" "warning"
else
warn "Failed to truncate $_lf_path"
fi
fi
done
set_strikes "appdata_growth_${_safe}" 0 "$STORAGE_WATCHDOG_STATE_FILE"
else
if [[ -n "$_log_summary" ]]; then
notify "$_cname appdata runaway on $(hostname) — ${_growth_gb}GB growth for ${_STRIKE_LIMIT} cycles — logs: ${_log_summary}" \
"Storage Watchdog" "critical"
else
notify "$_cname appdata runaway on $(hostname) — ${_growth_gb}GB growth for ${_STRIKE_LIMIT} cycles — data growth, manual investigation needed" \
"Storage Watchdog" "critical"
fi
fi
fi
# Mark found logs as handled to suppress Part 2 duplicates this cycle
for _lf_entry in "${_found_logs[@]}"; do
_HANDLED["$(echo "$_lf_entry" | cut -d' ' -f2-)"]=1
done
fi
done < <(du -sm "$_appdata_path"/*/ 2>/dev/null)
# Atomically update growth baseline
[[ -n "$_growth_tmp" ]] && mv "$_growth_tmp" "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null
# ── Part 2: Absolute log size scan ────────────────────────────────────────────────────────
# Catches *.log files already large but no longer actively growing this cycle.
# Same strike logic. Skips files already handled by Part 1 above.
declare -A _LOG_SEEN=()
while IFS= read -r _hit; do
[[ -z "$_hit" ]] && continue
_fpath=$(echo "$_hit" | cut -d' ' -f2-)
[[ -n "${_HANDLED[$_fpath]:-}" ]] && continue
_fsize_bytes=$(echo "$_hit" | awk '{print $1}')
_fsize_gb=$(awk "BEGIN {printf \"%.1f\", $_fsize_bytes / 1073741824}")
_safe_fkey=$(echo "$_fpath" | tr -cd '[:alnum:]_' | cut -c1-120)
_LOG_SEEN["appdata_log_${_safe_fkey}"]=1
_strikes=$(get_strikes "appdata_log_${_safe_fkey}" "$STORAGE_WATCHDOG_STATE_FILE")
_strikes="${_strikes//[^0-9]/}"; _strikes="${_strikes:-0}"
[[ "$_strikes" -lt "$_STRIKE_LIMIT" ]] && {
_strikes=$(( _strikes + 1 ))
set_strikes "appdata_log_${_safe_fkey}" "$_strikes" "$STORAGE_WATCHDOG_STATE_FILE"
}
warn "Oversized log: $_fpath (${_fsize_gb}GB) [strike ${_strikes}/${_STRIKE_LIMIT}]"
(( WARNINGS++ ))
if [[ "$_strikes" -lt "$_STRIKE_LIMIT" ]]; then
notify "Oversized log on $(hostname): $_fpath (${_fsize_gb}GB) [strike ${_strikes}/${_STRIKE_LIMIT}]" \
"Storage Watchdog" "warning"
else
error "Oversized log persists for ${_STRIKE_LIMIT} cycles: $_fpath (${_fsize_gb}GB)"
if [[ "$WATCHDOG_APPDATA_TRUNCATE_LOGS" == "true" ]]; then
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would truncate $_fpath (${_fsize_gb}GB)"
else
if truncate -s 0 "$_fpath" 2>/dev/null; then
success "Truncated oversized log: $_fpath (was ${_fsize_gb}GB)"
notify "Truncated oversized log on $(hostname): $_fpath (was ${_fsize_gb}GB)" \
"Storage Watchdog" "warning"
set_strikes "appdata_log_${_safe_fkey}" 0 "$STORAGE_WATCHDOG_STATE_FILE"
unset "_LOG_SEEN[appdata_log_${_safe_fkey}]"
else
warn "Failed to truncate $_fpath"
notify "Oversized log on $(hostname): $_fpath (${_fsize_gb}GB) — ${_STRIKE_LIMIT} cycles, truncate failed" \
"Storage Watchdog" "critical"
fi
fi
else
notify "Oversized log on $(hostname): $_fpath (${_fsize_gb}GB) — ${_STRIKE_LIMIT} cycles, manual intervention needed" \
"Storage Watchdog" "critical"
fi
fi
done < <(find "$_appdata_path" -maxdepth 4 -type f \
\( -name "*.log" -o -name "*.log.*" \) \
-size +${_LOG_KB}k -printf "%s %p\n" 2>/dev/null)
# Auto-clear strikes for log files no longer oversized this cycle
while IFS=':' read -r _sk _sv; do
[[ "$_sk" != appdata_log_* ]] && continue
_sv_clean="${_sv//[^0-9]/}"
[[ "${_sv_clean:-0}" -eq 0 ]] && continue
[[ -n "${_LOG_SEEN[$_sk]:-}" ]] && continue
set_strikes "$_sk" 0 "$STORAGE_WATCHDOG_STATE_FILE"
done < "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null
unset _LOG_SEEN
done
unset _PREV _HANDLED
# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
if [[ "$WARNINGS" -gt 0 ]]; then
echo ""
echo "━━━ $ICON_DISK Storage Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
echo "$ICON_WATCHDOG Warnings: $WARNINGS"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
else
echo "Storage healthy ✅ ($(date '+%H:%M:%S'))"
fi
Reference in New Issue View Git Blame Copy Permalink