Varaverk/Watchdogs/System/storage_watchdog.sh

#!/bin/bash
# ==============================================================================================
# ================================= Storage Watchdog ===========================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Pool and storage health monitoring — catches runaway data growth before it
# fills a pool. Runs as a single-pass script called by system_watchdog.sh each
# cycle. Sits between docker_watchdog.sh (container health) and
# stability_watchdog.sh (last line of defense). Never reboots — detects, alerts,
# and optionally remediates.
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Appdata Size Monitoring
#   Two complementary checks run every cycle:
#
#   Part 1 — Growth rate (zero-config catch-all):
#     Reads per-container dir totals via du, compares to previous cycle baseline.
#     Any container growing more than WATCHDOG_APPDATA_GROWTH_GB triggers a focused
#     *.log scan inside that container's dir. No per-container config required —
#     new containers are covered automatically. Baseline built on first cycle after
#     boot; growth detection active from cycle 2.
#
#   Part 2 — Absolute log size:
#     Finds *.log / *.log.* files over WATCHDOG_APPDATA_LOG_MAX_GB anywhere in
#     WATCHDOG_APPDATA_PATHS. Catches logs already large but no longer actively
#     growing. Independent strike counter per file.
#
# Strike System
#   Reuses the same strike pattern as CPU/HTTP checks in docker_watchdog.sh:
#
#   Strike 1 — warn + notify: condition first detected this run
#   Strike 2 — warn + escalated notify: still present next cycle
#   Strike 3 (WATCHDOG_APPDATA_STRIKE_LIMIT) — action cycle:
#     WATCHDOG_APPDATA_TRUNCATE_LOGS=true  → truncate *.log in-place, clear strikes
#     WATCHDOG_APPDATA_TRUNCATE_LOGS=false → critical notify, hold strikes until resolved
#   Condition resolves (growth stops / log drops below threshold) → strikes auto-clear
#
# Suppress Ceiling (WATCHDOG_APPDATA_SIZES)
#   Containers in HOST*_WATCHDOG_APPDATA_SIZES suppress growth warnings while
#   under their configured ceiling MB. Use ONLY when a container legitimately
#   holds large stable data and would otherwise false-alarm (e.g. Tdarr cache).
#   Zero-config growth detection covers everything else automatically.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Zero-config for new containers
#   Growth rate detection requires no per-container configuration. Add a game
#   server, spin up a new arr, install anything — it is monitored automatically
#   from the second cycle after it appears. The suppress ceiling in conf is the
#   exception, not the rule.
#
# Alert-only for data, truncate-only for logs
#   Non-log growth (databases, game saves, caches) is detected and alerted but
#   never touched. Only *.log / *.log.* files are candidates for truncation —
#   and only when WATCHDOG_APPDATA_TRUNCATE_LOGS=true. Truncation zeroes the
#   file in-place; the container keeps its open file handle, space is reclaimed
#   immediately. Never deletes.
#
# Strike before acting
#   One cycle of growth could be a legitimate library scan or game save burst.
#   Three consecutive cycles of growth is a runaway. The strike system separates
#   transient activity from sustained problems before any action fires.
#
# Silent when healthy
#   Produces no output when all checks pass. Loud only when something needs
#   attention.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Enforcement
#   Truncating container-owned log files requires root.
#
# Lock Acquisition
#   acquire_lock() prevents concurrent runs. Two instances would race on the
#   strike state file and the growth baseline, double-counting strikes and
#   potentially truncating a file one cycle early.
#
# Host Detection
#   detect_hosts() aliases HOST*_WATCHDOG_APPDATA_SIZES to the correct host's
#   suppress ceilings.
#
# WATCHDOG_CHECK_APPDATA Toggle
#   Exits cleanly before any scanning when the master toggle is off.
#
# Path Existence Guard
#   Every entry in WATCHDOG_APPDATA_PATHS is skipped unless it is a non-empty
#   string naming a real directory. An unconfigured array cannot cause a scan
#   from an unintended location.
#
# Truncate-Never-Delete
#   Action is always truncate -s 0, never rm. The container keeps its open file
#   handle and space is reclaimed immediately, so a still-running service does
#   not lose its log destination mid-write.
#
# Filename Restriction
#   Only *.log and *.log.* files are ever truncation candidates. Databases,
#   caches, game saves and every other growing file are alert-only — detected
#   and reported, never modified.
#
# Truncation Opt-In
#   WATCHDOG_APPDATA_TRUNCATE_LOGS defaults to false. Without it explicitly
#   enabled the action cycle escalates to a critical notification and holds
#   strikes rather than touching any file.
#
# Strike Threshold
#   Nothing acts on first detection. WATCHDOG_APPDATA_STRIKE_LIMIT consecutive
#   cycles are required, separating a legitimate library scan or save burst
#   from a genuine runaway. Strikes auto-clear when the condition resolves.
#
# Suppress Ceiling
#   Containers listed in WATCHDOG_APPDATA_SIZES are exempt from growth alerts
#   while under their configured ceiling — prevents known-large stable data
#   from generating recurring false alarms.
#
# Dry Run Support
#   --dry-run reports every truncation that would occur and performs none.
#
# Atomic Baseline Update
#   The growth baseline is written to a temp file and moved into place, so an
#   interrupted run cannot leave a half-written baseline that would read as
#   false growth on the next cycle.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   WATCHDOG_CHECK_APPDATA
#       Master toggle for all appdata checks (default: true)
#
#   WATCHDOG_APPDATA_PATHS
#       Array of paths to scan (e.g. "/mnt/docker-unraid/appdata")
#
#   WATCHDOG_APPDATA_GROWTH_GB
#       Per-cycle growth threshold in GB — flag containers growing more than this (default: 2)
#
#   WATCHDOG_APPDATA_LOG_MAX_GB
#       Absolute *.log file size alert threshold in GB (default: 2)
#
#   WATCHDOG_APPDATA_TRUNCATE_LOGS
#       Auto-truncate oversized *.log files on action cycle (default: false)
#
#   WATCHDOG_APPDATA_STRIKE_LIMIT
#       Consecutive cycles before action fires (default: 3)
#
#   WATCHDOG_APPDATA_GROWTH_FILE
#       Per-container size baseline — /tmp resets on reboot (correct: stale baseline
#       after reboot would give false growth readings on first cycle)
#
#   STORAGE_WATCHDOG_STATE_FILE
#       Strike counts for this script — /tmp resets on reboot
#
# host*.conf (aliased by detect_hosts())
#
#   HOST*_WATCHDOG_APPDATA_SIZES
#       Per-container growth suppress ceilings in MB. Suppress growth alerts while
#       a container's dir stays below this ceiling. Only needed when a container
#       legitimately has large stable data. Growth detection covers everything else.
#
# ==============================================================================================
# STATE FILES
# ==============================================================================================
#
#   STORAGE_WATCHDOG_STATE_FILE    — strike counts (/tmp — resets on reboot ✅)
#   WATCHDOG_APPDATA_GROWTH_FILE   — per-container size baseline (/tmp — resets on reboot ✅)
#
# /tmp files reset on reboot — correct. Pre-reboot strikes and growth baselines are
# meaningless after a reboot. Both rebuild cleanly from cycle 1.
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# storage_watchdog.sh
#     Single-pass storage health check. Silent if all healthy.
#
# storage_watchdog.sh --dry-run
#     Run all checks without truncating anything. Shows what would be actioned.
#
# storage_watchdog.sh --status
#     Show configuration, active strikes, and growth baseline status. Then exit.
#
# storage_watchdog.sh --log
#     Verbose output — every container checked, every size comparison, every decision.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source "$SCRIPT_DIR/../../load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock
detect_hosts

[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be truncated"

log "$ICON_GEAR Config: growth-thresh=${WATCHDOG_APPDATA_GROWTH_GB:-2}GB/cycle log-max=${WATCHDOG_APPDATA_LOG_MAX_GB:-2}GB truncate=${WATCHDOG_APPDATA_TRUNCATE_LOGS:-false} strikes=${WATCHDOG_APPDATA_STRIKE_LIMIT:-3}"
log "$ICON_DISK Paths:  ${WATCHDOG_APPDATA_PATHS[*]:-none configured}"
[[ ${#WATCHDOG_APPDATA_SIZES[@]} -gt 0 ]] && log "$ICON_GEAR Suppress ceilings: $(for k in "${!WATCHDOG_APPDATA_SIZES[@]}"; do printf '%s=%sMB ' "$k" "${WATCHDOG_APPDATA_SIZES[$k]}"; done)"

touch "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null
touch "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null

# ━━━ Strike helpers — wrap common.sh's wd_state_get()/wd_state_set() ━━━
get_strikes() { wd_state_get "$1" "$2"; }
set_strikes() { wd_state_set "$1" "$2" "$3"; }

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo ""
    echo "━━━━━ $ICON_SUMMARY STORAGE WATCHDOG STATUS ━━━━━"
    echo "$ICON_HOST       Identity:       $MY_ID ($LOCAL_SERVER_NAME)"
    echo "$ICON_GEAR       Appdata check:  ${WATCHDOG_CHECK_APPDATA:-true}"
    echo "$ICON_GEAR       Paths:          ${WATCHDOG_APPDATA_PATHS[*]:-none}"
    echo "$ICON_GEAR       Growth thresh:  ${WATCHDOG_APPDATA_GROWTH_GB:-2}GB/cycle"
    echo "$ICON_GEAR       Log max:        ${WATCHDOG_APPDATA_LOG_MAX_GB:-2}GB"
    echo "$ICON_GEAR       Truncate logs:  ${WATCHDOG_APPDATA_TRUNCATE_LOGS:-false}"
    echo "$ICON_GEAR       Strike limit:   ${WATCHDOG_APPDATA_STRIKE_LIMIT:-3}"
    echo ""
    echo "── Active Strikes ──"
    if [[ -s "$STORAGE_WATCHDOG_STATE_FILE" ]]; then
        while IFS=':' read -r _sk _sv; do
            _sv_clean="${_sv//[^0-9]/}"
            [[ "${_sv_clean:-0}" -gt 0 ]] && echo "  $_sk → $_sv_clean strikes"
        done < "$STORAGE_WATCHDOG_STATE_FILE"
    else
        echo "  none"
    fi
    echo ""
    echo "── Growth Baseline ──"
    if [[ -s "$WATCHDOG_APPDATA_GROWTH_FILE" ]]; then
        _entries=$(wc -l < "$WATCHDOG_APPDATA_GROWTH_FILE")
        _age=$(( $(date +%s) - $(stat -c %Y "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null || echo 0) ))
        echo "  Entries: $_entries containers   Age: $(( _age / 60 ))m ago"
    else
        echo "  No baseline yet (builds on first cycle after boot)"
    fi
    echo ""
    echo "── Suppress Ceilings (this host) ──"
    if [[ ${#WATCHDOG_APPDATA_SIZES[@]} -gt 0 ]]; then
        for _c in "${!WATCHDOG_APPDATA_SIZES[@]}"; do
            _ceil_gb=$(awk "BEGIN {printf \"%.0f\", ${WATCHDOG_APPDATA_SIZES[$_c]} / 1024}")
            echo "  $_c → ${_ceil_gb}GB"
        done
    else
        echo "  none configured"
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Appdata Size Monitoring ━━━
# ==============================================================================================
[[ "$WATCHDOG_CHECK_APPDATA" != "true" ]] && exit 0

echo "━━━ $ICON_DISK Storage Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"

WARNINGS=0

_STRIKE_LIMIT=${WATCHDOG_APPDATA_STRIKE_LIMIT:-3}
_GROWTH_MB=$(( ${WATCHDOG_APPDATA_GROWTH_GB:-2} * 1024 ))
_LOG_KB=$(( ${WATCHDOG_APPDATA_LOG_MAX_GB:-2} * 1024 * 1024 ))

# Tracks files handled by Part 1 to prevent duplicate alerts in Part 2
declare -A _HANDLED=()

for _appdata_path in "${WATCHDOG_APPDATA_PATHS[@]:-}"; do
    [[ -z "$_appdata_path" || ! -d "$_appdata_path" ]] && continue

    # ── Part 1: Growth rate scan ──────────────────────────────────────────────────────────────
    declare -A _PREV=()
    if [[ -f "$WATCHDOG_APPDATA_GROWTH_FILE" ]]; then
        while IFS='|' read -r _cn _cs _; do
            [[ -n "$_cn" ]] && _PREV["$_cn"]="$_cs"
        done < "$WATCHDOG_APPDATA_GROWTH_FILE"
    fi

    _growth_tmp=$(mktemp 2>/dev/null) || _growth_tmp=""
    _now=$(date +%s)

    while IFS= read -r _du_line; do
        _curr_mb=$(echo "$_du_line" | awk '{print $1}')
        _cdir=$(echo "$_du_line" | awk '{print $2}')
        _cname=$(basename "$_cdir")
        [[ -z "$_cname" || "$_cname" == "*" ]] && continue

        [[ -n "$_growth_tmp" ]] && echo "${_cname}|${_curr_mb}|${_now}" >> "$_growth_tmp"

        _prev_mb="${_PREV[$_cname]:-}"
        [[ -z "$_prev_mb" ]] && continue  # First run after boot — building baseline

        _growth_mb=$(( _curr_mb - _prev_mb ))
        _safe=$(echo "$_cname" | tr -cd '[:alnum:]_')

        # Condition resolved — growth stopped, clear strikes
        if [[ "$_growth_mb" -le 0 ]]; then
            _existing=$(get_strikes "appdata_growth_${_safe}" "$STORAGE_WATCHDOG_STATE_FILE")
            _existing="${_existing//[^0-9]/}"
            [[ "${_existing:-0}" -gt 0 ]] && \
                set_strikes "appdata_growth_${_safe}" 0 "$STORAGE_WATCHDOG_STATE_FILE"
            continue
        fi

        # Check suppress ceiling
        _ceiling="${WATCHDOG_APPDATA_SIZES[$_cname]:-}"
        if [[ -n "$_ceiling" && "$_curr_mb" -lt "$_ceiling" ]]; then
            log "$_cname — growth suppressed (${_curr_mb}MB < ${_ceiling}MB ceiling)"
            continue
        fi

        # Growth exceeds threshold — strike logic
        if [[ "$_growth_mb" -ge "$_GROWTH_MB" ]]; then
            _growth_gb=$(awk "BEGIN {printf \"%.1f\", $_growth_mb / 1024}")
            _curr_gb=$(awk "BEGIN {printf \"%.1f\", $_curr_mb / 1024}")

            _strikes=$(get_strikes "appdata_growth_${_safe}" "$STORAGE_WATCHDOG_STATE_FILE")
            _strikes="${_strikes//[^0-9]/}"; _strikes="${_strikes:-0}"

            [[ "$_strikes" -lt "$_STRIKE_LIMIT" ]] && {
                _strikes=$(( _strikes + 1 ))
                set_strikes "appdata_growth_${_safe}" "$_strikes" "$STORAGE_WATCHDOG_STATE_FILE"
            }

            warn "$_cname — grew ${_growth_gb}GB this cycle (total: ${_curr_gb}GB) [strike ${_strikes}/${_STRIKE_LIMIT}]"
            (( WARNINGS++ ))

            # Focused *.log scan inside the growing container's dir
            _found_logs=()
            while IFS= read -r _lf; do
                [[ -n "$_lf" ]] && _found_logs+=("$_lf")
            done < <(find "$_cdir" -maxdepth 3 -type f \
                \( -name "*.log" -o -name "*.log.*" \) \
                -size +${_LOG_KB}k -printf "%s %p\n" 2>/dev/null)

            _log_summary=""
            [[ ${#_found_logs[@]} -gt 0 ]] && _log_summary=$(printf '%s\n' "${_found_logs[@]}" | \
                awk '{printf "%.1fGB %s | ", $1/1073741824, $2}' | head -c 200)

            if [[ "$_strikes" -lt "$_STRIKE_LIMIT" ]]; then
                if [[ -n "$_log_summary" ]]; then
                    notify "$_cname grew ${_growth_gb}GB on $(hostname) [strike ${_strikes}/${_STRIKE_LIMIT}] — logs: ${_log_summary}" \
                        "Storage Watchdog" "warning"
                else
                    notify "$_cname grew ${_growth_gb}GB on $(hostname) [strike ${_strikes}/${_STRIKE_LIMIT}] — data growth, no log files" \
                        "Storage Watchdog" "warning"
                fi
            else
                # Action cycle
                error "$_cname — growth strike limit reached (${_STRIKE_LIMIT} consecutive cycles, ${_growth_gb}GB this cycle)"
                if [[ "$WATCHDOG_APPDATA_TRUNCATE_LOGS" == "true" && ${#_found_logs[@]} -gt 0 ]]; then
                    for _lf_entry in "${_found_logs[@]}"; do
                        _lf_path=$(echo "$_lf_entry" | cut -d' ' -f2-)
                        _lf_gb=$(echo "$_lf_entry" | awk '{printf "%.1f", $1/1073741824}')
                        _HANDLED["$_lf_path"]=1
                        if [[ "$DRY_RUN" == true ]]; then
                            warn "DRY RUN — would truncate $_lf_path (${_lf_gb}GB)"
                        else
                            if truncate -s 0 "$_lf_path" 2>/dev/null; then
                                success "Truncated runaway log: $_lf_path (was ${_lf_gb}GB)"
                                notify "Truncated runaway log on $(hostname): $_lf_path (was ${_lf_gb}GB)" \
                                    "Storage Watchdog" "warning"
                            else
                                warn "Failed to truncate $_lf_path"
                            fi
                        fi
                    done
                    set_strikes "appdata_growth_${_safe}" 0 "$STORAGE_WATCHDOG_STATE_FILE"
                else
                    if [[ -n "$_log_summary" ]]; then
                        notify "$_cname appdata runaway on $(hostname) — ${_growth_gb}GB growth for ${_STRIKE_LIMIT} cycles — logs: ${_log_summary}" \
                            "Storage Watchdog" "critical"
                    else
                        notify "$_cname appdata runaway on $(hostname) — ${_growth_gb}GB growth for ${_STRIKE_LIMIT} cycles — data growth, manual investigation needed" \
                            "Storage Watchdog" "critical"
                    fi
                fi
            fi

            # Mark found logs as handled to suppress Part 2 duplicates this cycle
            for _lf_entry in "${_found_logs[@]}"; do
                _HANDLED["$(echo "$_lf_entry" | cut -d' ' -f2-)"]=1
            done
        fi
    done < <(du -sm "$_appdata_path"/*/ 2>/dev/null)

    # Atomically update growth baseline
    [[ -n "$_growth_tmp" ]] && mv "$_growth_tmp" "$WATCHDOG_APPDATA_GROWTH_FILE" 2>/dev/null

    # ── Part 2: Absolute log size scan ────────────────────────────────────────────────────────
    # Catches *.log files already large but no longer actively growing this cycle.
    # Same strike logic. Skips files already handled by Part 1 above.
    declare -A _LOG_SEEN=()

    while IFS= read -r _hit; do
        [[ -z "$_hit" ]] && continue
        _fpath=$(echo "$_hit" | cut -d' ' -f2-)
        [[ -n "${_HANDLED[$_fpath]:-}" ]] && continue

        _fsize_bytes=$(echo "$_hit" | awk '{print $1}')
        _fsize_gb=$(awk "BEGIN {printf \"%.1f\", $_fsize_bytes / 1073741824}")
        _safe_fkey=$(echo "$_fpath" | tr -cd '[:alnum:]_' | cut -c1-120)
        _LOG_SEEN["appdata_log_${_safe_fkey}"]=1

        _strikes=$(get_strikes "appdata_log_${_safe_fkey}" "$STORAGE_WATCHDOG_STATE_FILE")
        _strikes="${_strikes//[^0-9]/}"; _strikes="${_strikes:-0}"

        [[ "$_strikes" -lt "$_STRIKE_LIMIT" ]] && {
            _strikes=$(( _strikes + 1 ))
            set_strikes "appdata_log_${_safe_fkey}" "$_strikes" "$STORAGE_WATCHDOG_STATE_FILE"
        }

        warn "Oversized log: $_fpath (${_fsize_gb}GB) [strike ${_strikes}/${_STRIKE_LIMIT}]"
        (( WARNINGS++ ))

        if [[ "$_strikes" -lt "$_STRIKE_LIMIT" ]]; then
            notify "Oversized log on $(hostname): $_fpath (${_fsize_gb}GB) [strike ${_strikes}/${_STRIKE_LIMIT}]" \
                "Storage Watchdog" "warning"
        else
            error "Oversized log persists for ${_STRIKE_LIMIT} cycles: $_fpath (${_fsize_gb}GB)"
            if [[ "$WATCHDOG_APPDATA_TRUNCATE_LOGS" == "true" ]]; then
                if [[ "$DRY_RUN" == true ]]; then
                    warn "DRY RUN — would truncate $_fpath (${_fsize_gb}GB)"
                else
                    if truncate -s 0 "$_fpath" 2>/dev/null; then
                        success "Truncated oversized log: $_fpath (was ${_fsize_gb}GB)"
                        notify "Truncated oversized log on $(hostname): $_fpath (was ${_fsize_gb}GB)" \
                            "Storage Watchdog" "warning"
                        set_strikes "appdata_log_${_safe_fkey}" 0 "$STORAGE_WATCHDOG_STATE_FILE"
                        unset "_LOG_SEEN[appdata_log_${_safe_fkey}]"
                    else
                        warn "Failed to truncate $_fpath"
                        notify "Oversized log on $(hostname): $_fpath (${_fsize_gb}GB) — ${_STRIKE_LIMIT} cycles, truncate failed" \
                            "Storage Watchdog" "critical"
                    fi
                fi
            else
                notify "Oversized log on $(hostname): $_fpath (${_fsize_gb}GB) — ${_STRIKE_LIMIT} cycles, manual intervention needed" \
                    "Storage Watchdog" "critical"
            fi
        fi
    done < <(find "$_appdata_path" -maxdepth 4 -type f \
        \( -name "*.log" -o -name "*.log.*" \) \
        -size +${_LOG_KB}k -printf "%s %p\n" 2>/dev/null)

    # Auto-clear strikes for log files no longer oversized this cycle
    while IFS=':' read -r _sk _sv; do
        [[ "$_sk" != appdata_log_* ]] && continue
        _sv_clean="${_sv//[^0-9]/}"
        [[ "${_sv_clean:-0}" -eq 0 ]] && continue
        [[ -n "${_LOG_SEEN[$_sk]:-}" ]] && continue
        set_strikes "$_sk" 0 "$STORAGE_WATCHDOG_STATE_FILE"
    done < "$STORAGE_WATCHDOG_STATE_FILE" 2>/dev/null

    unset _LOG_SEEN

done

unset _PREV _HANDLED

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
if [[ "$WARNINGS" -gt 0 ]]; then
    echo ""
    echo "━━━ $ICON_DISK Storage Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
    echo "$ICON_WATCHDOG Warnings: $WARNINGS"
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
else
    echo "Storage healthy ✅ ($(date '+%H:%M:%S'))"
fi