diff --git a/common.sh b/common.sh index ebf80b9..d5219c4 100644 --- a/common.sh +++ b/common.sh @@ -697,10 +697,10 @@ _release_on_exit() { # acquire_lock — acquire exclusive lock for this script # Mode: strict (default) — exit immediately if locked # wait — wait LOCK_WAIT_TIMEOUT seconds then exit -# continuous — for long-running scripts: skip gracefully if healthy, -# clear and restart if dead/stuck -# Stale lock: if PID in lock file is dead → clear and acquire -# Age warning: if lock older than LOCK_WARN_AGE → warn +# continuous — for long-running scripts: skip gracefully if healthy +# Lock file stores PID:scriptname — prevents PID reuse false positives +# Stale lock: if PID dead OR PID belongs to different process → clear and acquire +# Age warning: if lock older than LOCK_WARN_AGE → warn (skipped for continuous) # ----------------------------------------------------------------------------------------------- acquire_lock() { local mode="${1:-strict}" @@ -713,15 +713,23 @@ acquire_lock() { # Check for existing lock if [[ -f "$lockfile" ]]; then - local existing_pid - existing_pid=$(cat "$lockfile" 2>/dev/null) + local lock_content existing_pid locked_name + lock_content=$(cat "$lockfile" 2>/dev/null) + existing_pid="${lock_content%%:*}" + locked_name="${lock_content##*:}" - # Stale lock detection — PID no longer running - if [[ -n "$existing_pid" ]] && ! kill -0 "$existing_pid" 2>/dev/null; then + # Stale lock — PID dead + if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing" rm -f "$lockfile" + + # PID reuse — PID alive but belongs to a different process + elif [[ "$locked_name" != "$script_name" ]]; then + warn "Lock PID $existing_pid reused by different process ($locked_name ≠ $script_name) — clearing stale lock" + rm -f "$lockfile" + else - # Lock is active — check age (skip warning for continuous scripts) + # Lock is genuinely active — check age (skip warning for continuous scripts) local lock_age lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) )) if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]] && [[ "$mode" != "continuous" ]]; then @@ -729,8 +737,7 @@ acquire_lock() { fi if [[ "$mode" == "continuous" ]]; then - # Continuous scripts (watchdogs, failover) — healthy instance = always skip - # PID is alive and responding — this is correct behavior, not stuck + # Continuous scripts — healthy instance = always skip gracefully log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping" exit 0 elif [[ "$mode" == "wait" ]]; then @@ -739,12 +746,17 @@ acquire_lock() { while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do sleep 1 ((waited++)) - # Re-check for stale - existing_pid=$(cat "$lockfile" 2>/dev/null) - if [[ -n "$existing_pid" ]] && ! kill -0 "$existing_pid" 2>/dev/null; then + lock_content=$(cat "$lockfile" 2>/dev/null) + existing_pid="${lock_content%%:*}" + locked_name="${lock_content##*:}" + if [[ -z "$existing_pid" ]] || ! kill -0 "$existing_pid" 2>/dev/null; then warn "Lock became stale while waiting — clearing" rm -f "$lockfile" break + elif [[ "$locked_name" != "$script_name" ]]; then + warn "Lock PID reused while waiting — clearing" + rm -f "$lockfile" + break fi done if [[ -f "$lockfile" ]]; then @@ -753,7 +765,6 @@ acquire_lock() { fi else error "Another instance of $script_name is already running (PID $existing_pid) — exiting" - # Notify for critical scripts that should rarely overlap case "$script_name" in failover|transcode_management|media_management|daily_sync_maintenance|system_watchdog) notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning" @@ -764,8 +775,8 @@ acquire_lock() { fi fi - # Acquire lock - echo $$ > "$lockfile" + # Acquire lock — store PID:scriptname to prevent PID reuse false positives + echo "$$:$script_name" > "$lockfile" # Register EXIT trap to always release lock trap "_release_on_exit '$lockfile'" EXIT @@ -786,9 +797,11 @@ acquire_rsync_lock() { # Per-profile lock — same profile cannot run twice if [[ -f "$profile_lock" ]]; then - local existing_pid - existing_pid=$(cat "$profile_lock" 2>/dev/null) - if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null; then + local lock_content existing_pid locked_name + lock_content=$(cat "$profile_lock" 2>/dev/null) + existing_pid="${lock_content%%:*}" + locked_name="${lock_content##*:}" + if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null && [[ "$locked_name" == "rsync_${profile}" ]]; then error "rsync profile '$profile' is already running (PID $existing_pid) — exiting" exit 1 else @@ -823,7 +836,7 @@ acquire_rsync_lock() { fi # Acquire profile lock and increment counter - echo $$ > "$profile_lock" + echo "$$:rsync_${profile}" > "$profile_lock" echo $(( current_count + 1 )) > "$RSYNC_COUNT_FILE" # Register EXIT trap diff --git a/unRAID_Essentials/system_watchdog.sh b/unRAID_Essentials/system_watchdog.sh index f03721d..0e87385 100644 --- a/unRAID_Essentials/system_watchdog.sh +++ b/unRAID_Essentials/system_watchdog.sh @@ -1,883 +1,436 @@ #!/bin/bash # ----------------------------------------------------------------------------------------------- -# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ---------------------------- +# --------------------------------- System Watchdog -------------------------------------------- # ----------------------------------------------------------------------------------------------- -# Version: 2.9 -# ----------------------------------------------------------------------------------------------- -# Changelog: -# v1.0 — Initial stable framework -# v1.1 — format_duration moved here from daily_sync_maintenance.sh for shared use -# SSH_KEY collision resolved — gitea key renamed GITEA_SSH_KEY in Master.conf -# Version and changelog tracking added -# v1.2 — Consistent function header comment blocks across all functions -# check_connectivity added as standalone function -# check_connectivity friendlier error output with tailscale hint -# v1.3 — check_remote_rootfs added — aborts if remote rootfs exceeds ROOTFS_WARN threshold -# check_remote_share added — aborts if target directory is missing or empty on remote -# Both protect against rsync running when remote array is down or drives are missing -# v1.4 — check_remote_disks added — verifies all physical disks backing a share are mounted -# Discovers disk layout automatically at runtime, no configuration required -# Aborts if any single disk backing the share is offline or unmounted -# v1.5 — Full icon set expanded — each operation and state has its own distinct icon -# All function output updated to use correct icon per context -# Icons grouped and commented by category for clarity -# v1.6 — ICON_CONTAINERS added — 📦 anchors all container sections for visual consistency -# ICON_NOT_RUNNING changed to ⭕ — distinct from ICON_STOPPED 🔴 -# Section dividers updated from --- to ━━━ for cleaner log readability -# Summary passed/failed lines use ICON_SUCCESS and ICON_ERROR consistently -# v1.7 — ICON_MOVER added for mover operations -# ICON_CONTAINERS replaces ICON_DOCKER for docker/container operations -# validate_int added — reusable integer validation for any script -# v1.8 — ICON_PHP added for PHP-FPM operations -# v1.9 — ICON_REBOOT added for server reboot operations -# v2.0 — ICON_PLUGIN added for User Scripts plugin operations -# v2.1 — ICON_ZFS and ICON_MEM added for ZFS and memory diagnostics -# Diagnostics icon group added to icon block -# v2.2 — ICON_WATCHDOG added for Docker watchdog monitoring operations -# v2.3 — ICON_NOTIFY added for notification operations -# notify() added — shared notification function supporting unRAID native and Discord -# NOTIFY_UNRAID and DISCORD_WEBHOOK configured in Master.conf -# v2.4 — ICON_CLEAN, ICON_TRASH added for media cleaner operations -# ICON_PERMS, ICON_UNLOCKED added for media permissions operations -# ICON_REBOOT_SMART added for smart conditional reboot -# v2.5 — ICON_RAM added for ramdisk operations -# ICON_LINK added for symlink state and management -# Transcode scripts group added to ecosystem -# v2.6 — ICON_FAILOVER added for failover operations -# check_local_array added — verifies local /mnt/user is mounted and healthy -# check_remote_array added — verifies remote /mnt/user is mounted and healthy -# check_remote_docker added — verifies remote Docker daemon is responding -# ping_remote added — non-fatal ping returning status for failover use -# ping_internet added — non-fatal external ping for failover use -# v2.7 — ICON_WEBGUI added for WebGUI watchdog operations -# ICON_DOCKER_NET added for Docker network connect operations -# v2.8 — ICON_CERT added for SSL certificate monitoring operations -# v2.9 — ICON_MONITOR added for monitoring section headers -# v3.0 — Script locking system added — prevents concurrent execution conflicts -# acquire_lock() — create PID lock file, register EXIT trap -# release_lock() — remove lock file on exit -# acquire_rsync_lock() — per-profile lock + global concurrent limit -# release_rsync_lock() — decrement global counter, remove profile lock -# check_api() — pre-flight API reachability check -# ICON_LOCK added — 🔏 script instance lock acquired/released -# ICON_SMART added for drive SMART health operations -# ICON_BANDWIDTH added for bandwidth tracking operations -# ICON_DIGEST added for health digest operations -# ICON_EMBY added for Emby session reporting -# ICON_VERIFY added for backup verification operations -# Monitor/ folder added to ecosystem -# ----------------------------------------------------------------------------------------------- - -# ----------------------------------------------------------------------------------------------- -# ICONS -# Each icon has one job — do not reuse across different contexts. -# Adding a new icon: add it to the appropriate group below with a comment describing its job. -# ----------------------------------------------------------------------------------------------- - -# System / Host -ICON_HOST="🖥️" # host detection -ICON_NET="🌐" # network / IP resolution -ICON_PING="📡" # connectivity check -ICON_GEAR="⚙️" # setup section header / profile load - -# Health Checks -ICON_DISK="💾" # disk checks -ICON_HEALTH="🩺" # rootfs / share health checks -ICON_SHIELD="🛡️" # pre-flight section header - -# Containers -ICON_CONTAINERS="📦" # container section anchor -ICON_STOP="⛔" # stop command being issued -ICON_STOPPED="🔴" # container confirmed stopped -ICON_START="▶️" # start command being issued -ICON_STARTED="💚" # container confirmed started -ICON_RUNNING="🟢" # container already running when checked -ICON_SKIP="⏭️" # skipping — already running healthy instance -ICON_NOT_RUNNING="⭕" # container already stopped when checked - -# Transfer -ICON_SYNC="🔄" # transfer section header -ICON_RUN="🚀" # sync starting / rsync attempt -ICON_RETRY="🔁" # retry attempt -ICON_DONE="🏁" # transfer complete - -# Summary -ICON_SUMMARY="📋" # summary section header -ICON_TIME="⏱️" # duration line - -# System Operations -ICON_MOVER="🔃" # mover operations -ICON_REBOOT="⚡" # scheduled server reboot -ICON_REBOOT_SMART="🚨" # smart conditional reboot triggered -ICON_PLUGIN="🧩" # user scripts plugin operations -ICON_PHP="👥" # PHP-FPM operations -ICON_WEBGUI="💻" # WebGUI / nginx / emhttp operations - -# Diagnostics -ICON_ZFS="📊" # ZFS ARC statistics -ICON_MEM="🧠" # memory status -ICON_WATCHDOG="🐾" # docker watchdog monitoring operations - -# Media Operations -ICON_CLEAN="🧹" # media cleaner operations -ICON_TRASH="🗑️" # files being deleted -ICON_PERMS="🔐" # permissions operation / section header -ICON_UNLOCKED="🔓" # permissions successfully applied to a share -ICON_LOCK="🔏" # script instance lock — acquired/released - -# Transcode Operations -ICON_RAM="💨" # ramdisk operations — fast ephemeral storage -ICON_LINK="🔗" # symlink state and management - -# Failover Operations -ICON_FAILOVER="🔀" # failover state changes and operations - -# Docker Network Operations -ICON_DOCKER_NET="🔌" # Docker network connect operations - -# Security / Certificate Operations -ICON_CERT="🔒" # SSL certificate monitoring - -# Monitor Operations -ICON_MONITOR="📈" # monitoring section headers and general monitoring -ICON_SMART="🔧" # drive SMART health attribute monitoring -ICON_BANDWIDTH="📶" # bandwidth usage tracking and reporting -ICON_DIGEST="📰" # health digest — aggregated system summary -ICON_EMBY="🎬" # Emby media server session reporting -ICON_VERIFY="✔️" # backup verification — checksum comparison - -# Notifications -ICON_NOTIFY="🔔" # notification operations - -# Output -ICON_INFO="ℹ️" -ICON_WARN="⚠️" -ICON_ERROR="❌" -ICON_SUCCESS="✅" - -# ----------------------------------------------------------------------------------------------- -# OUTPUT HELPERS -# Standardised output functions used across all scripts. -# log() is gated by ENABLE_LOGGING — set in Master.conf or via --log flag. -# ----------------------------------------------------------------------------------------------- -info() { echo "$ICON_INFO [INFO] $*"; } -warn() { echo "$ICON_WARN [WARN] $*"; } -error() { echo "$ICON_ERROR [ERROR] $*"; } -success() { echo "$ICON_SUCCESS [OK] $*"; } - -log() { - [[ "${ENABLE_LOGGING:-false}" == true ]] && echo "[LOG] $*" -} - -# ----------------------------------------------------------------------------------------------- -# NOTIFICATION -# Sends a notification via unRAID native system and/or Discord webhook. -# Both channels are optional and independently controlled via Master.conf. -# Severity levels: normal, warning, alert -# Usage: notify "message" "subject" "severity" -# ----------------------------------------------------------------------------------------------- -notify() { - local message="$1" - local subject="${2:-unRAID Notification}" - local severity="${3:-normal}" - - log "$ICON_NOTIFY Sending notification: $subject — $message" - - if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then - local notify_script="/usr/local/emhttp/plugins/dynamix/scripts/notify" - if [[ -x "$notify_script" ]]; then - "$notify_script" -s "$subject" -d "$message" -i "$severity" 2>/dev/null - log "$ICON_NOTIFY unRAID notification sent" - else - log "$ICON_NOTIFY unRAID notify script not found — skipping" - fi - fi - - if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then - local payload - payload=$(printf '{"content": "%s — **%s**\\n%s"}' \ - "$ICON_NOTIFY" "$subject" "$message") - if curl -s -H "Content-Type: application/json" \ - -d "$payload" "$DISCORD_WEBHOOK" >/dev/null 2>&1; then - log "$ICON_NOTIFY Discord notification sent" - else - warn "Discord notification failed — check DISCORD_WEBHOOK in Master.conf" - fi - fi -} - -# ----------------------------------------------------------------------------------------------- -# DURATION FORMATTER -# Converts raw seconds into a human readable string — e.g. 10m53s or 47s -# ----------------------------------------------------------------------------------------------- -format_duration() { - local secs=$1 - local mins=$((secs / 60)) - local rem=$((secs % 60)) - [[ $mins -gt 0 ]] && echo "${mins}m${rem}s" || echo "${rem}s" -} - -# ----------------------------------------------------------------------------------------------- -# ARG PARSER -# Processes all flags and key=value pairs passed to any script. -# Supported flags: --dry-run, --log, --no-log, --status, --help -# Supported key=value: LOG=true/false, or any declared variable e.g. BW_LIMIT=5000 -# Unparsed positional args returned in PARSED_ARGS array. -# ----------------------------------------------------------------------------------------------- -parse_args() { - ENABLE_LOGGING=${ENABLE_LOGGING:-false} - DRY_RUN=${DRY_RUN:-false} - SHOW_STATUS=${SHOW_STATUS:-false} - CLEAN_ARGS=() - - for ARG in "$@"; do - if [[ "$ARG" == *=* ]]; then - VAR="${ARG%%=*}" - VAL="${ARG#*=}" - case "$VAR" in - LOG) - [[ "$VAL" == "true" ]] && ENABLE_LOGGING=true - [[ "$VAL" == "false" ]] && ENABLE_LOGGING=false - ;; - *) - if declare -p "$VAR" &>/dev/null; then - printf -v "$VAR" '%s' "$VAL" - log "Set $VAR=$VAL" - else - warn "Unknown variable: $VAR" - fi - ;; - esac - else - case "$ARG" in - --dry-run|-n) DRY_RUN=true ;; - --log) ENABLE_LOGGING=true ;; - --no-log) ENABLE_LOGGING=false ;; - --status|--summary) SHOW_STATUS=true ;; - --help|-h) - echo "Usage: script [--dry-run] [--log] [--status]" - exit 0 - ;; - *) CLEAN_ARGS+=("$ARG") ;; - esac - fi - done - - PARSED_ARGS=("${CLEAN_ARGS[@]}") -} - -# ----------------------------------------------------------------------------------------------- -# VALIDATION HELPERS -# ----------------------------------------------------------------------------------------------- - -# Exits with error if a required variable is empty or unset. -# Usage: require_var VAR_NAME -require_var() { - [[ -z "${!1:-}" ]] && error "Missing required: $1" && exit 1 -} - -# Exits with error if a variable is not a valid positive integer. -# Usage: validate_int VAR_NAME "$VAR_VALUE" -validate_int() { - local name="$1" value="$2" - if [[ -z "$value" ]]; then - error "$name is not set — check Master.conf" - exit 1 - fi - if ! [[ "$value" =~ ^[0-9]+$ ]]; then - error "$name must be a positive integer — got: '$value'" - exit 1 - fi - log "$name validated: $value" -} - -# ----------------------------------------------------------------------------------------------- -# HOST DETECTION -# Determines which server is local and which is remote by comparing hostname against -# HOST1 and HOST2 in Master.conf. Sets LOCAL_SERVER_NAME, REMOTE_SERVER_NAME and SSH_KEY. -# Both servers run identical scripts — this is what makes them bidirectional. -# ----------------------------------------------------------------------------------------------- -detect_hosts() { - LOCAL_HOSTNAME="$(hostname)" - - if [[ "$LOCAL_HOSTNAME" == "$HOST1" ]]; then - LOCAL_SERVER_NAME="$HOST1" - REMOTE_SERVER_NAME="$HOST2" - elif [[ "$LOCAL_HOSTNAME" == "$HOST2" ]]; then - LOCAL_SERVER_NAME="$HOST2" - REMOTE_SERVER_NAME="$HOST1" - else - error "Unknown host: $LOCAL_HOSTNAME" - exit 1 - fi - - declare -A SSH_KEYS - SSH_KEYS["$HOST1|$HOST2"]="$HOST1_SSH_KEY" - SSH_KEYS["$HOST2|$HOST1"]="$HOST2_SSH_KEY" - SSH_KEY="${SSH_KEYS[$LOCAL_SERVER_NAME|$REMOTE_SERVER_NAME]}" - - [[ -z "$SSH_KEY" ]] && error "Missing SSH key mapping" && exit 1 - info "$ICON_HOST Host: $LOCAL_SERVER_NAME → $REMOTE_SERVER_NAME" -} - -# ----------------------------------------------------------------------------------------------- -# REMOTE IP RESOLUTION -# Resolves the Tailscale IPv4 address of the remote server. -# Sets REMOTE_SERVER used by all subsequent SSH and rsync calls. -# Exits if resolution fails — Tailscale may be down or peer offline. -# ----------------------------------------------------------------------------------------------- -resolve_remote_ip() { - log "Resolving remote IP for $REMOTE_SERVER_NAME..." - REMOTE_SERVER=$(tailscale ip -4 "$REMOTE_SERVER_NAME" 2>/dev/null) - [[ -z "$REMOTE_SERVER" ]] && error "Failed to resolve Tailscale IP for $REMOTE_SERVER_NAME" && exit 1 - info "$ICON_NET Remote IP: $REMOTE_SERVER" -} - -# ----------------------------------------------------------------------------------------------- -# CONNECTIVITY CHECK — fatal, used by rsync scripts -# Pings remote and exits if unreachable. -# For failover use ping_remote() which returns status without exiting. -# ----------------------------------------------------------------------------------------------- -check_connectivity() { - log "Checking connectivity to $REMOTE_SERVER..." - if ! ping -c1 -W3 "$REMOTE_SERVER" &>/dev/null; then - error "$ICON_PING Remote $REMOTE_SERVER ($REMOTE_SERVER_NAME) is unreachable" - info "Hint: tailscale status | grep $REMOTE_SERVER_NAME" - exit 1 - fi - info "$ICON_PING $REMOTE_SERVER_NAME is reachable" -} - -# ----------------------------------------------------------------------------------------------- -# PING REMOTE — non-fatal, used by failover -# Returns 0 if reachable, 1 if not — does NOT exit. -# ----------------------------------------------------------------------------------------------- -ping_remote() { - ping -c2 -W3 "$REMOTE_SERVER" &>/dev/null -} - -# ----------------------------------------------------------------------------------------------- -# PING INTERNET — non-fatal external connectivity check -# Returns 0 if internet reachable, 1 if not — does NOT exit. -# ----------------------------------------------------------------------------------------------- -ping_internet() { - ping -c2 -W3 "${EXTERNAL_IP:-8.8.8.8}" &>/dev/null -} - -# ----------------------------------------------------------------------------------------------- -# LOCAL ARRAY CHECK — non-fatal, returns status -# Verifies local /mnt/user is mounted and has shares. -# Used by failover before starting remote containers locally. -# Returns 0 if healthy, 1 if not. -# ----------------------------------------------------------------------------------------------- -check_local_array() { - log "Checking local array..." - if ! mountpoint -q /mnt/user 2>/dev/null; then - error "$ICON_DISK Local array is not started — /mnt/user is not mounted" - return 1 - fi - local file_count - file_count=$(ls /mnt/user 2>/dev/null | wc -l) - if [[ "$file_count" -eq 0 ]]; then - error "$ICON_DISK Local array appears empty — shares may not be available" - return 1 - fi - log "Local array is healthy" - return 0 -} - -# ----------------------------------------------------------------------------------------------- -# REMOTE ARRAY CHECK — non-fatal, returns status -# Verifies remote /mnt/user is mounted via SSH. -# Used before handback rsync — syncing to remote with no array fills rootfs. -# Returns 0 if healthy, 1 if not. -# ----------------------------------------------------------------------------------------------- -check_remote_array() { - log "Checking remote array on $REMOTE_SERVER_NAME..." - local result - result=$(ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ - "mountpoint -q /mnt/user && echo yes || echo no" 2>/dev/null) - if [[ "$result" != "yes" ]]; then - error "$ICON_DISK Remote array not started on $REMOTE_SERVER_NAME" - return 1 - fi - log "Remote array is healthy" - return 0 -} - -# ----------------------------------------------------------------------------------------------- -# REMOTE DOCKER CHECK — non-fatal, returns status -# Verifies remote Docker daemon is responding before container operations. -# A hung daemon means start/stop commands will silently fail. -# Returns 0 if healthy, 1 if not. -# ----------------------------------------------------------------------------------------------- -check_remote_docker() { - log "Checking remote Docker daemon on $REMOTE_SERVER_NAME..." - if ! ssh -i "$SSH_KEY" -o ConnectTimeout=10 root@"$REMOTE_SERVER" \ - "timeout 10 docker ps" >/dev/null 2>&1; then - error "$ICON_CONTAINERS Remote Docker daemon not responding on $REMOTE_SERVER_NAME" - return 1 - fi - log "Remote Docker daemon is healthy" - return 0 -} - -# ----------------------------------------------------------------------------------------------- -# REMOTE ROOTFS SPACE CHECK — fatal -# Aborts if remote rootfs exceeds ROOTFS_WARN threshold. -# ----------------------------------------------------------------------------------------------- -check_remote_rootfs() { - log "Checking remote rootfs usage..." - REMOTE_USAGE=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ - "df / --output=pcent | tail -1 | tr -d ' %'" 2>/dev/null) - if [[ -z "$REMOTE_USAGE" ]]; then - error "Could not retrieve rootfs usage from $REMOTE_SERVER_NAME" - exit 1 - fi - if [[ "$REMOTE_USAGE" -ge "${ROOTFS_WARN:-75}" ]]; then - error "$ICON_HEALTH Remote rootfs ${REMOTE_USAGE}% — threshold ${ROOTFS_WARN:-75}%" - exit 1 - fi - info "$ICON_HEALTH Remote rootfs: ${REMOTE_USAGE}% (threshold: ${ROOTFS_WARN:-75}%)" -} - -# ----------------------------------------------------------------------------------------------- -# REMOTE SHARE VALIDATION — fatal -# Verifies target directory exists and is not empty on remote. -# Usage: check_remote_share "/mnt/user/Movies" -# ----------------------------------------------------------------------------------------------- -check_remote_share() { - local dir="$1" - log "Checking remote share: $dir..." - - SHARE_EXISTS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ - "[[ -d '$dir' ]] && echo yes || echo no" 2>/dev/null) - if [[ "$SHARE_EXISTS" != "yes" ]]; then - error "$ICON_HEALTH Remote share does not exist: $dir" - exit 1 - fi - - SHARE_EMPTY=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ - "[[ -z \"\$(ls -A '$dir' 2>/dev/null)\" ]] && echo yes || echo no" 2>/dev/null) - if [[ "$SHARE_EMPTY" == "yes" ]]; then - warn "$ICON_HEALTH Remote share exists but is empty: $dir — aborting to protect data" - exit 1 - fi - - info "$ICON_HEALTH Remote share verified: $dir" -} - -# ----------------------------------------------------------------------------------------------- -# REMOTE DISK CHECK — fatal -# Verifies all physical disks backing a share are online on the remote server. -# Skipped when PROFILE_SKIP_DISK_CHECK is true (ZFS pools have no /mnt/disk* structure). -# Usage: check_remote_disks "/mnt/user/Movies" -# ----------------------------------------------------------------------------------------------- -check_remote_disks() { - local dir="$1" - local share_name - share_name=$(basename "$dir") - - info "$ICON_DISK Checking disks backing $share_name on $REMOTE_SERVER_NAME..." - - DISK_PATHS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ - "ls -d /mnt/disk*/$share_name 2>/dev/null" 2>/dev/null) - - if [[ -z "$DISK_PATHS" ]]; then - error "$ICON_DISK No disks found backing $share_name on $REMOTE_SERVER_NAME" - exit 1 - fi - - local all_ok=true - while IFS= read -r disk_share_path; do - local disk_mount disk_name - disk_mount=$(dirname "$disk_share_path") - disk_name=$(basename "$disk_mount") - MOUNTED=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ - "mountpoint -q '$disk_mount' && echo yes || echo no" 2>/dev/null) - if [[ "$MOUNTED" == "yes" ]]; then - info "$ICON_DISK $disk_name $ICON_RUNNING — $share_name present" - else - error "$ICON_DISK $disk_name $ICON_STOPPED — $share_name missing" - all_ok=false - fi - done <<< "$DISK_PATHS" - - if [[ "$all_ok" == false ]]; then - error "One or more disks backing $share_name are offline on $REMOTE_SERVER_NAME" - exit 1 - fi - success "All disks backing $share_name are online" -} - -# ----------------------------------------------------------------------------------------------- -# CONTAINER MANAGEMENT — STOP (remote via SSH) -# Stops containers in CRITICAL_CONTAINER_NAMES on remote server. -# Tracks running containers in RUNNING_CONTAINERS for restart after rsync. -# ----------------------------------------------------------------------------------------------- -RUNNING_CONTAINERS=() - -stop_containers() { - if [[ ${#CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \ - [[ "${CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then - log "No remote containers configured for this profile, skipping stop." - return - fi - info "Stopping remote containers..." - RUNNING_CONTAINERS=() - for c in "${CRITICAL_CONTAINER_NAMES[@]}"; do - [[ -z "$c" ]] && continue - STATUS=$(ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" \ - "docker inspect -f '{{.State.Running}}' $c 2>/dev/null" 2>/dev/null || echo "unknown") - if [[ "$STATUS" == "true" ]]; then - echo "$ICON_STOP Stopping $c..." - RUNNING_CONTAINERS+=("$c") - if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker stop $c" >/dev/null; then - echo "$ICON_STOPPED $c stopped" - else - error "Failed to stop $c" - fi - elif [[ "$STATUS" == "false" ]]; then - echo "$ICON_NOT_RUNNING $c is not running, skipping" - else - log "$c not found on remote — skipping" - fi - done -} - -# ----------------------------------------------------------------------------------------------- -# CONTAINER MANAGEMENT — STOP LOCAL -# Stops containers on the LOCAL server before rsync pushes data out. -# Uses PROFILE_LOCAL_CRITICAL_CONTAINER_NAMES — same naming scheme as remote. -# If container not found on this server → skipped gracefully, not errored. -# Only containers that were running get tracked for restart. -# ----------------------------------------------------------------------------------------------- -stop_local_containers() { - if [[ ${#LOCAL_CRITICAL_CONTAINER_NAMES[@]} -eq 0 ]] || \ - [[ "${LOCAL_CRITICAL_CONTAINER_NAMES[*]}" == "" ]]; then - log "No local containers configured for this profile, skipping local stop." - return - fi - info "Stopping local containers..." - LOCAL_RUNNING_CONTAINERS=() - for c in "${LOCAL_CRITICAL_CONTAINER_NAMES[@]}"; do - [[ -z "$c" ]] && continue - STATUS=$(docker inspect -f '{{.State.Running}}' "$c" 2>/dev/null || echo "unknown") - if [[ "$STATUS" == "true" ]]; then - echo "$ICON_STOP Stopping local $c..." - LOCAL_RUNNING_CONTAINERS+=("$c") - if docker stop "$c" >/dev/null; then - echo "$ICON_STOPPED $c stopped" - else - error "Failed to stop local $c" - fi - elif [[ "$STATUS" == "false" ]]; then - echo "$ICON_NOT_RUNNING $c is not running, skipping" - else - log "$c not found locally — skipping" - fi - done -} - -# ----------------------------------------------------------------------------------------------- -# CONTAINER MANAGEMENT — START (remote via SSH) -# Restarts only containers tracked in RUNNING_CONTAINERS. -# Delayed containers receive CONTAINER_DELAY seconds before starting. -# ----------------------------------------------------------------------------------------------- -start_containers() { - if [[ ${#RUNNING_CONTAINERS[@]} -eq 0 ]]; then - log "No remote containers to restart." - return - fi - info "Starting remote containers..." - for c in "${RUNNING_CONTAINERS[@]}"; do - [[ -z "$c" ]] && continue - local needs_delay=false - for d in "${DELAYED_CONTAINERS[@]}"; do - [[ "$c" == "$d" ]] && needs_delay=true && break - done - if [[ "$needs_delay" == true ]]; then - info "Waiting ${CONTAINER_DELAY}s before starting $c..." - sleep "$CONTAINER_DELAY" - fi - echo "$ICON_START Starting $c..." - if ssh -i "$SSH_KEY" root@"$REMOTE_SERVER" "docker start $c" >/dev/null 2>&1; then - echo "$ICON_STARTED $c started" - else - error "Failed to start $c — start manually if needed" - fi - done -} - -# ----------------------------------------------------------------------------------------------- -# CONTAINER MANAGEMENT — START LOCAL -# Restarts only containers tracked in LOCAL_RUNNING_CONTAINERS. -# Respects DELAYED_CONTAINERS and CONTAINER_DELAY same as remote start. -# If container not found → skipped gracefully. -# ----------------------------------------------------------------------------------------------- -start_local_containers() { - if [[ ${#LOCAL_RUNNING_CONTAINERS[@]} -eq 0 ]]; then - log "No local containers to restart." - return - fi - info "Starting local containers..." - for c in "${LOCAL_RUNNING_CONTAINERS[@]}"; do - [[ -z "$c" ]] && continue - local needs_delay=false - for d in "${DELAYED_CONTAINERS[@]}"; do - [[ "$c" == "$d" ]] && needs_delay=true && break - done - if [[ "$needs_delay" == true ]]; then - info "Waiting ${CONTAINER_DELAY}s before starting local $c..." - sleep "$CONTAINER_DELAY" - fi - echo "$ICON_START Starting local $c..." - if docker start "$c" >/dev/null 2>&1; then - echo "$ICON_STARTED $c started" - else - error "Failed to start local $c — start manually if needed" - fi - done -} - -# ----------------------------------------------------------------------------------------------- -# RSYNC OPTIONS -# Loads rsync options for current profile. Falls back to DEFAULT_RSYNC_OPTS if no match. -# Profile opts do NOT inherit from defaults — list all desired flags explicitly. -# ----------------------------------------------------------------------------------------------- -get_rsync_opts() { - if [[ -n "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]:-}" ]]; then - read -r -a RSYNC_OPTS <<< "${PROFILE_RSYNC_OPTS[$PROFILE_NAME]}" - log "Using profile rsync opts for $PROFILE_NAME: ${RSYNC_OPTS[*]}" - else - RSYNC_OPTS=("${DEFAULT_RSYNC_OPTS[@]}") - log "Using default rsync opts: ${RSYNC_OPTS[*]}" - fi -} - -# ----------------------------------------------------------------------------------------------- -# SCRIPT LOCKING — v3.0 -# Prevents multiple instances of the same script running simultaneously. -# All lock files live in /tmp/unraid_locks/ — auto-cleared on reboot. +# Last line of defense — reboots the system cleanly if it is about to become unstable. +# Runs continuously as a background process — started by array_start.sh at array start. +# Works alongside docker_watchdog.sh which handles container-level healing first. # -# Usage in scripts: -# acquire_lock — strict: exit immediately if already running -# acquire_lock "wait" — wait mode: wait briefly then exit if still locked -# acquire_rsync_lock "$profile" — per-profile + global concurrent limit +# Checks (all toggleable in Master.conf): +# rootfs usage — high rootfs fills rapidly when array is down, crash imminent +# /var/log usage — log spam can fill rootfs, indicates something is broken +# free RAM — critically low RAM means OOM or swap imminent +# ZFS ARC pinned — ARC not releasing after reclaim means memory is stuck +# CPU temperature — sustained tjmax causes throttling or kernel panic +# load average — sustained high load means something is stuck or runaway +# zombie processes — large zombie count indicates serious process management failure +# Docker daemon — unresponsive daemon means containers cannot be managed +# Required containers — stopped containers that should be running (after watchdog skip list) # -# Stale lock detection — if lock file exists but PID is dead, clears and proceeds. -# Lock age warning — if lock is older than expected, warns but does not override. -# EXIT trap registered automatically — lock always released on exit, crash, or kill. +# Abort conditions (toggleable): +# ZFS pool unhealthy — reboot with bad pool risks data loss +# Parity running — aborting parity is better than crashing mid-check +# Mover running — aborting move is better than crashing mid-move +# +# Reboot loop protection: +# Tracks reboot timestamps in persistent log on /boot/ +# Rolling window — old entries purge automatically +# If reboot count hits limit in window → shutdown instead of reboot +# +# Continuous loop: +# Checks run every SYSTEM_WATCHDOG_INTERVAL seconds (default 300 = 5min) +# Master.conf re-sourced each cycle — config changes picked up without restart +# Silent when healthy — only verbose when trigger or reboot +# Clean shutdown on SIGTERM/SIGINT — sent by array stop +# +# All configuration in Master.conf under System Watchdog section. +# Supports --dry-run to show triggered conditions without rebooting. # ----------------------------------------------------------------------------------------------- -LOCK_DIR="/tmp/unraid_locks" -RSYNC_COUNT_FILE="$LOCK_DIR/rsync_active_count" -RSYNC_MAX_CONCURRENT=3 -LOCK_WARN_AGE=300 # seconds — warn if lock older than this (5min default) -LOCK_WAIT_TIMEOUT=30 # seconds — how long "wait" mode waits before giving up +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Internal — script name used as lock identifier -_lock_name() { - basename "${BASH_SOURCE[1]:-$0}" .sh +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup — runs once at start ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +acquire_lock "continuous" + +TOTAL_CORES=$(nproc) + +# Ensure state and persistent files exist +touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || { + error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE" + exit 1 } -# Internal — lock file path for this script -_lock_file() { - echo "$LOCK_DIR/${1:-$(_lock_name)}.lock" +touch "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || { + error "Cannot create reboot log: $SYS_WATCHDOG_REBOOT_LOG" + exit 1 } -# Internal — release lock on exit -_release_on_exit() { - local lockfile="$1" - [[ -f "$lockfile" ]] && rm -f "$lockfile" +touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || { + error "Cannot create failed container list: $SYS_WATCHDOG_FAILED_FILE" + exit 1 } # ----------------------------------------------------------------------------------------------- -# acquire_lock — acquire exclusive lock for this script -# Mode: strict (default) — exit immediately if locked -# wait — wait LOCK_WAIT_TIMEOUT seconds then exit -# continuous — for long-running scripts: skip gracefully if healthy, -# clear and restart if dead/stuck -# Stale lock: if PID in lock file is dead → clear and acquire -# Age warning: if lock older than LOCK_WARN_AGE → warn +# ━━━ $ICON_SUMMARY Status ━━━ # ----------------------------------------------------------------------------------------------- -acquire_lock() { - local mode="${1:-strict}" - local script_name - script_name=$(basename "${BASH_SOURCE[1]:-$0}" .sh) - local lockfile - lockfile="$(_lock_file "$script_name")" - - mkdir -p "$LOCK_DIR" - - # Check for existing lock - if [[ -f "$lockfile" ]]; then - local existing_pid - existing_pid=$(cat "$lockfile" 2>/dev/null) - - # Stale lock detection — PID no longer running - if [[ -n "$existing_pid" ]] && ! kill -0 "$existing_pid" 2>/dev/null; then - warn "Stale lock detected for $script_name (PID $existing_pid gone) — clearing" - rm -f "$lockfile" - else - # Lock is active — check age (skip warning for continuous scripts) - local lock_age - lock_age=$(( $(date +%s) - $(stat -c %Y "$lockfile" 2>/dev/null || echo 0) )) - if [[ "$lock_age" -gt "$LOCK_WARN_AGE" ]] && [[ "$mode" != "continuous" ]]; then - warn "$script_name has been running for ${lock_age}s — may be stuck (PID $existing_pid)" - fi - - if [[ "$mode" == "continuous" ]]; then - # Continuous scripts (watchdogs, failover) — healthy instance = always skip - # PID is alive and responding — this is correct behavior, not stuck - log "$ICON_SKIP $script_name already running healthy (PID $existing_pid) — skipping" - exit 0 - elif [[ "$mode" == "wait" ]]; then - info "Another instance of $script_name is running — waiting up to ${LOCK_WAIT_TIMEOUT}s" - local waited=0 - while [[ -f "$lockfile" ]] && [[ "$waited" -lt "$LOCK_WAIT_TIMEOUT" ]]; do - sleep 1 - ((waited++)) - # Re-check for stale - existing_pid=$(cat "$lockfile" 2>/dev/null) - if [[ -n "$existing_pid" ]] && ! kill -0 "$existing_pid" 2>/dev/null; then - warn "Lock became stale while waiting — clearing" - rm -f "$lockfile" - break - fi - done - if [[ -f "$lockfile" ]]; then - error "$script_name still locked after ${LOCK_WAIT_TIMEOUT}s — exiting" - exit 1 - fi - else - error "Another instance of $script_name is already running (PID $existing_pid) — exiting" - # Notify for critical scripts that should rarely overlap - case "$script_name" in - failover|transcode_management|media_management|daily_sync_maintenance|system_watchdog) - notify "$script_name lock collision on $(hostname) — concurrent instance detected" "$script_name" "warning" - ;; - esac - exit 1 - fi - fi - fi - - # Acquire lock - echo $$ > "$lockfile" - - # Register EXIT trap to always release lock - trap "_release_on_exit '$lockfile'" EXIT - - log "$ICON_LOCK Lock acquired: $script_name (PID $$)" -} - -# ----------------------------------------------------------------------------------------------- -# acquire_rsync_lock — per-profile lock + global concurrent limit -# Prevents same profile running twice and limits total concurrent rsync instances -# ----------------------------------------------------------------------------------------------- -acquire_rsync_lock() { - local profile="$1" - local profile_lock - profile_lock="$(_lock_file "rsync_${profile}")" - - mkdir -p "$LOCK_DIR" - - # Per-profile lock — same profile cannot run twice - if [[ -f "$profile_lock" ]]; then - local existing_pid - existing_pid=$(cat "$profile_lock" 2>/dev/null) - if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null; then - error "rsync profile '$profile' is already running (PID $existing_pid) — exiting" - exit 1 - else - warn "Stale rsync lock for profile '$profile' — clearing" - rm -f "$profile_lock" - fi - fi - - # Global concurrent limit - local current_count=0 - if [[ -f "$RSYNC_COUNT_FILE" ]]; then - current_count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 0) - # Validate count — clean up if stale - local actual_count=0 - for lf in "$LOCK_DIR"/rsync_*.lock; do - [[ -f "$lf" ]] || continue - local lpid - lpid=$(cat "$lf" 2>/dev/null) - kill -0 "$lpid" 2>/dev/null && ((actual_count++)) - done - if [[ "$actual_count" -ne "$current_count" ]]; then - log "rsync count corrected: $current_count → $actual_count" - current_count=$actual_count - echo "$current_count" > "$RSYNC_COUNT_FILE" - fi - fi - - if [[ "$current_count" -ge "$RSYNC_MAX_CONCURRENT" ]]; then - error "Maximum concurrent rsync limit ($RSYNC_MAX_CONCURRENT) reached — exiting" - info "Active rsync locks: $(ls "$LOCK_DIR"/rsync_*.lock 2>/dev/null | xargs -I{} basename {} .lock | tr '\n' ' ')" - exit 1 - fi - - # Acquire profile lock and increment counter - echo $$ > "$profile_lock" - echo $(( current_count + 1 )) > "$RSYNC_COUNT_FILE" - - # Register EXIT trap - trap "_release_rsync_on_exit '$profile_lock'" EXIT - - log "$ICON_LOCK rsync lock acquired: profile '$profile' (PID $$, active: $(( current_count + 1 ))/$RSYNC_MAX_CONCURRENT)" -} - -# Internal — release rsync lock on exit -_release_rsync_on_exit() { - local profile_lock="$1" - [[ -f "$profile_lock" ]] && rm -f "$profile_lock" - # Decrement global counter - if [[ -f "$RSYNC_COUNT_FILE" ]]; then - local count - count=$(cat "$RSYNC_COUNT_FILE" 2>/dev/null || echo 1) - count=$(( count - 1 )) - [[ "$count" -lt 0 ]] && count=0 - echo "$count" > "$RSYNC_COUNT_FILE" - fi -} - -# ----------------------------------------------------------------------------------------------- -# check_api — pre-flight API reachability check -# Verifies API endpoint is reachable before attempting operations -# Usage: check_api "http://localhost:8989" "Sonarr" || exit 1 -# ----------------------------------------------------------------------------------------------- -check_api() { - local url="$1" - local service="${2:-API}" - local timeout="${3:-10}" - - if curl -sf --max-time "$timeout" "$url" >/dev/null 2>&1; then - log "$service API reachable: $url" - return 0 - else - error "$service API not reachable: $url" - return 1 - fi -} - -# ----------------------------------------------------------------------------------------------- -# STATUS DISPLAY -# Prints current runtime configuration — triggered by --status flag. -# ----------------------------------------------------------------------------------------------- -show_status() { +if [[ "$SHOW_STATUS" == true ]]; then + echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "Local: $LOCAL_SERVER_NAME" - echo "Remote: $REMOTE_SERVER_NAME" - echo "IP: $REMOTE_SERVER" - echo "Profile: ${PROFILE_NAME:-n/a}" - echo "DryRun: $DRY_RUN" - echo "Logging: $ENABLE_LOGGING" - echo "Containers: ${CRITICAL_CONTAINER_NAMES[*]:-n/a}" - echo "Delayed: ${DELAYED_CONTAINERS[*]:-n/a}" - echo "Excludes: ${EXCLUDE_DIRS[*]:-n/a}" + echo "$ICON_HEALTH rootfs threshold: ${SYS_WATCHDOG_ROOTFS_PCT}% toggle: $SYS_WATCHDOG_CHECK_ROOTFS" + echo "$ICON_HEALTH log threshold: ${SYS_WATCHDOG_LOG_PCT}% toggle: $SYS_WATCHDOG_CHECK_LOG" + echo "$ICON_MEM RAM threshold: ${SYS_WATCHDOG_MEM_GB}GB free toggle: $SYS_WATCHDOG_CHECK_RAM" + echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}% toggle: $SYS_WATCHDOG_CHECK_ARC" + echo "$ICON_ZFS ARC release: ${SYS_WATCHDOG_ARC_RELEASE_PCT}%" + echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C toggle: $SYS_WATCHDOG_CHECK_CPU_TEMP" + echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x cores toggle: $SYS_WATCHDOG_CHECK_LOAD" + echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT} toggle: $SYS_WATCHDOG_CHECK_ZOMBIES" + echo "$ICON_CONTAINERS Docker daemon: toggle: $SYS_WATCHDOG_CHECK_DOCKER_DAEMON" + echo "$ICON_CONTAINERS Containers: toggle: $SYS_WATCHDOG_CHECK_CONTAINERS" + echo "$ICON_SHIELD Strike limit: $SYS_WATCHDOG_STRIKE_LIMIT" + echo "$ICON_TIME Interval: ${SYSTEM_WATCHDOG_INTERVAL}s" + echo "$ICON_REBOOT_SMART Reboot limit: $SYS_WATCHDOG_REBOOT_LIMIT in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs" + echo "$ICON_ZFS Abort ZFS unhealthy: $SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" + echo "$ICON_GEAR Abort on parity: $SYS_WATCHDOG_ABORT_ON_PARITY" + echo "$ICON_MOVER Abort on mover: $SYS_WATCHDOG_ABORT_ON_MOVER" + echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" -} \ No newline at end of file + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboot will be executed" + +# ----------------------------------------------------------------------------------------------- +# STATE HELPERS — defined once, used every cycle +# ----------------------------------------------------------------------------------------------- + +get_strikes() { + local key="$1" + grep -E "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2 +} + +set_strikes() { + local key="$1" count="$2" + grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp" + echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp" + mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE" +} + +increment_strikes() { + local key="$1" + local current + current=$(get_strikes "$key") + [[ -z "$current" ]] && current=0 + ((current++)) + set_strikes "$key" "$current" + echo "$current" +} + +reset_strikes() { + local key="$1" + set_strikes "$key" 0 +} + +purge_old_reboots() { + local now + now=$(date +%s) + local cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW )) + grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do + [[ "$ts" -gt "$cutoff" ]] && echo "$ts" + done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp" + mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG" +} + +count_recent_reboots() { + purge_old_reboots + grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0 +} + +log_reboot() { + date +%s >> "$SYS_WATCHDOG_REBOOT_LOG" +} + +check_abort_conditions() { + local should_abort=false + + if command -v zpool >/dev/null 2>&1; then + local unhealthy + unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true) + if [[ -n "$unhealthy" ]]; then + if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then + error "$ICON_ZFS ZFS pool unhealthy — aborting reboot" + notify "System watchdog aborted reboot on $(hostname) — ZFS pool unhealthy" "System Watchdog" "warning" + should_abort=true + else + warn "$ICON_ZFS ZFS pool unhealthy — continuing reboot" + fi + fi + fi + + if [[ -f /var/local/emhttp/parity-date.txt ]]; then + if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then + if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then + error "$ICON_DISK Parity check running — aborting reboot" + notify "System watchdog aborted reboot on $(hostname) — parity running" "System Watchdog" "warning" + should_abort=true + else + warn "$ICON_DISK Parity check running — continuing reboot" + fi + fi + fi + + if pgrep -f "mover" >/dev/null 2>&1; then + if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then + error "$ICON_MOVER Mover running — aborting reboot" + notify "System watchdog aborted reboot on $(hostname) — mover running" "System Watchdog" "warning" + should_abort=true + else + warn "$ICON_MOVER Mover running — continuing reboot" + fi + fi + + [[ "$should_abort" == true ]] && return 1 + return 0 +} + +run_strike_check() { + local key="$1" triggered="$2" description="$3" + if [[ "$triggered" == true ]]; then + local strikes + strikes=$(increment_strikes "$key") + warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT" + if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then + error "$description hit strike limit — reboot triggered" + reset_strikes "$key" + return 0 + fi + else + local current + current=$(get_strikes "$key") + if [[ -n "$current" && "$current" -gt 0 ]]; then + reset_strikes "$key" + fi + fi + return 1 +} + +do_reboot() { + local triggers=("$@") + + if ! check_abort_conditions; then + return + fi + + RECENT_REBOOTS=$(count_recent_reboots) + info "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT" + + if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then + error "Reboot limit hit — shutting down instead" + notify "Reboot loop detected on $(hostname) — shutting down after $RECENT_REBOOTS reboots — conditions: ${triggers[*]}" "System Watchdog" "warning" + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would shutdown now" + return + fi + sync + /sbin/poweroff + return + fi + + notify "System watchdog reboot triggered on $(hostname) — conditions: ${triggers[*]}" "System Watchdog" "warning" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — reboot sequence would begin now" + return + fi + + log_reboot + + info "Shutting down VMs..." + if command -v virsh >/dev/null 2>&1; then + for VM in $(virsh list --name 2>/dev/null); do + [[ -z "$VM" ]] && continue + virsh shutdown "$VM" >/dev/null 2>&1 + done + sleep 30 + fi + + info "Stopping Docker containers..." + if command -v docker >/dev/null 2>&1; then + docker ps -q | xargs -r docker stop >/dev/null 2>&1 + fi + + info "Stopping User Scripts..." + pkill -f "/tmp/user.scripts" 2>/dev/null || true + + info "Syncing disks..." + sync + + echo "" + echo "$ICON_REBOOT_SMART Rebooting system NOW..." + sleep 5 + /sbin/reboot +} + +# ----------------------------------------------------------------------------------------------- +# CLEAN SHUTDOWN — trap SIGTERM/SIGINT from array stop +# ----------------------------------------------------------------------------------------------- +WATCHDOG_RUNNING=true + +cleanup() { + echo "" + info "System watchdog received shutdown signal — stopping cleanly" + WATCHDOG_RUNNING=false + exit 0 +} + +trap cleanup SIGTERM SIGINT + +# ----------------------------------------------------------------------------------------------- +# ━━━ CONTINUOUS MONITORING LOOP ━━━ +# ----------------------------------------------------------------------------------------------- +info "System watchdog started — checking every ${SYSTEM_WATCHDOG_INTERVAL}s" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +CYCLE=0 + +while [[ "$WATCHDOG_RUNNING" == true ]]; do + ((CYCLE++)) + + # Re-source Master.conf each cycle — picks up config changes without restart + source "$SCRIPT_DIR/../Master.conf" + SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 )) + TOTAL_CORES=$(nproc) + + # Per-cycle triggers — cleared each iteration + TRIGGERS=() + + # ── rootfs usage ───────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then + ROOTFS_USED=$(df / --output=pcent | tail -1 | tr -d ' %') + TRIGGERED=false + [[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]] && TRIGGERED=true + run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && \ + TRIGGERS+=("rootfs=${ROOTFS_USED}%") + fi + + # ── /var/log usage ──────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then + LOG_USED=$(df -P /var/log | awk 'NR==2 {print $5}' | tr -d '%') + TRIGGERED=false + [[ "$LOG_USED" -ge "$SYS_WATCHDOG_LOG_PCT" ]] && TRIGGERED=true + run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && \ + TRIGGERS+=("log=${LOG_USED}%") + fi + + # ── Free RAM ───────────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then + MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) + MEM_GB=$((MEM_KB / 1024 / 1024)) + TRIGGERED=false + [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]] && TRIGGERED=true + run_strike_check "ram" "$TRIGGERED" "RAM ${MEM_GB}GB free" && \ + TRIGGERS+=("low_ram=${MEM_GB}GB") + fi + + # ── ZFS ARC ────────────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then + ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX )) + TRIGGERED=false + if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then + sync; echo 3 > /proc/sys/vm/drop_caches; sleep 5 + ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX )) + [[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]] && TRIGGERED=true + fi + run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned" && \ + TRIGGERS+=("arc_pinned=${ARC_PCT}%") + fi + + # ── CPU temperature ────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then + CPU_TEMP="" + if command -v sensors >/dev/null 2>&1; then + CPU_TEMP=$(sensors 2>/dev/null | grep -i "Package id 0\|Tctl\|CPU Temp" | \ + awk '{print $NF}' | tr -d '+°C' | head -1) + fi + if [[ -n "$CPU_TEMP" ]]; then + CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP") + TRIGGERED=false + [[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]] && TRIGGERED=true + run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && \ + TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C") + fi + fi + + # ── Load average ───────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then + LOAD=$(awk '{print $1}' /proc/loadavg) + LOAD_INT=$(printf "%.0f" "$LOAD") + LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) + TRIGGERED=false + [[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]] && TRIGGERED=true + run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && \ + TRIGGERS+=("load=${LOAD}") + fi + + # ── Zombie processes ───────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then + ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null) + ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}" + ZOMBIE_COUNT="${ZOMBIE_COUNT//[^0-9]/}" + ZOMBIE_COUNT="${ZOMBIE_COUNT:-0}" + TRIGGERED=false + [[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]] && TRIGGERED=true + run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && \ + TRIGGERS+=("zombies=${ZOMBIE_COUNT}") + fi + + # ── Docker daemon ──────────────────────────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then + TRIGGERED=false + ! timeout 10 docker ps >/dev/null 2>&1 && TRIGGERED=true + run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon unresponsive" && \ + TRIGGERS+=("docker_daemon") + fi + + # ── Required containers from skip list ─────────────────────────────────────────────────── + if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then + FAILED_CONTAINERS=() + while IFS= read -r container; do + [[ -z "$container" ]] && continue + STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") + [[ "$STATUS" != "true" ]] && FAILED_CONTAINERS+=("$container") + done < "$SYS_WATCHDOG_FAILED_FILE" + + if [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]]; then + run_strike_check "failed_containers" "true" "required containers stopped" && \ + TRIGGERS+=("containers=${FAILED_CONTAINERS[*]}") + fi + fi + + # ── Evaluate triggers ──────────────────────────────────────────────────────────────────── + if [[ ${#TRIGGERS[@]} -gt 0 ]]; then + echo "" + echo "━━━ $ICON_REBOOT_SMART System Watchdog — Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━" + for t in "${TRIGGERS[@]}"; do + echo " $ICON_REBOOT_SMART $t" + done + echo "" + do_reboot "${TRIGGERS[@]}" + else + log "Cycle $CYCLE — system healthy ($(date '+%H:%M:%S'))" + fi + + # Sleep until next cycle — interruptible by SIGTERM + sleep "$SYSTEM_WATCHDOG_INTERVAL" & + wait $! + +done \ No newline at end of file