diff --git a/Docker_essentials/docker_watchdog.sh b/Docker_essentials/docker_watchdog.sh new file mode 100644 index 0000000..a0a4849 --- /dev/null +++ b/Docker_essentials/docker_watchdog.sh @@ -0,0 +1,305 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Docker Watchdog -------------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Self-healing watchdog for Docker containers — monitors memory, CPU and HTTP responsiveness. +# Restarts containers that exceed configured thresholds using a strike system for CPU and +# responsiveness checks to avoid restarting on brief spikes. +# +# Behaviour: +# Memory — immediate restart if hard limit is exceeded +# CPU — strike system, restarts after CPU_FAIL_LIMIT consecutive over-threshold checks +# HTTP — strike system, restarts after RESP_FAIL_LIMIT consecutive failed curl checks +# +# Strike system: +# Strikes persist between runs via WATCHDOG_STATE_FILE (/tmp — resets on reboot) +# Strike cadence depends on cron schedule: +# Every 15min + 2 strikes = 30min sustained abuse before restart +# Every 10min + 2 strikes = 20min sustained abuse before restart +# Every 5min + 2 strikes = 10min sustained abuse before restart +# +# All configuration lives in Master.conf under the Docker Watchdog section. +# Supports --dry-run to show what would be restarted without taking any action. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# Auto-detect total CPU cores for normalisation +TOTAL_CORES=$(nproc) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +# ROOT CHECK +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" +info "$ICON_WATCHDOG Watchdog initialising — $TOTAL_CORES cores detected" + +# Ensure state file exists +touch "$WATCHDOG_STATE_FILE" 2>/dev/null || { + error "Cannot create state file: $WATCHDOG_STATE_FILE" + exit 1 +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_WATCHDOG Containers monitored: ${!WATCHDOG_CONTAINERS[*]}" + echo "$ICON_MEM Soft mem threshold: ${SOFT_MEM_THRESHOLD}% of per-container limit" + echo "$ICON_ZFS CPU soft threshold: ${SOFT_CPU_THRESHOLD}%" + echo "$ICON_ZFS CPU hard threshold: ${HARD_CPU_THRESHOLD}%" + echo "$ICON_RETRY CPU fail limit: ${CPU_FAIL_LIMIT} strikes" + echo "$ICON_PING Resp fail limit: ${RESP_FAIL_LIMIT} strikes" + echo "$ICON_TIME Curl timeout: ${CURL_TIMEOUT}s" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" + +# ----------------------------------------------------------------------------------------------- +# STATE HELPERS +# Reads and writes per-container strike counts to the state file. +# State file format: container:metric:count +# ----------------------------------------------------------------------------------------------- + +# Returns current strike count for a container/metric pair. +# Usage: get_strikes "Emby" "CPU" +get_strikes() { + local container="$1" metric="$2" + grep -E "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f3 +} + +# Sets strike count for a container/metric pair. +# Usage: set_strikes "Emby" "CPU" 2 +set_strikes() { + local container="$1" metric="$2" count="$3" + grep -vE "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null > "${WATCHDOG_STATE_FILE}.tmp" + echo "${container}:${metric}:${count}" >> "${WATCHDOG_STATE_FILE}.tmp" + mv "${WATCHDOG_STATE_FILE}.tmp" "$WATCHDOG_STATE_FILE" +} + +# ----------------------------------------------------------------------------------------------- +# HELPERS +# ----------------------------------------------------------------------------------------------- + +# Fetches memory and CPU stats for a container in a single docker stats call. +# Returns: MEM_USAGE|CPU_PERCENT +parse_stats() { + local container="$1" + docker stats --no-stream --format "{{.MemUsage}}|{{.CPUPerc}}" "$container" +} + +# Converts a memory value and unit to MB. +# Supports KiB, MiB, GiB — returns UNKNOWN for unrecognised units. +convert_to_mb() { + local value="$1" unit="$2" + case "$unit" in + KiB) awk "BEGIN {print $value / 1024}" ;; + MiB) echo "$value" ;; + GiB) awk "BEGIN {print $value * 1024}" ;; + *) echo "UNKNOWN" ;; + esac +} + +# Restarts a container locally. +# In dry run mode reports what would happen without acting. +restart_container() { + local container="$1" reason="$2" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart $container ($reason)" + return + fi + + info "Restarting $container ($reason)..." + if docker restart "$container" >/dev/null 2>&1; then + echo "$ICON_STARTED $container restarted" + log "Restarted $container — reason: $reason" + else + error "Failed to restart $container" + fi +} + +# ----------------------------------------------------------------------------------------------- +# MEMORY CHECK +# Compares current container memory usage against its configured hard limit. +# Restarts immediately if at or above 100% of limit. +# Warns if at or above SOFT_MEM_THRESHOLD % of limit. +# Usage: check_memory "Emby" 16384 +# ----------------------------------------------------------------------------------------------- +check_memory() { + local container="$1" limit_mb="$2" + local stats mem_raw mem_val mem_unit mem_mb usage_pct + + stats=$(parse_stats "$container") + mem_raw=$(echo "$stats" | awk -F'|' '{print $1}' | awk '{print $1}') + mem_val=$(echo "$mem_raw" | sed -E 's/([0-9.]+).*/\1/') + mem_unit=$(echo "$mem_raw" | sed -E 's/[0-9.]+([a-zA-Z]+).*/\1/') + mem_mb=$(convert_to_mb "$mem_val" "$mem_unit") + local mem_int + mem_int=$(printf "%.0f" "$mem_mb") + usage_pct=$(( (mem_int * 100) / limit_mb )) + + if (( usage_pct >= 100 )); then + error "$ICON_MEM $container memory ${mem_int}MB (${usage_pct}%) — exceeded ${limit_mb}MB hard limit" + restart_container "$container" "memory hard limit" + set_strikes "$container" "CPU" 0 + set_strikes "$container" "RESP" 0 + elif (( usage_pct >= SOFT_MEM_THRESHOLD )); then + warn "$ICON_MEM $container memory ${mem_int}MB (${usage_pct}% of ${limit_mb}MB limit)" + else + success "$ICON_MEM $container memory ${mem_int}MB (${usage_pct}% of ${limit_mb}MB limit)" + fi +} + +# ----------------------------------------------------------------------------------------------- +# CPU CHECK +# Normalises CPU usage against total core count and applies the strike system. +# Warns at SOFT_CPU_THRESHOLD, strikes at HARD_CPU_THRESHOLD. +# Restarts after CPU_FAIL_LIMIT consecutive strikes — resets strikes on restart or recovery. +# Usage: check_cpu "Emby" +# ----------------------------------------------------------------------------------------------- +check_cpu() { + local container="$1" + local stats cpu_raw cpu_norm cpu_int violations + + stats=$(parse_stats "$container") + cpu_raw=$(echo "$stats" | awk -F'|' '{print $2}' | tr -d '%') + cpu_norm=$(awk "BEGIN {print $cpu_raw / $TOTAL_CORES}") + cpu_int=$(printf "%.0f" "$cpu_norm") + + violations=$(get_strikes "$container" "CPU") + [[ -z "$violations" ]] && violations=0 + + if (( cpu_int >= HARD_CPU_THRESHOLD )); then + ((violations++)) + error "$ICON_ZFS $container CPU ${cpu_int}% — hard threshold hit ($violations/$CPU_FAIL_LIMIT strikes)" + set_strikes "$container" "CPU" "$violations" + elif (( cpu_int >= SOFT_CPU_THRESHOLD )); then + ((violations++)) + warn "$ICON_ZFS $container CPU ${cpu_int}% — soft threshold hit ($violations/$CPU_FAIL_LIMIT strikes)" + set_strikes "$container" "CPU" "$violations" + else + if (( violations > 0 )); then + info "$ICON_ZFS $container CPU ${cpu_int}% — recovered, resetting strikes" + else + success "$ICON_ZFS $container CPU ${cpu_int}%" + fi + set_strikes "$container" "CPU" 0 + violations=0 + fi + + if (( violations >= CPU_FAIL_LIMIT )); then + error "$ICON_ZFS $container hit CPU limit for $CPU_FAIL_LIMIT consecutive checks" + restart_container "$container" "sustained CPU abuse" + set_strikes "$container" "CPU" 0 + fi +} + +# ----------------------------------------------------------------------------------------------- +# RESPONSIVENESS CHECK +# Sends an HTTP request to the container's configured URL. +# Skips containers with no URL defined in WATCHDOG_CONTAINER_URLS. +# Applies the same strike system as CPU — restarts after RESP_FAIL_LIMIT consecutive failures. +# Usage: check_responsiveness "Emby" +# ----------------------------------------------------------------------------------------------- +check_responsiveness() { + local container="$1" + local url="${WATCHDOG_CONTAINER_URLS[$container]:-}" + + [[ -z "$url" ]] && return + + local fails + fails=$(get_strikes "$container" "RESP") + [[ -z "$fails" ]] && fails=0 + + if ! curl -s --max-time "$CURL_TIMEOUT" "$url" >/dev/null 2>&1; then + ((fails++)) + warn "$ICON_PING $container unresponsive at $url ($fails/$RESP_FAIL_LIMIT strikes)" + set_strikes "$container" "RESP" "$fails" + else + if (( fails > 0 )); then + info "$ICON_PING $container responsive again — resetting strikes" + else + success "$ICON_PING $container responsive at $url" + fi + set_strikes "$container" "RESP" 0 + fails=0 + fi + + if (( fails >= RESP_FAIL_LIMIT )); then + error "$ICON_PING $container unresponsive for $RESP_FAIL_LIMIT consecutive checks" + restart_container "$container" "HTTP unresponsive" + set_strikes "$container" "RESP" 0 + fi +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_WATCHDOG Watchdog Check ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_WATCHDOG Watchdog Check — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "" + +RESTARTED=() +SKIPPED=() + +START=$(date +%s) + +for container in "${!WATCHDOG_CONTAINERS[@]}"; do + echo "━━━ $ICON_CONTAINERS $container ━━━" + + # Verify container exists + if ! docker inspect "$container" &>/dev/null; then + warn "$container not found on this host — skipping" + SKIPPED+=("$container") + echo "" + continue + fi + + # Verify container is running + if ! docker ps --filter "name=^/${container}$" --format "{{.Names}}" | grep -qw "$container"; then + warn "$ICON_NOT_RUNNING $container is not running — skipping" + SKIPPED+=("$container") + echo "" + continue + fi + + check_memory "$container" "${WATCHDOG_CONTAINERS[$container]}" + check_cpu "$container" + check_responsiveness "$container" + + echo "" +done + +END=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "━━━━━ $ICON_SUMMARY WATCHDOG SUMMARY ━━━━━" +echo "$ICON_TIME $(date '+%Y-%m-%d %H:%M:%S')" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_WATCHDOG Monitored: ${#WATCHDOG_CONTAINERS[@]} containers" +echo "$ICON_NOT_RUNNING Skipped: ${#SKIPPED[@]} containers" +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Dry Run: no restarts executed" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Master.conf b/Master.conf index e1cfe7c..9a2f80d 100644 --- a/Master.conf +++ b/Master.conf @@ -2,7 +2,7 @@ # ---------------------------------------------------------------------------------------------- # ---------------------------- Master Variables for unRAID scripts ----------------------------- # ---------------------------------------------------------------------------------------------- - + # ━━━ Host Configuration ━━━ # List the Hostnames of both servers HOST1="unRAID-Gmer4Lfe" @@ -11,16 +11,16 @@ # Must have its key added to HOST1_SSH_Key same applies for HOST2 HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" - + # ━━━ Logging ━━━ ENABLE_LOGGING=true # false = only echo user-facing messages - + # ━━━ Git, Pull & Execute Script ━━━ - REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git" - TARGET_DIR="/mnt/user/appdata/unraid_scripts" - GITEA_SSH_KEY="/root/.ssh/id_gitea_rsync" - SSH_PORT=221 - +REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git" +TARGET_DIR="/mnt/user/appdata/unraid_scripts" +GITEA_SSH_KEY="/root/.ssh/id_gitea_rsync" +SSH_PORT=221 + # ━━━ Rsync Script Defaults ━━━ # These are the fallback values used when no matching profile is found. # Any share whose directory basename does not match a profile key below @@ -39,38 +39,34 @@ # Directories to exclude during transfer EXCLUDE_DIRS=() # Default global rsync options - DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --inplace --no-whole-file) - + DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) + # ━━━ Remote Health Checks ━━━ # Abort if remote rootfs usage is at or above this percentage. # Protects against rsync writing to rootfs when the remote array is down or drives are missing. # Recommended: 75 — gives headroom before the server becomes unstable ROOTFS_WARN=75 - + # ━━━ Daily Sync Shares ━━━ # Shares synced once daily by Orchestrators/daily_sync.sh # Add or remove paths here to manage what gets synced. # These shares have no profile entry and fall through to DEFAULT_RSYNC_OPTS above. - DAILY_SYNC_SHARES=( - /mnt/user/Anime_Movies-Old - /mnt/user/Anime_Shows-Old - /mnt/user/Anime_Shows - /mnt/user/appdata-Failover/Gmer4Lfe/ - /mnt/user/Intros - /mnt/user/Kids_Movies - /mnt/user/Kids_Tv_Shows - /mnt/user/Movies - /mnt/user/Music-New - /mnt/user/Music_Videos - /mnt/user/Nextcloud - /mnt/user/stand-up_comedy - /mnt/user/Tv_Shows - +DAILY_SYNC_SHARES=( + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Shows-Old + /mnt/user/Anime_Shows + /mnt/user/Books + /mnt/user/Intros + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movies + /mnt/user/Music_Videos + /mnt/user/Nextcloud + /mnt/user/stand-up_comedy + /mnt/user/Tv_Shows ) - + # ━━━ unRAID Essential Scripts ━━━ -# clear_logs.sh system log file paths - LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) # unRAID reboot script user warning time (seconds) REBOOT_SLEEP=300 # unRAID mover stop script timeout (seconds) @@ -81,7 +77,42 @@ PHP_CONF="/etc/php-fpm.d/www.conf" # php_fpm_max_children.sh max children value PHP_MAX_CHILDREN=250 - +# clear_logs.sh system log file paths + LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) + +# ━━━ Docker Watchdog ━━━ +# Containers to monitor with their memory hard limits in MB +# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240 8GB=8192 6GB=6144 4GB=4096 1GB=1024 +declare -A WATCHDOG_CONTAINERS=( + ["Emby"]=16384 + ["jellyfin_with_request"]=12288 + ["LidaTube"]=6144 + ["Tdarr"]=6144 + ["Code-Server"]=1024 +) + +# Containers to check for HTTP responsiveness — omit a container to skip its check +declare -A WATCHDOG_CONTAINER_URLS=( + ["Emby"]="http://localhost:8096" + ["Jellyfin-Gmer4Lfe"]="http://localhost:8095" +) + +# State file for tracking CPU and responsiveness strikes between runs +# Lives in /tmp — resets on reboot which is correct behaviour for strike tracking + WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" + +# CPU thresholds — normalised against total core count automatically at runtime + SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU + HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU + CPU_FAIL_LIMIT=2 # consecutive strikes before restart + +# Memory threshold + SOFT_MEM_THRESHOLD=80 # warn at this % of per-container hard limit + +# Responsiveness check settings + RESP_FAIL_LIMIT=2 # consecutive failures before restart + CURL_TIMEOUT=5 # seconds before curl gives up + # ━━━ Profile System ━━━ # Profiles are matched by directory basename (lowercased). # Example: /mnt/user/appdata-Failover/Arrs_Stack → profile key = arrs_stack @@ -98,7 +129,7 @@ # # Note: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS — # if you define it for a profile you must list all desired options explicitly - + # SPACE-SEPARATED STRINGS declare -A PROFILE_RSYNC_OPTS=( [arrs_stack]="-av --info=progress2 --human-readable --bwlimit=$BW_LIMIT --delete --inplace" @@ -107,7 +138,7 @@ declare -A PROFILE_RSYNC_OPTS=( [important-data]="-av --human-readable --bwlimit=$BW_LIMIT" [emby]="-av --human-readable --bwlimit=$BW_LIMIT" ) - + declare -A PROFILE_BW_LIMIT=( [arrs_stack]=5000 [critical-data]=9500 @@ -115,7 +146,7 @@ declare -A PROFILE_BW_LIMIT=( [important-data]=9500 [emby]=8000 ) - + declare -A PROFILE_RETRY_COUNT=( [arrs_stack]=3 [critical-data]=3 @@ -123,7 +154,7 @@ declare -A PROFILE_RETRY_COUNT=( [important-data]=3 [emby]=3 ) - + declare -A PROFILE_SLEEP=( [arrs_stack]=300 [critical-data]=300 @@ -131,7 +162,7 @@ declare -A PROFILE_SLEEP=( [important-data]=300 [emby]=300 ) - + # SPACE-SEPARATED STRINGS declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [arrs_stack]="Sonarr Lidarr Readarr Radarr Prowlarr Bazarr Pinchflat" @@ -140,7 +171,7 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [important-data]="Postgres-NextCloud NextCloud" [emby]="" ) - + # SPACE-SEPARATED STRINGS — containers that need a delay before starting declare -A PROFILE_DELAYED_CONTAINERS=( [arrs_stack]="" @@ -149,7 +180,7 @@ declare -A PROFILE_DELAYED_CONTAINERS=( [important-data]="NextCloud" [emby]="" ) - + declare -A PROFILE_CONTAINER_DELAY=( [arrs_stack]=5 [critical-data]=10 @@ -157,7 +188,7 @@ declare -A PROFILE_CONTAINER_DELAY=( [important-data]=10 [emby]=5 ) - + # SPACE-SEPARATED STRINGS declare -A PROFILE_EXCLUDE_DIRS=( [arrs_stack]="logs *.tmp" @@ -168,4 +199,4 @@ declare -A PROFILE_EXCLUDE_DIRS=( ) # ---------------------------------------------------------------------------------------------- # ---------------------- End Of User Variables, Please adjust above as needed ------------------ -# ---------------------------------------------------------------------------------------------- +# ---------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/common.sh b/common.sh index 7939641..3dc6ddf 100644 --- a/common.sh +++ b/common.sh @@ -2,7 +2,7 @@ # ----------------------------------------------------------------------------------------------- # ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ---------------------------- # ----------------------------------------------------------------------------------------------- -# Version: 2.1 +# Version: 2.2 # ----------------------------------------------------------------------------------------------- # Changelog: # v1.0 — Initial stable framework @@ -33,6 +33,7 @@ # v2.0 — ICON_PLUGIN added for User Scripts plugin operations # v2.1 — ICON_ZFS and ICON_MEM added for ZFS and memory diagnostics # Diagnostics icon group added to icon block +# v2.2 — ICON_WATCHDOG added for Docker watchdog monitoring operations # ----------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------- @@ -79,6 +80,7 @@ ICON_PHP="👥" # PHP-FPM operations # Diagnostics ICON_ZFS="📊" # ZFS ARC statistics ICON_MEM="🧠" # memory status +ICON_WATCHDOG="🐾" # docker watchdog monitoring operations # Output ICON_INFO="ℹ️"