#!/bin/bash # ============================================================================================== # ================================= Docker Watchdog ============================================ # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Two-tier self-healing container monitoring system. Called by # watchdog_orchestrator.sh via cron every 15 minutes as a single-pass run. # Tier 1 applies specific thresholds to explicitly configured containers. # Tier 2 scans everything else for generic health problems. Silent on clean # cycles, loud when something needs attention. # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # Tier 1 — Strict Per-Container Monitoring # Applies only to containers explicitly configured in host*.conf. # # Memory hard limits — immediate restart if container exceeds MB ceiling # Memory soft threshold — warn at SOFT_MEM_THRESHOLD % of limit (no restart) # CPU thresholds — strike system: warn at SOFT_CPU_THRESHOLD, restart after # CPU_FAIL_LIMIT consecutive strikes at HARD_CPU_THRESHOLD # HTTP responsiveness — strike system: restart after RESP_FAIL_LIMIT consecutive # failures against the configured endpoint # Required containers — must always be running; strike system before restart; # skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in window # # Tier 2 — Global Health Scan # Scans ALL running containers when WATCHDOG_SCAN_ALL=true. # Containers in WATCHDOG_SCAN_IGNORE are excluded. # # Unhealthy status — Docker HEALTHCHECK unhealthy → restart # OOM killed — kernel OOM kill detected → restart + notify # Crash loop — RestartCount climbing → notify; above WATCHDOG_CRASH_LIMIT # → restart → skip list if restart limit hit # Dead containers — remove + start (dead state cannot be restarted directly) # Unexpected exits — non-zero exit code → restart # # Cross-Cutting Intelligence # Applies to both tiers on every cycle. # # Startup grace period — no restarts for WATCHDOG_STARTUP_GRACE seconds after boot # Dependency ordering — dependency restarted first, dependent skipped this cycle # Restart loop protect — skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in window # Skip list auto-clear — removed when container seen running again # Notification batching — one summary per cycle, not one ping per event # Parity awareness — skips restart actions during parity check # Timeout protection — all docker commands wrapped in timeout # Docker daemon check — each cycle begins with daemon health check; hung daemon → # restart via rc.docker → stability_watchdog.sh escalates if needed # RAM emergency defer — reads RW_STATE_FILE; stands down while # resource_watchdog.sh is managing a RAM emergency # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Tiered Monitoring # Not all containers need the same monitoring strategy. Tier 1 gives explicit # control over the containers that matter most. Tier 2 is the catch-all that # requires no configuration and protects everything else. # # Strike vs Immediate # CPU spikes and HTTP failures are transient — brief spikes are normal during # transcoding or library scans. Memory leaks are not transient. CPU and HTTP # use a strike system to distinguish sustained problems from momentary ones. # Memory triggers immediate restart because a container at its ceiling is # actively leaking, not spiking. # # Loop Protection Over Persistence # A watchdog that keeps restarting a broken container is not helpful — it risks # making a database corruption worse. After WATCHDOG_CONTAINER_RESTART_LIMIT # attempts the container is skip-listed and the operator is notified. Automated # recovery stops. Human investigation begins. # # Dependency-Safe Ordering # When a container and its dependency are both down, restart the dependency # first and skip the dependent this cycle. Prevents false-alarm skip-listing # of containers whose only failure was starting before their dependency was ready. # # Silent When Healthy # Runs 96 times per day. Producing output on every clean cycle would make # logs useless. Output only when something needs attention or a heartbeat fires. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Lock Acquisition # Prevents concurrent execution via acquire_lock(). Safe at array start — # only one watchdog instance runs at a time. # # Host Detection # detect_hosts() identifies which server is running the script and aliases # all HOST*_WATCHDOG_* arrays to the correct host's values. # # Startup Grace Period # Restart actions suppressed for WATCHDOG_STARTUP_GRACE seconds after the # watchdog starts. Checks still run and log — only restarts are suppressed. # Prevents false-positive restarts while containers are still initialising. # # Restart Loop Protection # WATCHDOG_CONTAINER_RESTART_LIMIT restarts within WATCHDOG_CONTAINER_RESTART_WINDOW # hours triggers skip-listing and a critical notification. Skip list persists on # /boot/config/ — survives reboots intentionally. Auto-clears when container # is seen running again. # # Docker Daemon Health Check # First operation every cycle. Daemon not responding within DOCKER_TIMEOUT → # restart via /etc/rc.d/rc.docker → verify recovery. If still hung: log # critical, skip cycle. stability_watchdog.sh handles further escalation. # # RAM Emergency Deferral # Reads RW_STATE_FILE each cycle. If resource_watchdog.sh has set # mem_shutdown_active=true, all restart logic defers until the flag clears. # Stale state guard: if file is >2 hours old with flag still set, # resource_watchdog.sh has likely stopped — watchdog resumes normal operation. # # Timeout Protection # All docker commands wrapped in timeout. Daemon hangs cannot stall the # watchdog and leave containers unmonitored between cycles. # # Notification Batching # Events collected across a full cycle and sent as a single summary. # Prevents notification floods when a shared dependency failure cascades. # # ============================================================================================== # STATE FILES # ============================================================================================== # # WATCHDOG_STATE_FILE — strike counts, daemon flags (STATE_DIR — survives reboots) # DOCKER_WATCHDOG_FAILED_FILE — container skip list (STATE_DIR — survives reboots) # WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection (DATA_DIR) # RW_STATE_FILE — read-only: resource_watchdog RAM emergency flag # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # host*.conf # # HOST*_WATCHDOG_CONTAINERS # Memory hard limits per container. Format: "ContainerName:LimitInMB" # Aliased by detect_hosts() → WATCHDOG_CONTAINERS # # HOST*_WATCHDOG_CONTAINER_URLS # HTTP health check endpoints. Format: "ContainerName:http://host:port" # Aliased by detect_hosts() → WATCHDOG_CONTAINER_URLS # # HOST*_WATCHDOG_CONTAINER_API_CHECKS # API liveness checks — deeper than HTTP. Format: "ContainerName:URL|APIKey" # Use endpoints that require a live DB round-trip (e.g. Emby /System/Info). # Aliased by detect_hosts() → WATCHDOG_CONTAINER_API_CHECKS # # HOST*_WATCHDOG_REQUIRED_CONTAINERS # Containers that must always be running. Aliased by detect_hosts() → # WATCHDOG_REQUIRED_CONTAINERS # # HOST*_WATCHDOG_SCAN_IGNORE # Containers excluded from Tier 2 global scan. Aliased by detect_hosts() → # WATCHDOG_SCAN_IGNORE # # HOST*_WATCHDOG_DEPENDENCIES # Dependency ordering. Format: "Dependent:dep1 dep2". Aliased by # detect_hosts() → WATCHDOG_DEPENDENCIES # # master.conf # # WATCHDOG_STARTUP_GRACE # Seconds before restart actions begin after watchdog starts (default: 600) # # SOFT_MEM_THRESHOLD # Warn at this % of hard memory limit — no restart (default: 80) # # SOFT_CPU_THRESHOLD / HARD_CPU_THRESHOLD / CPU_FAIL_LIMIT # CPU monitoring thresholds and strike limit # # CURL_TIMEOUT / RESP_FAIL_LIMIT # HTTP health check timeout and consecutive failure limit # # WATCHDOG_SCAN_ALL # Enable Tier 2 global health scan (default: true) # # WATCHDOG_RESTART_UNHEALTHY / WATCHDOG_RESTART_DEAD / WATCHDOG_RESTART_CRASHED # Tier 2 action toggles # # WATCHDOG_NOTIFY_OOM / WATCHDOG_NOTIFY_CRASHLOOP / WATCHDOG_CRASH_LIMIT # OOM and crash loop detection toggles and threshold # # WATCHDOG_CONTAINER_RESTART_LIMIT / WATCHDOG_CONTAINER_RESTART_WINDOW # Restart loop protection: attempt limit and rolling window in hours # # WATCHDOG_BATCH_NOTIFY # Collect cycle events and send as one notification (default: true) # # DOCKER_WATCHDOG_HEARTBEAT_HOURS # Hours between alive heartbeat log entries # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # docker_watchdog.sh # Single-pass monitoring cycle — called by watchdog_orchestrator.sh every 15 min # # docker_watchdog.sh --dry-run # Run a full watchdog cycle without restarting anything. Shows what would # happen based on current container states. Use to verify configuration. # # docker_watchdog.sh --status # Show strike counts, skip list contents, grace period status, RAM emergency # deferral state, and last cycle timing. Then exit. # # docker_watchdog.sh --log # Verbose output — full detail for every container checked and every decision. # Use to debug why a container is or is not being restarted. # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" # ============================================================================================== # ━━━ Setup — runs once at start ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi # Skip if another instance is running — no pile-up during long operations acquire_lock # detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays detect_hosts if ! is_docker_enabled; then echo "Docker not enabled in Unraid settings — skipping cycle" exit 0 fi if ! command -v docker >/dev/null 2>&1; then error "Docker not found — cannot start watchdog" exit 1 fi [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" log "$ICON_GEAR Config: grace=${WATCHDOG_STARTUP_GRACE}s mem-soft=${SOFT_MEM_THRESHOLD}% cpu-soft=${SOFT_CPU_THRESHOLD}% cpu-hard=${HARD_CPU_THRESHOLD}% cpu-limit=${CPU_FAIL_LIMIT} http-limit=${RESP_FAIL_LIMIT} daemon-timeout=${DOCKER_TIMEOUT}s" log "$ICON_CONTAINERS Tier1: watched=${#WATCHDOG_CONTAINERS[@]} required=${#WATCHDOG_REQUIRED_CONTAINERS[@]} urls=${#WATCHDOG_CONTAINER_URLS[@]} restart-limit=${WATCHDOG_CONTAINER_RESTART_LIMIT}/${WATCHDOG_CONTAINER_RESTART_WINDOW}h" # Validate unRAID-specific commands used by this script # If rc.docker is missing or changed, daemon restart will fail — better to know now platform_require_cmd "/etc/rc.d/rc.docker" "" "" "Docker rc.d script" || warn "rc.docker not found — daemon restart unavailable if needed" platform_require_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" "" "" "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled" # Ensure state files exist touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \ "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null # Timeout for all docker commands — configurable via WATCHDOG_DAEMON_TIMEOUT in master.conf DOCKER_TIMEOUT="${WATCHDOG_DAEMON_TIMEOUT:-20}" # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" _watched="${!WATCHDOG_CONTAINERS[*]}"; echo "$ICON_CONTAINERS Watched: ${_watched:-none}" echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}" echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL" echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}" echo "$ICON_WATCHDOG Schedule: every 15 min (cron via watchdog_orchestrator)" echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s" echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY" echo "$ICON_WATCHDOG Docker timeout: ${DOCKER_TIMEOUT}s" echo "$ICON_GEAR Dry Run: $DRY_RUN" UPTIME_S=$(awk '{print int($1)}' /proc/uptime) echo "$ICON_TIME System uptime: $(format_duration $UPTIME_S)" [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]] && \ warn "Within startup grace period — restarts suppressed" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── # ============================================================================================== # Get strike count for a container from state file get_strikes() { grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0" } # Set strike count for a container in state file set_strikes() { local container="$1" count="$2" file="$3" if grep -q "^${container}:" "$file" 2>/dev/null; then sed -i "s/^${container}:.*/${container}:${count}/" "$file" else echo "${container}:${count}" >> "$file" fi } # Check if container is on the persistent skip list is_skipped() { grep -q "^${1}$" "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null } # Add container to persistent skip list — manual intervention required to recover add_to_skip_list() { local container="$1" reason="$2" if ! is_skipped "$container"; then echo "$container" >> "$DOCKER_WATCHDOG_FAILED_FILE" error "$container added to skip list — $reason" queue_notify "$container added to skip list on $(hostname) — $reason — manual intervention needed" "critical" fi } # Remove container from skip list — called when container is seen running again remove_from_skip_list() { sed -i "/^${1}$/d" "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null warn "$1 recovered — removed from skip list ✅" } # Log a restart event to the rolling restart history file log_restart() { local container="$1" local now now=$(date '+%Y-%m-%d %H:%M:%S') local cutoff cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S') echo "${container}|${now}" >> "$WATCHDOG_CONTAINER_RESTART_LOG" # Trim entries older than the rolling window local tmp="${WATCHDOG_CONTAINER_RESTART_LOG}.tmp" awk -F'|' -v cutoff="$cutoff" '$2 >= cutoff' \ "$WATCHDOG_CONTAINER_RESTART_LOG" > "$tmp" && \ mv "$tmp" "$WATCHDOG_CONTAINER_RESTART_LOG" } # Get number of times a container was restarted within the rolling window get_restart_count() { local container="$1" local cutoff cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S') awk -F'|' -v c="$container" -v cutoff="$cutoff" \ '$1==c && $2>=cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l } # Check if all dependencies of a container are currently running. # Returns 0 if all deps running (or no deps), 1 if any dep is down. dependencies_satisfied() { local container="$1" local deps="${WATCHDOG_DEPENDENCIES[$container]:-}" [[ -z "$deps" ]] && return 0 for dep in $deps; do local status status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null) if [[ "$status" != "true" ]]; then warn "$container — dependency $dep is not running — skipping restart this cycle" return 1 fi done return 0 } # Safe restart with all guards: # - Restart loop protection (skip list after limit) # - Dependency check (don't restart if deps down) # - Startup grace period (no restarts while booting) # - Dry run support # - Timeout protection on docker restart # # Returns: 0=restarted, 1=skipped, 2=added to skip list safe_restart() { local container="$1" reason="$2" # Restart loop protection — skip list if over limit local restart_count restart_count=$(get_restart_count "$container") if [[ "$restart_count" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then add_to_skip_list "$container" \ "restarted $restart_count times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h" return 2 fi # Dependency check dependencies_satisfied "$container" || return 1 # Startup grace period local uptime_s uptime_s=$(awk '{print int($1)}' /proc/uptime) if [[ "$uptime_s" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then warn "$container — within startup grace period (${uptime_s}s < ${WATCHDOG_STARTUP_GRACE}s) — skipping" return 1 fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart $container ($reason)" return 0 fi log "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]" if timeout "$DOCKER_TIMEOUT" docker restart "$container" >/dev/null 2>&1; then success "$ICON_STARTED $container restarted" log_restart "$container" return 0 else error "Failed to restart $container (timeout or error)" return 1 fi } # Queue a notification event for batch sending at end of cycle queue_notify() { local message="$1" severity="${2:-warning}" NOTIFY_EVENTS+=("${severity}|${message}") log "Queued: $message" } # Send all queued notifications — one batched summary or individual per event flush_notify() { [[ ${#NOTIFY_EVENTS[@]} -eq 0 ]] && return if [[ "$WATCHDOG_BATCH_NOTIFY" == "true" ]]; then local highest_severity="normal" local messages=() for event in "${NOTIFY_EVENTS[@]}"; do local sev="${event%%|*}" msg="${event#*|}" messages+=("$msg") [[ "$sev" == "critical" ]] && highest_severity="warning" [[ "$sev" == "warning" && "$highest_severity" == "normal" ]] && highest_severity="warning" done local summary summary=$(printf '%s. ' "${messages[@]}") notify "Docker Watchdog on $(hostname) — ${#NOTIFY_EVENTS[@]} event(s): $summary" \ "Docker Watchdog" "$highest_severity" else for event in "${NOTIFY_EVENTS[@]}"; do local sev="${event%%|*}" msg="${event#*|}" [[ "$sev" == "critical" ]] && sev="warning" notify "$msg" "Docker Watchdog" "$sev" done fi NOTIFY_EVENTS=() } # is_parity_running — delegates to adapter is_parity_running() { platform_is_maintenance_running; } # ============================================================================================== # ── DOCKER DAEMON HEALTH CHECK ──────────────────────────────────────────────────────────────── # ============================================================================================== # Checks Docker daemon responsiveness at the start of every cycle. # A hung daemon makes all container operations useless — check first, short-circuit if down. # # Strike system: # Each consecutive failed check adds a strike # At WATCHDOG_DAEMON_STRIKE_LIMIT → attempt daemon restart via rc.docker # After restart → wait WATCHDOG_DAEMON_RESTART_WAIT seconds → verify # If verified → clear strikes, continue cycle ✅ # If still hung → notify critical, skip cycle → stability_watchdog.sh escalates from here # # Returns: 0 = daemon healthy | 1 = daemon down, skip this cycle WATCHDOG_DAEMON_STRIKE_LIMIT="${WATCHDOG_DAEMON_STRIKE_LIMIT:-3}" # consecutive failed checks before restart attempt (master.conf) WATCHDOG_DAEMON_RESTART_WAIT="${WATCHDOG_DAEMON_RESTART_WAIT:-900}" # seconds to wait after restart before verifying (master.conf) # Loaded from state file — persists across single-pass runs WATCHDOG_DAEMON_STRIKES=$(get_strikes "daemon_strikes" "$WATCHDOG_STATE_FILE") WATCHDOG_DAEMON_STRIKES="${WATCHDOG_DAEMON_STRIKES//[^0-9]/}"; WATCHDOG_DAEMON_STRIKES="${WATCHDOG_DAEMON_STRIKES:-0}" _dr_raw=$(get_strikes "daemon_restarted_flag" "$WATCHDOG_STATE_FILE") [[ "$_dr_raw" == "true" ]] && WATCHDOG_DAEMON_RESTARTED=true || WATCHDOG_DAEMON_RESTARTED=false # ============================================================================================== # ── SYSTEM WATCHDOG COORDINATION ────────────────────────────────────────────────────────────── # ============================================================================================== # Reads system_watchdog.sh state file to check if a RAM emergency shutdown is active. # During RAM emergency: system_watchdog.sh has stopped non-essential containers to free RAM. # docker_watchdog.sh must not restart them — that would undo the emergency shutdown and # prevent RAM from recovering, creating an infinite restart/shutdown loop. # # Returns: # 0 = normal — run all checks # 1 = RAM emergency active — defer container management this cycle check_resource_watchdog_state() { # Returns 0 = normal operation | 1 = defer, resource_watchdog RAM emergency active local state_file="$RW_STATE_FILE" # No state file = resource_watchdog not yet run — assume normal [[ ! -f "$state_file" ]] && return 0 local mem_shutdown mem_shutdown=$(grep "^mem_shutdown_active=" "$state_file" 2>/dev/null | cut -d= -f2) [[ "$mem_shutdown" != "true" ]] && return 0 # ── Stale state guard ───────────────────────────────────────────────────────────────────── # If mem_shutdown_active=true but state file hasn't been updated in > 2 hours, # resource_watchdog.sh may not be running — don't defer indefinitely on stale state. local state_mtime now age_seconds stale_limit=7200 # 2 hours state_mtime=$(stat -c %Y "$state_file" 2>/dev/null || echo 0) now=$(date +%s) age_seconds=$(( now - state_mtime )) if [[ "$age_seconds" -gt "$stale_limit" ]]; then warn "mem_shutdown_active=true but state file is ${age_seconds}s old — may be stale" warn "resource_watchdog.sh may not be running — resuming normal container management" return 0 # Resume normal — don't defer indefinitely on stale state fi return 1 # Defer — RAM emergency confirmed and state is fresh } check_docker_daemon() { # docker info is more definitive than docker ps for daemon health if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then if [[ "$WATCHDOG_DAEMON_STRIKES" -gt 0 ]]; then log "$ICON_STARTED Docker daemon recovered — clearing strikes" queue_notify "Docker daemon recovered on $(hostname)" "normal" WATCHDOG_DAEMON_STRIKES=0 WATCHDOG_DAEMON_RESTARTED=false set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE" set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE" fi # Always clear confirmed-down flag when daemon is healthy — unconditional so it # clears correctly after a reboot (strikes reset to 0 but flag persists on flash). set_strikes "daemon_confirmed_down" "false" "$WATCHDOG_STATE_FILE" return 0 fi WATCHDOG_DAEMON_STRIKES=$(( WATCHDOG_DAEMON_STRIKES + 1 )) set_strikes "daemon_strikes" "$WATCHDOG_DAEMON_STRIKES" "$WATCHDOG_STATE_FILE" warn "$ICON_WATCHDOG Docker daemon not responding (strike $WATCHDOG_DAEMON_STRIKES/$WATCHDOG_DAEMON_STRIKE_LIMIT)" if [[ "$WATCHDOG_DAEMON_STRIKES" -lt "$WATCHDOG_DAEMON_STRIKE_LIMIT" ]]; then warn "Skipping monitoring cycle — waiting for daemon to recover" return 1 fi if [[ "$WATCHDOG_DAEMON_RESTARTED" == true ]]; then # Restart was already attempted last cycle and daemon is still down. # Write confirmed-down flag — stability_watchdog reads this and strikes toward reboot. error "Docker daemon still unresponsive after restart attempt" set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE" queue_notify "Docker daemon hung on $(hostname) — restart failed — stability_watchdog escalating" "critical" flush_notify return 1 fi error "Docker daemon unresponsive — attempting restart" notify "Docker daemon hung on $(hostname) — attempting restart" "Docker Watchdog" "warning" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart Docker daemon via platform_restart_service docker" return 1 fi # timeout 180: Docker daemon stop can block for 30+ min on a busy host. WATCHDOG_DAEMON_RESTARTED=true set_strikes "daemon_restarted_flag" "true" "$WATCHDOG_STATE_FILE" if timeout 180 platform_restart_service docker; then log "Docker daemon restart issued — waiting ${WATCHDOG_DAEMON_RESTART_WAIT}s..." sleep "$WATCHDOG_DAEMON_RESTART_WAIT" if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then success "Docker daemon restarted successfully ✅" notify "Docker daemon restarted successfully on $(hostname)" "Docker Watchdog" "normal" WATCHDOG_DAEMON_STRIKES=0 WATCHDOG_DAEMON_RESTARTED=false set_strikes "daemon_strikes" 0 "$WATCHDOG_STATE_FILE" set_strikes "daemon_restarted_flag" "false" "$WATCHDOG_STATE_FILE" set_strikes "daemon_confirmed_down" "false" "$WATCHDOG_STATE_FILE" return 0 else # Restart issued but daemon still down — flag for stability_watchdog on next cycle error "Docker daemon did not recover after restart" set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE" queue_notify "Docker daemon restart failed on $(hostname) — stability_watchdog escalating" "critical" flush_notify return 1 fi else # platform_restart_service returned non-zero — flag immediately, stability_watchdog escalates error "Failed to issue Docker daemon restart — platform_restart_service docker failed" set_strikes "daemon_confirmed_down" "true" "$WATCHDOG_STATE_FILE" queue_notify "Docker daemon restart command failed on $(hostname) — stability_watchdog escalating" "critical" flush_notify return 1 fi } # ============================================================================================== # ━━━ Single-Pass Monitoring Run ━━━ # ============================================================================================== echo "━━━ $ICON_WATCHDOG Docker Watchdog — $(date '+%Y-%m-%d %H:%M:%S') ━━━" CYCLE_START=$(date +%s) # ── Per-run state ───────────────────────────────────────────────────────────────────────── NOTIFY_EVENTS=() T1_RESTARTS=0 T1_WARNINGS=0 T2_RESTARTS=0 T2_WARNINGS=0 # OOM handled set — tracks containers already handled for OOM this cycle # Prevents restart loop from OOMKilled flag persisting after restart declare -A OOM_HANDLED # Rebuild ignore map each cycle (config may have changed) declare -A IGNORE_MAP for c in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do [[ -n "$c" ]] && IGNORE_MAP["$c"]=1 done # ── Skip list visibility ───────────────────────────────────────────────────────────────── _skip_contents=$(cat "$DOCKER_WATCHDOG_FAILED_FILE" 2>/dev/null | tr '\n' ' ' | xargs) [[ -n "$_skip_contents" ]] && warn "$ICON_SKIP Skip list active: $_skip_contents — manual intervention needed" # ── Docker daemon health check — first check every run ────────────────────────────────── # If daemon is hung all container operations will fail — check first, skip run if down if ! check_docker_daemon; then exit 0 fi # ── Parity check — skip restarts during parity ─────────────────────────────────────────── if is_parity_running; then echo "Parity in progress — restart actions skipped ($(date '+%H:%M:%S'))" exit 0 fi # ── RAM emergency check — resource_watchdog.sh managing containers ──────────────────────── # If resource_watchdog.sh has triggered a hard RAM shutdown, defer all container # management this run to prevent undoing the emergency stop and re-pressuring RAM. if ! check_resource_watchdog_state; then MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) MEM_GB=$(( MEM_KB / 1024 / 1024 )) warn "RAM emergency active (${MEM_GB}GB free) — resource_watchdog.sh managing containers" warn "Deferring all container restart logic this run" log "Waiting for RAM to recover above ${RW_RAM_RECOVER_GB:-20}GB before resuming" exit 0 fi # ── Startup grace period ────────────────────────────────────────────────────────────────── UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime) IN_GRACE_PERIOD=false [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]] && IN_GRACE_PERIOD=true # ========================================================================================== # ── TIER 1 — Strict Monitoring ──────────────────────────────────────────────────────────── # ========================================================================================== # ── Required containers ─────────────────────────────────────────────────────────────────── # Must always be running — strike system before restart, skip list after limit if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do [[ -z "$container" ]] && continue STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \ "$container" 2>/dev/null) if is_skipped "$container"; then if [[ "$STATUS" == "true" ]]; then # Container recovered — remove from skip list and clear its history remove_from_skip_list "$container" set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" sed -i "/^${container}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null queue_notify "$container recovered on $(hostname) — removed from skip list" "normal" else warn "$container — on skip list, manual intervention needed" fi continue fi if [[ "$STATUS" == "true" ]]; then # Running — clear any strikes set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" log "$ICON_RUNNING $container — running ✅" else STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE") STRIKES=$(( STRIKES + 1 )) set_strikes "$container" "$STRIKES" "$WATCHDOG_STATE_FILE" warn "$container — not running (strike $STRIKES/$SYS_WATCHDOG_STRIKE_LIMIT)" ((T1_WARNINGS++)) if [[ "$STRIKES" -ge "$SYS_WATCHDOG_STRIKE_LIMIT" ]]; then result=0 safe_restart "$container" "required container down" || result=$? case $result in 0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE" ((T1_RESTARTS++)) queue_notify "$container was down and restarted on $(hostname)" "warning" ;; 2) : ;; # Added to skip list — already notified *) queue_notify "$container failed to restart on $(hostname)" "warning" ;; esac fi fi done fi # ── Memory and CPU monitoring ───────────────────────────────────────────────────────────── if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then STATS=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \ --format "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}" 2>/dev/null) TOTAL_CORES=$(nproc 2>/dev/null || echo 1) for container in "${!WATCHDOG_CONTAINERS[@]}"; do MEM_LIMIT_MB="${WATCHDOG_CONTAINERS[$container]}" CONTAINER_STATS=$(echo "$STATS" | grep "^${container}|" | head -1) [[ -z "$CONTAINER_STATS" ]] && continue # ── Memory ──────────────────────────────────────────────────────────────────────── MEM_USAGE=$(echo "$CONTAINER_STATS" | cut -d'|' -f2 | awk '{print $1}') MEM_UNIT=$(echo "$MEM_USAGE" | grep -oE '[A-Za-z]+') MEM_VALUE=$(echo "$MEM_USAGE" | grep -oE '[0-9.]+') case "$MEM_UNIT" in GiB|GB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE * 1024}") ;; MiB|MB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE}") ;; KiB|KB) MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_VALUE / 1024}") ;; *) MEM_MB=0 ;; esac # Soft memory threshold — warn when approaching hard limit SOFT_MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_LIMIT_MB * $SOFT_MEM_THRESHOLD / 100}") if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then # Hard limit exceeded — immediate restart error "$container — memory ${MEM_MB}MB exceeded hard limit ${MEM_LIMIT_MB}MB" safe_restart "$container" "memory hard limit exceeded" ((T1_RESTARTS++)) queue_notify "$container exceeded memory hard limit on $(hostname) — restarted" "warning" elif [[ "$MEM_MB" -ge "$SOFT_MEM_MB" ]]; then # Soft threshold — warn only, no restart warn "$container — memory ${MEM_MB}MB approaching limit (${SOFT_MEM_THRESHOLD}% of ${MEM_LIMIT_MB}MB)" ((T1_WARNINGS++)) fi # ── CPU ─────────────────────────────────────────────────────────────────────────── CPU_RAW=$(echo "$CONTAINER_STATS" | cut -d'|' -f3 | tr -d '%') CPU_NORM=$(awk "BEGIN {printf \"%.1f\", $CPU_RAW / $TOTAL_CORES}") CPU_INT=$(printf "%.0f" "$CPU_NORM") if [[ "$CPU_INT" -ge "$HARD_CPU_THRESHOLD" ]]; then # Hard CPU threshold — strike system → restart CPU_STRIKES=$(get_strikes "${container}_cpu" "$WATCHDOG_STATE_FILE") CPU_STRIKES=$(( CPU_STRIKES + 1 )) set_strikes "${container}_cpu" "$CPU_STRIKES" "$WATCHDOG_STATE_FILE" warn "$container — CPU ${CPU_NORM}% (strike $CPU_STRIKES/$CPU_FAIL_LIMIT)" if [[ "$CPU_STRIKES" -ge "$CPU_FAIL_LIMIT" ]]; then safe_restart "$container" "CPU hard threshold exceeded ${CPU_NORM}%" set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" ((T1_RESTARTS++)) queue_notify "$container CPU ${CPU_NORM}% on $(hostname) — restarted" "warning" fi elif [[ "$CPU_INT" -ge "$SOFT_CPU_THRESHOLD" ]]; then # Soft CPU threshold — warn only, no restart, clear strikes warn "$container — CPU ${CPU_NORM}% (above soft threshold ${SOFT_CPU_THRESHOLD}%)" ((T1_WARNINGS++)) set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" else # Normal — clear CPU strikes set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE" log "$container — CPU ${CPU_NORM}% | MEM ${MEM_MB}MB / ${MEM_LIMIT_MB}MB ✅" fi done fi # ── HTTP responsiveness ─────────────────────────────────────────────────────────────────── if [[ ${#WATCHDOG_CONTAINER_URLS[@]} -gt 0 ]]; then for container in "${!WATCHDOG_CONTAINER_URLS[@]}"; do URL="${WATCHDOG_CONTAINER_URLS[$container]}" if curl -sf --max-time "$CURL_TIMEOUT" "$URL" >/dev/null 2>&1; then set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE" else HTTP_STRIKES=$(get_strikes "${container}_http" "$WATCHDOG_STATE_FILE") HTTP_STRIKES=$(( HTTP_STRIKES + 1 )) set_strikes "${container}_http" "$HTTP_STRIKES" "$WATCHDOG_STATE_FILE" warn "$container — not responding at $URL (strike $HTTP_STRIKES/$RESP_FAIL_LIMIT)" ((T1_WARNINGS++)) if [[ "$HTTP_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then result=0 safe_restart "$container" "HTTP unresponsive at $URL" || result=$? if [[ $result -eq 0 ]]; then set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE" ((T1_RESTARTS++)) queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning" fi fi fi done fi # ── API liveness checks ─────────────────────────────────────────────────────────────────── # Catches containers that serve HTTP 200 but are internally frozen (DB lock, deadlocked # thread, etc.). Endpoint must require a live DB round-trip to respond successfully. # Format per entry: "URL|APIKey" if [[ ${#WATCHDOG_CONTAINER_API_CHECKS[@]} -gt 0 ]]; then for container in "${!WATCHDOG_CONTAINER_API_CHECKS[@]}"; do IFS='|' read -r _api_url _api_key <<< "${WATCHDOG_CONTAINER_API_CHECKS[$container]}" # Skip entirely if key is absent or a placeholder — check is optional protection if [[ -z "$_api_key" || "$_api_key" == "YOUR_API_KEY"* || "$_api_key" == "placeholder"* ]]; then log "$container — API check skipped (no key configured)" continue fi _api_http=$(curl -s --max-time "$CURL_TIMEOUT" \ -H "X-Emby-Token: ${_api_key}" \ -o /tmp/_varaverk_api_check \ -w "%{http_code}" \ "$_api_url" 2>/dev/null) _resp=$(cat /tmp/_varaverk_api_check 2>/dev/null) # 401/403 = wrong key — skip silently, don't penalise the container if [[ "$_api_http" == "401" || "$_api_http" == "403" ]]; then log "$container — API check skipped (HTTP $_api_http — key may be wrong or revoked)" continue fi if echo "$_resp" | jq -e '.ServerName // .Id // .Version' >/dev/null 2>&1; then set_strikes "${container}_api" 0 "$WATCHDOG_STATE_FILE" else API_STRIKES=$(get_strikes "${container}_api" "$WATCHDOG_STATE_FILE") API_STRIKES=$(( API_STRIKES + 1 )) set_strikes "${container}_api" "$API_STRIKES" "$WATCHDOG_STATE_FILE" warn "$container — API unresponsive at $_api_url (strike $API_STRIKES/$RESP_FAIL_LIMIT)" ((T1_WARNINGS++)) if [[ "$API_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then result=0 safe_restart "$container" "API unresponsive at $_api_url" || result=$? if [[ $result -eq 0 ]]; then set_strikes "${container}_api" 0 "$WATCHDOG_STATE_FILE" ((T1_RESTARTS++)) queue_notify "$container API unresponsive at $_api_url on $(hostname) — restarted" "warning" fi fi fi done fi # ========================================================================================== # ── TIER 2 — Global Health Scan ─────────────────────────────────────────────────────────── # ========================================================================================== if [[ "$WATCHDOG_SCAN_ALL" == "true" ]]; then ALL_CONTAINERS=$(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null) # ── Unhealthy containers ────────────────────────────────────────────────────────────── if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \ --filter health=unhealthy --format "{{.Names}}" 2>/dev/null) while IFS= read -r container; do [[ -z "$container" ]] && continue [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue is_skipped "$container" && continue error "$container — Docker HEALTHCHECK unhealthy" ((T2_WARNINGS++)) result=0 safe_restart "$container" "unhealthy health status" || result=$? if [[ $result -eq 0 ]]; then ((T2_RESTARTS++)) queue_notify "$container unhealthy on $(hostname) — restarted" "warning" fi done <<< "$UNHEALTHY" fi # ── OOM killed ──────────────────────────────────────────────────────────────────────── # OOMKilled flag persists after restart — track handled containers per-cycle # to prevent the same container triggering a restart loop every cycle if [[ "$WATCHDOG_NOTIFY_OOM" == "true" ]]; then while IFS= read -r container; do [[ -z "$container" ]] && continue [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue is_skipped "$container" && continue [[ -n "${OOM_HANDLED[$container]:-}" ]] && continue # already handled this cycle OOM=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ '{{.State.OOMKilled}}' "$container" 2>/dev/null) if [[ "$OOM" == "true" ]]; then error "$container — OOM killed by kernel" ((T2_WARNINGS++)) OOM_HANDLED["$container"]=1 result=0 safe_restart "$container" "OOM killed" || result=$? if [[ $result -eq 0 ]]; then ((T2_RESTARTS++)) queue_notify "$container OOM killed on $(hostname) — restarted" "warning" fi fi done <<< "$ALL_CONTAINERS" fi # ── Crash loop detection ────────────────────────────────────────────────────────────── # Tracks Docker's own RestartCount climbing between cycles. # Below WATCHDOG_CRASH_LIMIT: notify only — Docker's restart policy is handling it. # At or above WATCHDOG_CRASH_LIMIT: safe_restart() which will add to skip list # if WATCHDOG_CONTAINER_RESTART_LIMIT is also hit — ensures eventual quarantine. if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then while IFS= read -r container; do [[ -z "$container" ]] && continue [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue is_skipped "$container" && continue RESTART_COUNT=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \ '{{.RestartCount}}' "$container" 2>/dev/null || echo 0) PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" \ 2>/dev/null | cut -d: -f2 || echo 0) set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE" if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then ((T2_WARNINGS++)) if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then error "$container — crash loop CRITICAL: $RESTART_COUNT restarts" # Attempt restart via safe_restart — will add to skip list if over limit result=0 safe_restart "$container" "crash loop — $RESTART_COUNT restarts" || result=$? case $result in 0) ((T2_RESTARTS++)) queue_notify "$container crash loop on $(hostname) — restarted" "critical" ;; 2) : ;; # Added to skip list *) queue_notify "$container crash loop CRITICAL on $(hostname) — manual intervention needed" "critical" ;; esac else warn "$container — restarted since last check (Docker count: $RESTART_COUNT)" queue_notify "$container restarted on $(hostname) — Docker count: $RESTART_COUNT" "warning" fi fi done <<< "$ALL_CONTAINERS" fi # ── Dead containers ─────────────────────────────────────────────────────────────────── # Routes through safe_restart() — ensures restart loop protection applies if [[ "$WATCHDOG_RESTART_DEAD" == "true" ]]; then DEAD=$(timeout "$DOCKER_TIMEOUT" docker ps -a \ --filter status=dead --format "{{.Names}}" 2>/dev/null) while IFS= read -r container; do [[ -z "$container" ]] && continue [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue is_skipped "$container" && continue error "$container — dead" ((T2_WARNINGS++)) if [[ "$DRY_RUN" == false ]]; then timeout "$DOCKER_TIMEOUT" docker rm "$container" >/dev/null 2>&1 fi result=0 safe_restart "$container" "dead container" || result=$? if [[ $result -eq 0 ]]; then ((T2_RESTARTS++)) queue_notify "$container was dead on $(hostname) — removed and restarted" "warning" fi done <<< "$DEAD" fi # ── Unexpected exits ────────────────────────────────────────────────────────────────── # Only non-zero exit codes — exit 0 is a clean stop, not a crash # Skips containers already covered by WATCHDOG_REQUIRED_CONTAINERS (handled in Tier 1) if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then CRASHED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \ --filter status=exited \ --format "{{.Names}}|{{.Status}}" 2>/dev/null | \ grep -v "Exited (0)") while IFS='|' read -r container status; do [[ -z "$container" ]] && continue [[ -n "${IGNORE_MAP[$container]:-}" ]] && continue is_skipped "$container" && continue # Skip containers already monitored by required containers (Tier 1) already_required=false for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]:-}"; do [[ "$container" == "$req" ]] && already_required=true && break done [[ "$already_required" == true ]] && continue error "$container — $status (unexpected exit)" ((T2_WARNINGS++)) result=0 safe_restart "$container" "unexpected exit: $status" || result=$? if [[ $result -eq 0 ]]; then ((T2_RESTARTS++)) queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning" fi done <<< "$CRASHED" fi fi # WATCHDOG_SCAN_ALL # ── Send notifications ──────────────────────────────────────────────────────────────────── flush_notify # ── Cycle summary — quiet when healthy ─────────────────────────────────────────────────── TOTAL_RESTARTS=$(( T1_RESTARTS + T2_RESTARTS )) TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS )) CYCLE_END=$(date +%s) CONTAINER_COUNT=$(docker ps -q 2>/dev/null | wc -l | tr -d ' ') if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings T2: $T2_RESTARTS restarts / $T2_WARNINGS warnings" echo "$ICON_TIME Duration: $(format_duration $(( CYCLE_END - CYCLE_START ))) Containers: $CONTAINER_COUNT" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" else echo "All healthy ✅ — ${CONTAINER_COUNT} containers ($(date '+%H:%M:%S'))" fi