Gmer4LfeandClaude Sonnet 4.6
309546e615
Introduce a four-layer self-healing stack replacing the continuous-loop watchdogs: - resource_manager.sh (new): single-pass pressure reduction layer; throttles SABnzbd/qBit at level 1, docker-pauses background containers at level 2, docker-stops optional containers and signals docker_watchdog to defer at level 3; graduated recovery with hysteresis - watchdog_orchestrator.sh (new, Orchestrators/): runs resource_manager → docker_watchdog → system_watchdog in sequence; intended for per-minute cron via User Scripts; startup grace, acquire_lock to prevent pile-up, heartbeat - docker_watchdog.sh: de-looped to single-pass; daemon strikes persisted to state file across runs; cross-script coordination reads RM_STATE_FILE instead of SYS_WATCHDOG_STATE_FILE - system_watchdog.sh: de-looped to single-pass; stripped of all container management (shutdown_non_essential_containers removed); reboot-only last resort - master.conf: removed system_watchdog and docker_watchdog from ARRAY_START_SCRIPTS; added WATCHDOG ORCHESTRATOR and RESOURCE MANAGER sections - master_host1.conf: added RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS arrays - common.sh: aliased RM_PAUSE_CONTAINERS and RM_STOP_CONTAINERS via detect_hosts() - continuous_scripts_status.sh: moved to Tools/ (preserved for future use) - sunday_morning_coffee_report.sh: watchdog section updated to use state file mtime checks instead of is_running; added Resource Manager subsection; fixed mem_shutdown grep filter pointing to wrong state file Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
193 lines
9.8 KiB
Bash
Executable File
193 lines
9.8 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ============================ Watchdog Orchestrator ===========================================
|
||
# ==============================================================================================
|
||
# Runs the three-layer watchdog system in the correct sequence each cycle.
|
||
# Schedule: * * * * * (every minute via User Scripts plugin)
|
||
#
|
||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||
# 1. resource_manager.sh — reduce system pressure intelligently
|
||
# 2. docker_watchdog.sh — heal containers with freed resources
|
||
# 3. system_watchdog.sh — reboot if all else fails (last line of defense)
|
||
#
|
||
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
|
||
# Resource Manager first — frees RAM and CPU before healing attempts container restarts.
|
||
# Containers restarted into a resource-pressured system just fail again.
|
||
# Docker Watchdog second — restarts with pressure already reduced, more likely to stabilise.
|
||
# System Watchdog last — only triggers if prior layers could not resolve the issue.
|
||
# Rebooting without first reducing pressure may reboot into the same state.
|
||
#
|
||
# ── STARTUP GRACE ─────────────────────────────────────────────────────────────────────────────
|
||
# No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds.
|
||
# Prevents false positives from containers still starting at array launch.
|
||
# Each sub-script enforces this independently — orchestrator exits early to avoid log noise.
|
||
#
|
||
# ── OVERLAP PROTECTION ────────────────────────────────────────────────────────────────────────
|
||
# acquire_lock() — exits immediately if a prior cycle is still in progress.
|
||
# Prevents pile-up when a cycle runs long (daemon restart attempt = 30s, etc.).
|
||
#
|
||
# ── REPLACES ──────────────────────────────────────────────────────────────────────────────────
|
||
# Continuous loops previously in system_watchdog.sh and docker_watchdog.sh.
|
||
# Those scripts are now single-pass — this orchestrator provides the cadence.
|
||
# Remove system_watchdog.sh and docker_watchdog.sh from ARRAY_START_SCRIPTS.
|
||
#
|
||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||
# WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate
|
||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle
|
||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
|
||
#
|
||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||
# watchdog_orchestrator.sh — normal run (called by cron every minute)
|
||
# watchdog_orchestrator.sh --dry-run — pass --dry-run to all three sub-scripts
|
||
# watchdog_orchestrator.sh --status — show script paths and current grace state
|
||
# watchdog_orchestrator.sh --log — verbose output from all sub-scripts
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||
|
||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
# Skip immediately if another cycle is still running — no pile-up
|
||
acquire_lock
|
||
|
||
detect_hosts
|
||
|
||
RESOURCE_MANAGER="$ECOSYSTEM_ROOT/unRAID_Essentials/resource_manager.sh"
|
||
DOCKER_WATCHDOG="$ECOSYSTEM_ROOT/Docker_Essentials/docker_watchdog.sh"
|
||
SYSTEM_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/system_watchdog.sh"
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY WATCHDOG ORCHESTRATOR STATUS ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo ""
|
||
|
||
UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
|
||
if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||
warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
|
||
else
|
||
log "Past startup grace — $(format_duration $UPTIME_S) uptime"
|
||
fi
|
||
|
||
echo ""
|
||
echo "── Sub-scripts ──"
|
||
for pair in \
|
||
"Resource Manager:$RESOURCE_MANAGER" \
|
||
"Docker Watchdog:$DOCKER_WATCHDOG" \
|
||
"System Watchdog:$SYSTEM_WATCHDOG"; do
|
||
label="${pair%%:*}"
|
||
script="${pair#*:}"
|
||
if [[ -f "$script" ]]; then
|
||
[[ -x "$script" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
|
||
echo " $icon $label — ${script##*/}"
|
||
else
|
||
echo " $ICON_ERROR $label — NOT FOUND: $script"
|
||
fi
|
||
done
|
||
|
||
echo ""
|
||
echo " Schedule: * * * * * (every minute via User Scripts)"
|
||
echo " Heartbeat: ${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true} / every ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Startup Grace ━━━
|
||
# ==============================================================================================
|
||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||
log "Startup grace — ${UPTIME_SECONDS}s / ${WATCHDOG_STARTUP_GRACE}s — skipping cycle"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Run Watchdog Cycle ━━━
|
||
# ==============================================================================================
|
||
CYCLE_START=$(date +%s)
|
||
PASS=()
|
||
FAIL=()
|
||
|
||
run_watchdog() {
|
||
local name="$1" script="$2"
|
||
|
||
if [[ ! -f "$script" ]]; then
|
||
error "$name — not found: $script"
|
||
FAIL+=("$name:missing")
|
||
return 1
|
||
fi
|
||
|
||
[[ ! -x "$script" ]] && chmod +x "$script"
|
||
|
||
local extra_args=()
|
||
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
||
[[ "$VERBOSE" == true ]] && extra_args+=("--log")
|
||
|
||
log "$ICON_START $name"
|
||
if bash "$script" "${extra_args[@]}"; then
|
||
PASS+=("$name")
|
||
return 0
|
||
else
|
||
error "$name — non-zero exit"
|
||
FAIL+=("$name")
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
run_watchdog "Resource Manager" "$RESOURCE_MANAGER"
|
||
run_watchdog "Docker Watchdog" "$DOCKER_WATCHDOG"
|
||
run_watchdog "System Watchdog" "$SYSTEM_WATCHDOG"
|
||
|
||
CYCLE_END=$(date +%s)
|
||
DURATION=$(( CYCLE_END - CYCLE_START ))
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Heartbeat ━━━
|
||
# ==============================================================================================
|
||
if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
|
||
HB_SECONDS=$(( ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1} * 3600 ))
|
||
HB_COUNT_FILE="/tmp/watchdog_orch_hb.count"
|
||
HB_COUNT=$(cat "$HB_COUNT_FILE" 2>/dev/null || echo 0)
|
||
HB_COUNT=$(( HB_COUNT + 1 ))
|
||
echo "$HB_COUNT" > "$HB_COUNT_FILE"
|
||
# Each cron run = ~60s — use count × 60 as uptime approximation
|
||
HB_ELAPSED=$(( HB_COUNT * 60 ))
|
||
if [[ "$HB_SECONDS" -gt 0 ]] && (( HB_ELAPSED % HB_SECONDS < 60 )) && [[ "$HB_COUNT" -gt 1 ]]; then
|
||
HB_HR=$(( HB_ELAPSED / 3600 ))
|
||
warn "♥ watchdog_orchestrator alive — $MY_ID — ~${HB_HR}hr ($(date '+%H:%M:%S'))"
|
||
fi
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Summary — only shown on failures or --log ━━━
|
||
# ==============================================================================================
|
||
if [[ "${#FAIL[@]}" -gt 0 || "$VERBOSE" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||
for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done
|
||
for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
|
||
if [[ "${#FAIL[@]}" -gt 0 ]]; then
|
||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||
"Watchdog Orchestrator" "warning"
|
||
fi
|
||
fi
|