feat: orchestrator housekeeping — watchdog array, script mapping, orch reorder

- master.conf: add WATCHDOG_ORCHESTRATOR_SCRIPTS array; watchdog config moved
  from standalone section into orch block alongside all other tiers
- watchdog_orchestrator.sh: iterate WATCHDOG_ORCHESTRATOR_SCRIPTS instead of
  hardcoded paths; display name derived from filename; status block updated
- master.conf orch section reordered: Array Stop → Array Start → Watchdog (1min)
  → Critical (15min) → Intermediate (4hr) → Daily (1am) → Weekly (Sunday)
- arrs_failed_stalled_recovery.sh added to INTERMEDIATE_MAINTENANCE_SCRIPTS
- lidarr_missing_art.sh moved from intermediate to daily
- clear_logs.sh added to WEEKLY_MAINTENANCE_SCRIPTS (was documented but orphaned)
- playback_aware_lidarr_discovery.sh placed in weekly (commented out, WIP)
- master_host1.conf: Slskd added to DAILY_RESTART_CONTAINERS
This commit is contained in:
Gmer4Lfe
2026-05-19 21:01:35 -04:00
parent e13f2fa14f
commit b4d2a90568
3 changed files with 61 additions and 51 deletions
+24 -22
View File
@@ -2,13 +2,12 @@
# ==============================================================================================
# ============================ Watchdog Orchestrator ===========================================
# ==============================================================================================
# Runs the three-layer watchdog system in the correct sequence each cycle.
# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
# Schedule: * * * * * (every minute via User Scripts plugin)
#
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
# 1. resource_watchdog.sh — reduce system pressure intelligently
# 2. docker_watchdog.sh — heal containers with freed resources
# 3. system_watchdog.sh — reboot if all else fails (last line of defense)
# Driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf — add, remove, or reorder there.
# Default: resource_watchdog → docker_watchdog → system_watchdog
#
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
# Resource Watchdog first — frees RAM and CPU before healing attempts container restarts.
@@ -32,13 +31,14 @@
# Remove system_watchdog.sh and docker_watchdog.sh from ARRAY_START_SCRIPTS.
#
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
# WATCHDOG_ORCHESTRATOR_SCRIPTS — watchdogs to run, in order
# WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate
# WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle
# WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
#
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
# watchdog_orchestrator.sh — normal run (called by cron every minute)
# watchdog_orchestrator.sh --dry-run — pass --dry-run to all three sub-scripts
# watchdog_orchestrator.sh --dry-run — pass --dry-run to all sub-scripts
# watchdog_orchestrator.sh --status — show script paths and current grace state
# watchdog_orchestrator.sh --log — verbose output from all sub-scripts
# ==============================================================================================
@@ -63,12 +63,17 @@ acquire_lock
detect_hosts
RESOURCE_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/resource_watchdog.sh"
DOCKER_WATCHDOG="$ECOSYSTEM_ROOT/Docker_Essentials/docker_watchdog.sh"
SYSTEM_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/system_watchdog.sh"
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
_watchdog_display_name() {
local path="$1"
local base="${path##*/}"
base="${base%.sh}"
base="${base//_/ }"
echo "$base" | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}'
}
# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
@@ -87,17 +92,14 @@ if [[ "$SHOW_STATUS" == true ]]; then
echo ""
echo "── Sub-scripts ──"
for pair in \
"Resource Watchdog:$RESOURCE_WATCHDOG" \
"Docker Watchdog:$DOCKER_WATCHDOG" \
"System Watchdog:$SYSTEM_WATCHDOG"; do
label="${pair%%:*}"
script="${pair#*:}"
if [[ -f "$script" ]]; then
[[ -x "$script" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
echo " $icon $label${script##*/}"
for entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
local_path="$ECOSYSTEM_ROOT/$entry"
label="$(_watchdog_display_name "$entry")"
if [[ -f "$local_path" ]]; then
[[ -x "$local_path" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
echo " $icon $label${local_path##*/}"
else
echo " $ICON_ERROR $label — NOT FOUND: $script"
echo " $ICON_ERROR $label — NOT FOUND: $local_path"
fi
done
@@ -150,9 +152,9 @@ run_watchdog() {
fi
}
run_watchdog "Resource Watchdog" "$RESOURCE_WATCHDOG"
run_watchdog "Docker Watchdog" "$DOCKER_WATCHDOG"
run_watchdog "System Watchdog" "$SYSTEM_WATCHDOG"
for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry"
done
CYCLE_END=$(date +%s)
DURATION=$(( CYCLE_END - CYCLE_START ))