feat: orchestrator housekeeping — watchdog array, script mapping, orch reorder
- master.conf: add WATCHDOG_ORCHESTRATOR_SCRIPTS array; watchdog config moved from standalone section into orch block alongside all other tiers - watchdog_orchestrator.sh: iterate WATCHDOG_ORCHESTRATOR_SCRIPTS instead of hardcoded paths; display name derived from filename; status block updated - master.conf orch section reordered: Array Stop → Array Start → Watchdog (1min) → Critical (15min) → Intermediate (4hr) → Daily (1am) → Weekly (Sunday) - arrs_failed_stalled_recovery.sh added to INTERMEDIATE_MAINTENANCE_SCRIPTS - lidarr_missing_art.sh moved from intermediate to daily - clear_logs.sh added to WEEKLY_MAINTENANCE_SCRIPTS (was documented but orphaned) - playback_aware_lidarr_discovery.sh placed in weekly (commented out, WIP) - master_host1.conf: Slskd added to DAILY_RESTART_CONTAINERS
This commit is contained in:
@@ -2,13 +2,12 @@
|
||||
# ==============================================================================================
|
||||
# ============================ Watchdog Orchestrator ===========================================
|
||||
# ==============================================================================================
|
||||
# Runs the three-layer watchdog system in the correct sequence each cycle.
|
||||
# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
|
||||
# Schedule: * * * * * (every minute via User Scripts plugin)
|
||||
#
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. resource_watchdog.sh — reduce system pressure intelligently
|
||||
# 2. docker_watchdog.sh — heal containers with freed resources
|
||||
# 3. system_watchdog.sh — reboot if all else fails (last line of defense)
|
||||
# Driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf — add, remove, or reorder there.
|
||||
# Default: resource_watchdog → docker_watchdog → system_watchdog
|
||||
#
|
||||
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
|
||||
# Resource Watchdog first — frees RAM and CPU before healing attempts container restarts.
|
||||
@@ -32,13 +31,14 @@
|
||||
# Remove system_watchdog.sh and docker_watchdog.sh from ARRAY_START_SCRIPTS.
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# WATCHDOG_ORCHESTRATOR_SCRIPTS — watchdogs to run, in order
|
||||
# WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate
|
||||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle
|
||||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# watchdog_orchestrator.sh — normal run (called by cron every minute)
|
||||
# watchdog_orchestrator.sh --dry-run — pass --dry-run to all three sub-scripts
|
||||
# watchdog_orchestrator.sh --dry-run — pass --dry-run to all sub-scripts
|
||||
# watchdog_orchestrator.sh --status — show script paths and current grace state
|
||||
# watchdog_orchestrator.sh --log — verbose output from all sub-scripts
|
||||
# ==============================================================================================
|
||||
@@ -63,12 +63,17 @@ acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
RESOURCE_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/resource_watchdog.sh"
|
||||
DOCKER_WATCHDOG="$ECOSYSTEM_ROOT/Docker_Essentials/docker_watchdog.sh"
|
||||
SYSTEM_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/system_watchdog.sh"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
|
||||
|
||||
# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
|
||||
_watchdog_display_name() {
|
||||
local path="$1"
|
||||
local base="${path##*/}"
|
||||
base="${base%.sh}"
|
||||
base="${base//_/ }"
|
||||
echo "$base" | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}'
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
@@ -87,17 +92,14 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
|
||||
echo ""
|
||||
echo "── Sub-scripts ──"
|
||||
for pair in \
|
||||
"Resource Watchdog:$RESOURCE_WATCHDOG" \
|
||||
"Docker Watchdog:$DOCKER_WATCHDOG" \
|
||||
"System Watchdog:$SYSTEM_WATCHDOG"; do
|
||||
label="${pair%%:*}"
|
||||
script="${pair#*:}"
|
||||
if [[ -f "$script" ]]; then
|
||||
[[ -x "$script" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
|
||||
echo " $icon $label — ${script##*/}"
|
||||
for entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||||
local_path="$ECOSYSTEM_ROOT/$entry"
|
||||
label="$(_watchdog_display_name "$entry")"
|
||||
if [[ -f "$local_path" ]]; then
|
||||
[[ -x "$local_path" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
|
||||
echo " $icon $label — ${local_path##*/}"
|
||||
else
|
||||
echo " $ICON_ERROR $label — NOT FOUND: $script"
|
||||
echo " $ICON_ERROR $label — NOT FOUND: $local_path"
|
||||
fi
|
||||
done
|
||||
|
||||
@@ -150,9 +152,9 @@ run_watchdog() {
|
||||
fi
|
||||
}
|
||||
|
||||
run_watchdog "Resource Watchdog" "$RESOURCE_WATCHDOG"
|
||||
run_watchdog "Docker Watchdog" "$DOCKER_WATCHDOG"
|
||||
run_watchdog "System Watchdog" "$SYSTEM_WATCHDOG"
|
||||
for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||||
run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry"
|
||||
done
|
||||
|
||||
CYCLE_END=$(date +%s)
|
||||
DURATION=$(( CYCLE_END - CYCLE_START ))
|
||||
|
||||
Reference in New Issue
Block a user