feat: orchestrator housekeeping — watchdog array, script mapping, orch reorder
- master.conf: add WATCHDOG_ORCHESTRATOR_SCRIPTS array; watchdog config moved from standalone section into orch block alongside all other tiers - watchdog_orchestrator.sh: iterate WATCHDOG_ORCHESTRATOR_SCRIPTS instead of hardcoded paths; display name derived from filename; status block updated - master.conf orch section reordered: Array Stop → Array Start → Watchdog (1min) → Critical (15min) → Intermediate (4hr) → Daily (1am) → Weekly (Sunday) - arrs_failed_stalled_recovery.sh added to INTERMEDIATE_MAINTENANCE_SCRIPTS - lidarr_missing_art.sh moved from intermediate to daily - clear_logs.sh added to WEEKLY_MAINTENANCE_SCRIPTS (was documented but orphaned) - playback_aware_lidarr_discovery.sh placed in weekly (commented out, WIP) - master_host1.conf: Slskd added to DAILY_RESTART_CONTAINERS
This commit is contained in:
Gmer4Lfe
@@ -2,13 +2,12 @@
|
||||
# ==============================================================================================
|
||||
# ============================ Watchdog Orchestrator ===========================================
|
||||
# ==============================================================================================
|
||||
# Runs the three-layer watchdog system in the correct sequence each cycle.
|
||||
# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
|
||||
# Schedule: * * * * * (every minute via User Scripts plugin)
|
||||
#
|
||||
# ── EXECUTION ORDER ───────────────────────────────────────────────────────────────────────────
|
||||
# 1. resource_watchdog.sh — reduce system pressure intelligently
|
||||
# 2. docker_watchdog.sh — heal containers with freed resources
|
||||
# 3. system_watchdog.sh — reboot if all else fails (last line of defense)
|
||||
# Driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf — add, remove, or reorder there.
|
||||
# Default: resource_watchdog → docker_watchdog → system_watchdog
|
||||
#
|
||||
# ── WHY ORDER MATTERS ─────────────────────────────────────────────────────────────────────────
|
||||
# Resource Watchdog first — frees RAM and CPU before healing attempts container restarts.
|
||||
@@ -32,13 +31,14 @@
|
||||
# Remove system_watchdog.sh and docker_watchdog.sh from ARRAY_START_SCRIPTS.
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# WATCHDOG_ORCHESTRATOR_SCRIPTS — watchdogs to run, in order
|
||||
# WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate
|
||||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle
|
||||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# watchdog_orchestrator.sh — normal run (called by cron every minute)
|
||||
# watchdog_orchestrator.sh --dry-run — pass --dry-run to all three sub-scripts
|
||||
# watchdog_orchestrator.sh --dry-run — pass --dry-run to all sub-scripts
|
||||
# watchdog_orchestrator.sh --status — show script paths and current grace state
|
||||
# watchdog_orchestrator.sh --log — verbose output from all sub-scripts
|
||||
# ==============================================================================================
|
||||
@@ -63,12 +63,17 @@ acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
RESOURCE_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/resource_watchdog.sh"
|
||||
DOCKER_WATCHDOG="$ECOSYSTEM_ROOT/Docker_Essentials/docker_watchdog.sh"
|
||||
SYSTEM_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/system_watchdog.sh"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
|
||||
|
||||
# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
|
||||
_watchdog_display_name() {
|
||||
local path="$1"
|
||||
local base="${path##*/}"
|
||||
base="${base%.sh}"
|
||||
base="${base//_/ }"
|
||||
echo "$base" | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}'
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
@@ -87,17 +92,14 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
|
||||
echo ""
|
||||
echo "── Sub-scripts ──"
|
||||
for pair in \
|
||||
"Resource Watchdog:$RESOURCE_WATCHDOG" \
|
||||
"Docker Watchdog:$DOCKER_WATCHDOG" \
|
||||
"System Watchdog:$SYSTEM_WATCHDOG"; do
|
||||
label="${pair%%:*}"
|
||||
script="${pair#*:}"
|
||||
if [[ -f "$script" ]]; then
|
||||
[[ -x "$script" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
|
||||
echo " $icon $label — ${script##*/}"
|
||||
for entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||||
local_path="$ECOSYSTEM_ROOT/$entry"
|
||||
label="$(_watchdog_display_name "$entry")"
|
||||
if [[ -f "$local_path" ]]; then
|
||||
[[ -x "$local_path" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
|
||||
echo " $icon $label — ${local_path##*/}"
|
||||
else
|
||||
echo " $ICON_ERROR $label — NOT FOUND: $script"
|
||||
echo " $ICON_ERROR $label — NOT FOUND: $local_path"
|
||||
fi
|
||||
done
|
||||
|
||||
@@ -150,9 +152,9 @@ run_watchdog() {
|
||||
fi
|
||||
}
|
||||
|
||||
run_watchdog "Resource Watchdog" "$RESOURCE_WATCHDOG"
|
||||
run_watchdog "Docker Watchdog" "$DOCKER_WATCHDOG"
|
||||
run_watchdog "System Watchdog" "$SYSTEM_WATCHDOG"
|
||||
for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||||
run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry"
|
||||
done
|
||||
|
||||
CYCLE_END=$(date +%s)
|
||||
DURATION=$(( CYCLE_END - CYCLE_START ))
|
||||
|
||||
+36
-29
@@ -262,6 +262,7 @@
|
||||
# ==============================================================================================
|
||||
# All orchestrator job lists live here — edit arrays to add/remove scripts.
|
||||
# No changes to orchestrator scripts needed when adding or removing jobs.
|
||||
# Sections ordered by run frequency: array events first, then shortest interval to longest.
|
||||
|
||||
# ━━━ Array Stop ━━━
|
||||
# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order.
|
||||
@@ -292,6 +293,35 @@
|
||||
"Fallback/fallback.sh" # mutual failover — continuous
|
||||
)
|
||||
|
||||
# ━━━ Watchdog Orchestrator ━━━
|
||||
# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
|
||||
# Schedule: * * * * * (every minute)
|
||||
# NOT in ARRAY_START_SCRIPTS — has its own cron entry.
|
||||
# Order matters — resource first (frees pressure), docker second (heals with freed resources),
|
||||
# system last (reboots only if prior layers failed).
|
||||
WATCHDOG_ORCHESTRATOR_SCRIPTS=(
|
||||
"unRAID_Essentials/resource_watchdog.sh" # reduce system pressure before healing attempts
|
||||
"Docker_Essentials/docker_watchdog.sh" # heal containers with freed resources
|
||||
"unRAID_Essentials/system_watchdog.sh" # reboot if all else fails — last line of defense
|
||||
)
|
||||
|
||||
WATCHDOG_ORCHESTRATOR_HEARTBEAT=true
|
||||
WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS=1
|
||||
|
||||
# ━━━ Critical Sync Maintenance ━━━
|
||||
# critical_sync_maintenance.sh runs every 15 minutes.
|
||||
# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check
|
||||
# partnership --check always runs last regardless of rsync gate.
|
||||
# Comment out entries to disable without removing.
|
||||
CRITICAL_MAINTENANCE_SCRIPTS=(
|
||||
"Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 15min
|
||||
)
|
||||
|
||||
# Shares synced every 15 minutes — defined per host in master_host*.conf.
|
||||
# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
|
||||
# ━━━ Intermediate Sync Maintenance ━━━
|
||||
# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch,
|
||||
# and optional mid-day rsync for any shares that need sub-daily propagation.
|
||||
@@ -300,7 +330,7 @@
|
||||
INTERMEDIATE_RSYNC_ENABLED=true # set false to disable mid-day rsync without removing shares
|
||||
|
||||
INTERMEDIATE_MAINTENANCE_SCRIPTS=(
|
||||
"Media/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards)
|
||||
"Media/arrs_failed_stalled_recovery.sh" # blocklist + re-search failed/stalled arr queue items
|
||||
)
|
||||
# arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
|
||||
# It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).
|
||||
@@ -317,8 +347,9 @@
|
||||
#"Media/lidarr_cleanup.sh" # remove orphaned music files — enable when ready
|
||||
#"Media/sonarr_cleanup.sh" # remove orphaned TV files — enable when ready
|
||||
#"Media/radarr_cleanup.sh" # remove orphaned movie files — enable when ready
|
||||
"Media/radarr_tmdb_removed.sh" # remove movies dropped from TMDb
|
||||
"Media/sonarr_tvdb_removed.sh" # remove series dropped from TVDB
|
||||
"Media/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards)
|
||||
"Media/radarr_tmdb_removed.sh" # remove movies dropped from TMDb
|
||||
"Media/sonarr_tvdb_removed.sh" # remove series dropped from TVDB
|
||||
"Docker_Essentials/docker_update.sh" # pull container image updates before restart
|
||||
"Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last
|
||||
)
|
||||
@@ -348,6 +379,8 @@
|
||||
WEEKLY_MAINTENANCE_SCRIPTS=(
|
||||
"Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync
|
||||
"Docker_Essentials/docker_update_remaining.sh" # pull updates for all other containers
|
||||
"unRAID_Essentials/clear_logs.sh" # purge aged logs — Sunday only, low priority
|
||||
#"Media/playback_aware_lidarr_discovery.sh" # behavior-driven music discovery using weekly Emby playback history — WIP
|
||||
)
|
||||
|
||||
# Pull updates for all running containers NOT in daily/weekly restart lists.
|
||||
@@ -367,22 +400,6 @@
|
||||
WEEKLY_SYNC_UPDATES=true # pull container updates locally during weekly window
|
||||
WEEKLY_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH
|
||||
|
||||
# ━━━ Critical Sync Maintenance ━━━
|
||||
# critical_sync_maintenance.sh runs every 15 minutes.
|
||||
# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check
|
||||
# partnership --check always runs last regardless of rsync gate.
|
||||
|
||||
# Jobs run every 15 minutes before the rsync shares.
|
||||
# Comment out to disable without removing.
|
||||
CRITICAL_MAINTENANCE_SCRIPTS=(
|
||||
"Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 15min
|
||||
)
|
||||
|
||||
# Shares synced every 15 minutes — defined per host in master_host*.conf.
|
||||
# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
@@ -1115,16 +1132,6 @@
|
||||
EMBY_REPORT_DAYS=7 # days to include in the report period
|
||||
EMBY_REPORT_TOP_N=10 # number of top content items to show
|
||||
|
||||
# ==============================================================================================
|
||||
# ── WATCHDOG ORCHESTRATOR ─────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Runs resource_watchdog → docker_watchdog → system_watchdog in sequence each cron cycle.
|
||||
# Schedule: * * * * * (every minute via User Scripts plugin)
|
||||
# NOT in ARRAY_START_SCRIPTS — has its own cron entry.
|
||||
|
||||
WATCHDOG_ORCHESTRATOR_HEARTBEAT=true
|
||||
WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS=1
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
@@ -252,6 +252,7 @@
|
||||
"Dispatcharr" # Live TV scheduler — degrades without daily restart
|
||||
"Dispatcharr-Basic"
|
||||
"ErsatzTV-Emby"
|
||||
"Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
|
||||
Reference in New Issue
Block a user