diff --git a/Orchestrators/watchdog_orchestrator.sh b/Orchestrators/watchdog_orchestrator.sh index a8b5942..166f93a 100755 --- a/Orchestrators/watchdog_orchestrator.sh +++ b/Orchestrators/watchdog_orchestrator.sh @@ -2,13 +2,12 @@ # ============================================================================================== # ============================ Watchdog Orchestrator =========================================== # ============================================================================================== -# Runs the three-layer watchdog system in the correct sequence each cycle. +# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. # Schedule: * * * * * (every minute via User Scripts plugin) # # ── EXECUTION ORDER ─────────────────────────────────────────────────────────────────────────── -# 1. resource_watchdog.sh — reduce system pressure intelligently -# 2. docker_watchdog.sh — heal containers with freed resources -# 3. system_watchdog.sh — reboot if all else fails (last line of defense) +# Driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf — add, remove, or reorder there. +# Default: resource_watchdog → docker_watchdog → system_watchdog # # ── WHY ORDER MATTERS ───────────────────────────────────────────────────────────────────────── # Resource Watchdog first — frees RAM and CPU before healing attempts container restarts. @@ -32,13 +31,14 @@ # Remove system_watchdog.sh and docker_watchdog.sh from ARRAY_START_SCRIPTS. # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# WATCHDOG_ORCHESTRATOR_SCRIPTS — watchdogs to run, in order # WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate # WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle # WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # watchdog_orchestrator.sh — normal run (called by cron every minute) -# watchdog_orchestrator.sh --dry-run — pass --dry-run to all three sub-scripts +# watchdog_orchestrator.sh --dry-run — pass --dry-run to all sub-scripts # watchdog_orchestrator.sh --status — show script paths and current grace state # watchdog_orchestrator.sh --log — verbose output from all sub-scripts # ============================================================================================== @@ -63,12 +63,17 @@ acquire_lock detect_hosts -RESOURCE_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/resource_watchdog.sh" -DOCKER_WATCHDOG="$ECOSYSTEM_ROOT/Docker_Essentials/docker_watchdog.sh" -SYSTEM_WATCHDOG="$ECOSYSTEM_ROOT/unRAID_Essentials/system_watchdog.sh" - [[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts" +# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog" +_watchdog_display_name() { + local path="$1" + local base="${path##*/}" + base="${base%.sh}" + base="${base//_/ }" + echo "$base" | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}' +} + # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== @@ -87,17 +92,14 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "── Sub-scripts ──" - for pair in \ - "Resource Watchdog:$RESOURCE_WATCHDOG" \ - "Docker Watchdog:$DOCKER_WATCHDOG" \ - "System Watchdog:$SYSTEM_WATCHDOG"; do - label="${pair%%:*}" - script="${pair#*:}" - if [[ -f "$script" ]]; then - [[ -x "$script" ]] && icon="$ICON_DONE" || icon="$ICON_WARN" - echo " $icon $label — ${script##*/}" + for entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do + local_path="$ECOSYSTEM_ROOT/$entry" + label="$(_watchdog_display_name "$entry")" + if [[ -f "$local_path" ]]; then + [[ -x "$local_path" ]] && icon="$ICON_DONE" || icon="$ICON_WARN" + echo " $icon $label — ${local_path##*/}" else - echo " $ICON_ERROR $label — NOT FOUND: $script" + echo " $ICON_ERROR $label — NOT FOUND: $local_path" fi done @@ -150,9 +152,9 @@ run_watchdog() { fi } -run_watchdog "Resource Watchdog" "$RESOURCE_WATCHDOG" -run_watchdog "Docker Watchdog" "$DOCKER_WATCHDOG" -run_watchdog "System Watchdog" "$SYSTEM_WATCHDOG" +for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do + run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry" +done CYCLE_END=$(date +%s) DURATION=$(( CYCLE_END - CYCLE_START )) diff --git a/master.conf b/master.conf index ce9d153..99e0e34 100644 --- a/master.conf +++ b/master.conf @@ -262,6 +262,7 @@ # ============================================================================================== # All orchestrator job lists live here — edit arrays to add/remove scripts. # No changes to orchestrator scripts needed when adding or removing jobs. +# Sections ordered by run frequency: array events first, then shortest interval to longest. # ━━━ Array Stop ━━━ # Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order. @@ -292,6 +293,35 @@ "Fallback/fallback.sh" # mutual failover — continuous ) +# ━━━ Watchdog Orchestrator ━━━ +# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. +# Schedule: * * * * * (every minute) +# NOT in ARRAY_START_SCRIPTS — has its own cron entry. +# Order matters — resource first (frees pressure), docker second (heals with freed resources), +# system last (reboots only if prior layers failed). + WATCHDOG_ORCHESTRATOR_SCRIPTS=( + "unRAID_Essentials/resource_watchdog.sh" # reduce system pressure before healing attempts + "Docker_Essentials/docker_watchdog.sh" # heal containers with freed resources + "unRAID_Essentials/system_watchdog.sh" # reboot if all else fails — last line of defense + ) + + WATCHDOG_ORCHESTRATOR_HEARTBEAT=true + WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS=1 + +# ━━━ Critical Sync Maintenance ━━━ +# critical_sync_maintenance.sh runs every 15 minutes. +# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check +# partnership --check always runs last regardless of rsync gate. +# Comment out entries to disable without removing. + CRITICAL_MAINTENANCE_SCRIPTS=( + "Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 15min + ) + +# Shares synced every 15 minutes — defined per host in master_host*.conf. +# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES +# Format: "/path/to/share" or "/path/to/share|profile-name" +# Order matters — Critical-Data first (auth stack), then Emby dirty sync. + # ━━━ Intermediate Sync Maintenance ━━━ # intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch, # and optional mid-day rsync for any shares that need sub-daily propagation. @@ -300,7 +330,7 @@ INTERMEDIATE_RSYNC_ENABLED=true # set false to disable mid-day rsync without removing shares INTERMEDIATE_MAINTENANCE_SCRIPTS=( - "Media/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards) + "Media/arrs_failed_stalled_recovery.sh" # blocklist + re-search failed/stalled arr queue items ) # arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here. # It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above). @@ -317,8 +347,9 @@ #"Media/lidarr_cleanup.sh" # remove orphaned music files — enable when ready #"Media/sonarr_cleanup.sh" # remove orphaned TV files — enable when ready #"Media/radarr_cleanup.sh" # remove orphaned movie files — enable when ready - "Media/radarr_tmdb_removed.sh" # remove movies dropped from TMDb - "Media/sonarr_tvdb_removed.sh" # remove series dropped from TVDB + "Media/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards) + "Media/radarr_tmdb_removed.sh" # remove movies dropped from TMDb + "Media/sonarr_tvdb_removed.sh" # remove series dropped from TVDB "Docker_Essentials/docker_update.sh" # pull container image updates before restart "Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last ) @@ -348,6 +379,8 @@ WEEKLY_MAINTENANCE_SCRIPTS=( "Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync "Docker_Essentials/docker_update_remaining.sh" # pull updates for all other containers + "unRAID_Essentials/clear_logs.sh" # purge aged logs — Sunday only, low priority + #"Media/playback_aware_lidarr_discovery.sh" # behavior-driven music discovery using weekly Emby playback history — WIP ) # Pull updates for all running containers NOT in daily/weekly restart lists. @@ -367,22 +400,6 @@ WEEKLY_SYNC_UPDATES=true # pull container updates locally during weekly window WEEKLY_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH -# ━━━ Critical Sync Maintenance ━━━ -# critical_sync_maintenance.sh runs every 15 minutes. -# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check -# partnership --check always runs last regardless of rsync gate. - -# Jobs run every 15 minutes before the rsync shares. -# Comment out to disable without removing. - CRITICAL_MAINTENANCE_SCRIPTS=( - "Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 15min - ) - -# Shares synced every 15 minutes — defined per host in master_host*.conf. -# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES -# Format: "/path/to/share" or "/path/to/share|profile-name" -# Order matters — Critical-Data first (auth stack), then Emby dirty sync. - # ============================================================================================== # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== @@ -1115,16 +1132,6 @@ EMBY_REPORT_DAYS=7 # days to include in the report period EMBY_REPORT_TOP_N=10 # number of top content items to show -# ============================================================================================== -# ── WATCHDOG ORCHESTRATOR ───────────────────────────────────────────────────────────────────── -# ============================================================================================== -# Runs resource_watchdog → docker_watchdog → system_watchdog in sequence each cron cycle. -# Schedule: * * * * * (every minute via User Scripts plugin) -# NOT in ARRAY_START_SCRIPTS — has its own cron entry. - - WATCHDOG_ORCHESTRATOR_HEARTBEAT=true - WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS=1 - # ============================================================================================== # ── RESOURCE MANAGER ────────────────────────────────────────────────────────────────────────── # ============================================================================================== diff --git a/master_host1.conf b/master_host1.conf index f911926..1114d5b 100644 --- a/master_host1.conf +++ b/master_host1.conf @@ -252,6 +252,7 @@ "Dispatcharr" # Live TV scheduler — degrades without daily restart "Dispatcharr-Basic" "ErsatzTV-Emby" + "Slskd" # Soulseek connection drops after extended uptime; restart refreshes share index ) # ━━━ Docker Weekly Restart ━━━