feat: orchestrator housekeeping — watchdog array, script mapping, orch reorder
- master.conf: add WATCHDOG_ORCHESTRATOR_SCRIPTS array; watchdog config moved from standalone section into orch block alongside all other tiers - watchdog_orchestrator.sh: iterate WATCHDOG_ORCHESTRATOR_SCRIPTS instead of hardcoded paths; display name derived from filename; status block updated - master.conf orch section reordered: Array Stop → Array Start → Watchdog (1min) → Critical (15min) → Intermediate (4hr) → Daily (1am) → Weekly (Sunday) - arrs_failed_stalled_recovery.sh added to INTERMEDIATE_MAINTENANCE_SCRIPTS - lidarr_missing_art.sh moved from intermediate to daily - clear_logs.sh added to WEEKLY_MAINTENANCE_SCRIPTS (was documented but orphaned) - playback_aware_lidarr_discovery.sh placed in weekly (commented out, WIP) - master_host1.conf: Slskd added to DAILY_RESTART_CONTAINERS
This commit is contained in:
Gmer4Lfe
+36
-29
@@ -262,6 +262,7 @@
|
||||
# ==============================================================================================
|
||||
# All orchestrator job lists live here — edit arrays to add/remove scripts.
|
||||
# No changes to orchestrator scripts needed when adding or removing jobs.
|
||||
# Sections ordered by run frequency: array events first, then shortest interval to longest.
|
||||
|
||||
# ━━━ Array Stop ━━━
|
||||
# Scripts run by array_stopping.sh for a planned shutdown — stops everything cleanly in order.
|
||||
@@ -292,6 +293,35 @@
|
||||
"Fallback/fallback.sh" # mutual failover — continuous
|
||||
)
|
||||
|
||||
# ━━━ Watchdog Orchestrator ━━━
|
||||
# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
|
||||
# Schedule: * * * * * (every minute)
|
||||
# NOT in ARRAY_START_SCRIPTS — has its own cron entry.
|
||||
# Order matters — resource first (frees pressure), docker second (heals with freed resources),
|
||||
# system last (reboots only if prior layers failed).
|
||||
WATCHDOG_ORCHESTRATOR_SCRIPTS=(
|
||||
"unRAID_Essentials/resource_watchdog.sh" # reduce system pressure before healing attempts
|
||||
"Docker_Essentials/docker_watchdog.sh" # heal containers with freed resources
|
||||
"unRAID_Essentials/system_watchdog.sh" # reboot if all else fails — last line of defense
|
||||
)
|
||||
|
||||
WATCHDOG_ORCHESTRATOR_HEARTBEAT=true
|
||||
WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS=1
|
||||
|
||||
# ━━━ Critical Sync Maintenance ━━━
|
||||
# critical_sync_maintenance.sh runs every 15 minutes.
|
||||
# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check
|
||||
# partnership --check always runs last regardless of rsync gate.
|
||||
# Comment out entries to disable without removing.
|
||||
CRITICAL_MAINTENANCE_SCRIPTS=(
|
||||
"Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 15min
|
||||
)
|
||||
|
||||
# Shares synced every 15 minutes — defined per host in master_host*.conf.
|
||||
# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
|
||||
# ━━━ Intermediate Sync Maintenance ━━━
|
||||
# intermediate_sync_maintenance.sh runs every 4 hours — arr library sync, artwork fetch,
|
||||
# and optional mid-day rsync for any shares that need sub-daily propagation.
|
||||
@@ -300,7 +330,7 @@
|
||||
INTERMEDIATE_RSYNC_ENABLED=true # set false to disable mid-day rsync without removing shares
|
||||
|
||||
INTERMEDIATE_MAINTENANCE_SCRIPTS=(
|
||||
"Media/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards)
|
||||
"Media/arrs_failed_stalled_recovery.sh" # blocklist + re-search failed/stalled arr queue items
|
||||
)
|
||||
# arr_sync.sh runs as a fixed first step in intermediate_sync_maintenance.sh — not listed here.
|
||||
# It is controlled by ARR_SYNC_ENABLED (see Arr Sync section above).
|
||||
@@ -317,8 +347,9 @@
|
||||
#"Media/lidarr_cleanup.sh" # remove orphaned music files — enable when ready
|
||||
#"Media/sonarr_cleanup.sh" # remove orphaned TV files — enable when ready
|
||||
#"Media/radarr_cleanup.sh" # remove orphaned movie files — enable when ready
|
||||
"Media/radarr_tmdb_removed.sh" # remove movies dropped from TMDb
|
||||
"Media/sonarr_tvdb_removed.sh" # remove series dropped from TVDB
|
||||
"Media/lidarr_missing_art.sh" # fetch missing album/artist artwork (HOST1 only — self-guards)
|
||||
"Media/radarr_tmdb_removed.sh" # remove movies dropped from TMDb
|
||||
"Media/sonarr_tvdb_removed.sh" # remove series dropped from TVDB
|
||||
"Docker_Essentials/docker_update.sh" # pull container image updates before restart
|
||||
"Docker_Essentials/docker_daily_restart.sh" # daily container restarts — runs last
|
||||
)
|
||||
@@ -348,6 +379,8 @@
|
||||
WEEKLY_MAINTENANCE_SCRIPTS=(
|
||||
"Docker_Essentials/docker_weekly_restart.sh" # weekly container restarts after sync
|
||||
"Docker_Essentials/docker_update_remaining.sh" # pull updates for all other containers
|
||||
"unRAID_Essentials/clear_logs.sh" # purge aged logs — Sunday only, low priority
|
||||
#"Media/playback_aware_lidarr_discovery.sh" # behavior-driven music discovery using weekly Emby playback history — WIP
|
||||
)
|
||||
|
||||
# Pull updates for all running containers NOT in daily/weekly restart lists.
|
||||
@@ -367,22 +400,6 @@
|
||||
WEEKLY_SYNC_UPDATES=true # pull container updates locally during weekly window
|
||||
WEEKLY_SYNC_UPDATES_REMOTE=true # pull container updates on remote via SSH
|
||||
|
||||
# ━━━ Critical Sync Maintenance ━━━
|
||||
# critical_sync_maintenance.sh runs every 15 minutes.
|
||||
# Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check
|
||||
# partnership --check always runs last regardless of rsync gate.
|
||||
|
||||
# Jobs run every 15 minutes before the rsync shares.
|
||||
# Comment out to disable without removing.
|
||||
CRITICAL_MAINTENANCE_SCRIPTS=(
|
||||
"Docker_Essentials/downloaders_reset.sh" # clear stuck download states every 15min
|
||||
)
|
||||
|
||||
# Shares synced every 15 minutes — defined per host in master_host*.conf.
|
||||
# HOST1_CRITICAL_SYNC_SHARES / HOST2_CRITICAL_SYNC_SHARES
|
||||
# Format: "/path/to/share" or "/path/to/share|profile-name"
|
||||
# Order matters — Critical-Data first (auth stack), then Emby dirty sync.
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
@@ -1115,16 +1132,6 @@
|
||||
EMBY_REPORT_DAYS=7 # days to include in the report period
|
||||
EMBY_REPORT_TOP_N=10 # number of top content items to show
|
||||
|
||||
# ==============================================================================================
|
||||
# ── WATCHDOG ORCHESTRATOR ─────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Runs resource_watchdog → docker_watchdog → system_watchdog in sequence each cron cycle.
|
||||
# Schedule: * * * * * (every minute via User Scripts plugin)
|
||||
# NOT in ARRAY_START_SCRIPTS — has its own cron entry.
|
||||
|
||||
WATCHDOG_ORCHESTRATOR_HEARTBEAT=true
|
||||
WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS=1
|
||||
|
||||
# ==============================================================================================
|
||||
# ── RESOURCE MANAGER ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
Reference in New Issue
Block a user