From 6fdc27b00ba7923adcb42d2bc1e948b2f56f8bde Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Mon, 18 May 2026 21:41:59 -0400 Subject: [PATCH] docs: update user_script_plug-in.sh to v2.4 Add array_stopping.sh (At Stopping of Array) and watchdog_orchestrator.sh (every minute) as new scheduled orchestrators. Add docker_container_stop.sh and resource_watchdog.sh individual script entries. Update system_watchdog and docker_watchdog descriptions to reflect single-pass architecture. Fix paths: git_pull_execute.sh (root-level), continuous_scripts_status.sh (Tools/). Co-Authored-By: Claude Sonnet 4.6 --- user_script_plug-in.sh | 112 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 9 deletions(-) diff --git a/user_script_plug-in.sh b/user_script_plug-in.sh index 8f17496..e9620b2 100644 --- a/user_script_plug-in.sh +++ b/user_script_plug-in.sh @@ -15,6 +15,8 @@ # You do not need to schedule every script below. The orchestrators cover it all: # # array_started.sh at array start — launches ALL startup scripts in order +# array_stopping.sh at array stop — graceful shutdown sequence +# watchdog_orchestrator.sh every minute — resource + docker + system watchdog # transcode_management.sh every 3 min — cleanup then manager (order critical) # critical_sync_maintenance.sh every 15 min — auth + Emby dirty sync + partnership # intermediate_sync_maintenance.sh every 4 hours — arr library sync + artwork fetch @@ -80,6 +82,13 @@ # Initial_run/ section: ssh_setup.sh + partnership_onboard.sh. # Moved: arrs_failed_stalled_recovery → Media/, fallback_state_reset + # watchdog_skip_list_manager → Tools/. Renamed: partnership_manager.sh. +# v2.4 — array_stopping.sh: At Stopping of Array orchestrator (graceful shutdown sequence). +# watchdog_orchestrator.sh: every-minute cron replacing continuous loops in +# system_watchdog + docker_watchdog (both now single-pass, no longer started by +# array_started.sh — remove from ARRAY_START_SCRIPTS in master.conf). +# resource_watchdog.sh: three-level pressure reduction (soft/medium/hard). +# docker_container_stop.sh: sequential verified container stop for shutdown. +# Path fixes: git_pull_execute.sh (root), continuous_scripts_status.sh (Tools/). # ============================================================================================== @@ -108,13 +117,59 @@ # php_fpm_max_children.sh set PHP-FPM workers to 250 BEFORE WebGUI serves requests # ramdisk_setup.sh create tmpfs ramdisk + transcode symlink BEFORE Emby starts # docker_network_connect.sh ensure custom networks exist and containers are connected -# system_watchdog.sh [continuous] three-tier server stability watchdog -# docker_watchdog.sh [continuous] two-tier container self-healing watchdog # fallback.sh [continuous] mutual fallback state machine # +# NOTE: system_watchdog.sh and docker_watchdog.sh are no longer started by array_started.sh. +# They are now single-pass scripts called every minute by watchdog_orchestrator.sh (see below). +# Remove them from ARRAY_START_SCRIPTS in master.conf if present. +# # bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_started.sh +# ── ARRAY STOPPING ──────────────────────────────────────────────────────────────────────────── +# Schedule: At Stopping of Array +# Background: YES +# +# The ONLY "At Stopping of Array" entry needed. Reads ARRAY_STOP_SCRIPTS from master.conf +# and runs each sequentially — each must complete before the next starts. +# +# Runs in order: +# user_scripts_stop.sh stop background user scripts first (prevents new operations) +# rsync_stop.sh --rsync-only kill rsync without triggering container recovery +# mover_stop.sh stop mover after rsync (both write to same paths) +# docker_container_stop.sh stop all containers one-by-one with verification +# +# A failed step is logged and notified but does not prevent remaining steps from running. +# Unlike array_started.sh, all steps run in the foreground — each confirmed done before next. +# +# bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_stopping.sh +# bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_stopping.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Orchestrators/array_stopping.sh --status + + +# ── WATCHDOG ORCHESTRATOR ───────────────────────────────────────────────────────────────────── +# Schedule: * * * * * (every minute) +# Background: YES +# +# Runs the three-layer watchdog system in the correct sequence each cycle. Replaces the +# continuous loops that used to run inside system_watchdog.sh and docker_watchdog.sh — +# those scripts are now single-pass and called here. +# +# Execution order (non-negotiable): +# 1. resource_watchdog.sh free RAM and reduce CPU pressure BEFORE healing attempts +# 2. docker_watchdog.sh restart containers with pressure already reduced +# 3. system_watchdog.sh reboot if prior layers could not resolve the issue +# +# Startup grace: no action until uptime >= WATCHDOG_STARTUP_GRACE (600s). Exits early +# so all three sub-scripts don't each log startup-grace skips on every minute. +# Overlap protection: acquire_lock() exits immediately if a prior cycle is still running — +# prevents pile-up when a restart attempt or daemon check takes longer than 60 seconds. +# +# bash /mnt/user/appdata/unraid_scripts/Orchestrators/watchdog_orchestrator.sh +# bash /mnt/user/appdata/unraid_scripts/Orchestrators/watchdog_orchestrator.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Orchestrators/watchdog_orchestrator.sh --status + + # ── TRANSCODE MANAGEMENT ────────────────────────────────────────────────────────────────────── # Schedule: */3 * * * * (every 3 minutes) # Background: YES @@ -509,8 +564,8 @@ # DOCKER ESSENTIALS # ────────────────────────────────────────────────────────────────────────────────────────────── -# docker_watchdog.sh — [continuous] two-tier container self-healing monitor -# Started by array_started.sh. Every DOCKER_WATCHDOG_INTERVAL (900s = 15min). +# docker_watchdog.sh — two-tier container self-healing monitor (single-pass) +# Called every minute by watchdog_orchestrator.sh — NOT started by array_started.sh. Single-pass. # # Tier 1 — explicit per-container (configured in master_host*.conf): # Memory hard limits: immediate restart when exceeded — no strikes, no waiting @@ -553,6 +608,21 @@ # bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --clear-all # bash /mnt/user/appdata/unraid_scripts/Tools/watchdog_skip_list_manager.sh --dry-run +# docker_container_stop.sh — stop all running containers sequentially with verification +# Called by array_stopping.sh as the final step in the planned shutdown sequence. +# Safe to run standalone before array maintenance, disk replacement, or planned reboots. +# Never moves to the next container until the current one is confirmed stopped. +# +# Stop sequence per container: +# docker stop -t 30 (SIGTERM + 30s grace — docker sends SIGKILL if needed) +# Retry up to RETRY_COUNT times if still running +# docker kill (SIGKILL) if all retries exhausted +# Notify on any container that cannot be stopped +# +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_container_stop.sh --status +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_container_stop.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/Docker_Essentials/docker_container_stop.sh + # docker_daily_restart.sh — restart degradation-prone containers nightly # Called by daily_sync_maintenance.sh. Also safe to run ad hoc. # Running → docker restart (graceful). Stopped → left stopped (state respected). Missing → skip. @@ -636,8 +706,8 @@ # UNRAID ESSENTIALS # ────────────────────────────────────────────────────────────────────────────────────────────── -# system_watchdog.sh — [continuous] three-tier server stability last-resort watchdog -# Started by array_started.sh. Every SYSTEM_WATCHDOG_INTERVAL (300s = 5min). +# system_watchdog.sh — three-tier server stability last-resort watchdog (single-pass) +# Called every minute by watchdog_orchestrator.sh — NOT started by array_started.sh. Single-pass. # All 18 checks independently toggleable per host in master_host*.conf. # # Tier 1 CRITICAL — bypass ALL strikes, reboot immediately: @@ -663,6 +733,24 @@ # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh --status # bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/system_watchdog.sh --dry-run +# resource_watchdog.sh — three-level pressure reduction layer (single-pass, called by watchdog_orchestrator) +# Reduces system load intelligently BEFORE docker_watchdog attempts container restarts. +# Containers restarted into a resource-pressured system just fail again — this runs first. +# +# Three pressure levels (thresholds configured in master.conf): +# Level 1 SOFT (low RAM or high load): throttle SABnzbd + qBittorrent download speeds +# Level 2 MEDIUM (lower RAM or higher load): further throttle + docker pause non-critical containers +# Level 3 HARD (critically low RAM): docker stop heavy services + signal docker_watchdog to defer restarts +# +# Recovery: pressure must stay below current threshold for RW_RECOVER_CYCLES consecutive runs +# before de-escalating. One level at a time — prevents flip-flopping. +# Coordination: at Level 3 writes mem_shutdown_active=true → docker_watchdog defers all restarts. +# HOST*_RW_PAUSE_CONTAINERS and HOST*_RW_STOP_CONTAINERS configured per host in master_host*.conf. +# +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/resource_watchdog.sh --status +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/resource_watchdog.sh --dry-run +# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/resource_watchdog.sh + # inotify_tuning.sh — raise Linux inotify kernel limits at array start # Called by array_started.sh FIRST — must run before containers start (they inherit limits). # Settings reset on each reboot — script reapplies on every array start. Idempotent. @@ -737,7 +825,7 @@ # Deployment mechanism for the ecosystem. Push from VS Code → Gitea → run on both servers. # Called automatically by daily_sync_maintenance.sh as the first job in the daily window. # -# bash /mnt/user/appdata/unraid_scripts/unRAID_Essentials/git_pull_execute.sh +# bash /mnt/user/appdata/unraid_scripts/git_pull_execute.sh # ────────────────────────────────────────────────────────────────────────────────────────────── @@ -1045,8 +1133,8 @@ # Fallback state, active tiers, outage duration, remote Tailscale visibility. # No schedule — on demand only. Run any time you want a complete status snapshot. # -# bash /mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh -# bash /mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh --log +# bash /mnt/user/appdata/unraid_scripts/Tools/continuous_scripts_status.sh +# bash /mnt/user/appdata/unraid_scripts/Tools/continuous_scripts_status.sh --log # ────────────────────────────────────────────────────────────────────────────────────────────── @@ -1200,6 +1288,12 @@ # At Startup of Array: # Orchestrators/array_started.sh (single entry — handles everything) # +# At Stopping of Array: +# Orchestrators/array_stopping.sh (single entry — graceful shutdown) +# +# * * * * * every minute: +# Orchestrators/watchdog_orchestrator.sh +# # */3 * * * * every 3 minutes: # Orchestrators/transcode_management.sh #