Gmer4Lfe
2a062e5140
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
257 lines
11 KiB
Bash
Executable File
257 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
||
# ==============================================================================================
|
||
# ============================ Watchdog Orchestrator ===========================================
|
||
# ==============================================================================================
|
||
#
|
||
# PURPOSE
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. Replaces the
|
||
# continuous loops previously embedded in individual watchdog scripts — those
|
||
# are now single-pass; this orchestrator provides the cadence.
|
||
# Schedule: */15 * * * * (every 15 minutes)
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL MODEL
|
||
# ==============================================================================================
|
||
#
|
||
# Order driven by WATCHDOG_ORCHESTRATOR_SCRIPTS in master.conf.
|
||
# Default: resource_watchdog → docker_watchdog → system_watchdog →
|
||
# unraid_api_key_renew → stability_watchdog
|
||
#
|
||
# ARRAY CHECK
|
||
# Exits immediately if /mnt/user is not mounted as shfs. Watchdogs check
|
||
# Docker containers and storage — meaningless without the array. Prevents
|
||
# false positives and unnecessary reboots when array is stopped or stopping.
|
||
#
|
||
# STARTUP GRACE
|
||
# No action until system uptime >= WATCHDOG_STARTUP_GRACE seconds. Prevents
|
||
# false positives from containers still starting at array launch. Each
|
||
# sub-script enforces this independently — orchestrator exits early to avoid
|
||
# log noise.
|
||
#
|
||
# ==============================================================================================
|
||
# DESIGN PRINCIPLES
|
||
# ==============================================================================================
|
||
#
|
||
# Pressure Before Healing
|
||
# Resource Watchdog runs first — it frees RAM and CPU before any container
|
||
# restart is attempted. Containers restarted into a resource-pressured system
|
||
# just fail again. Docker Watchdog restarts with pressure already reduced.
|
||
# System Watchdog checks component health after containers are healed.
|
||
# Stability Watchdog reboots only when all prior layers could not resolve the
|
||
# issue. API key renew is check-first and silent when valid.
|
||
#
|
||
# Never Queue
|
||
# acquire_lock exits immediately if a prior cycle is still running. Prevents
|
||
# pile-up when a cycle runs long (daemon restart attempt = 30s, etc.).
|
||
#
|
||
# ==============================================================================================
|
||
# OPERATIONAL SAFEGUARDS
|
||
# ==============================================================================================
|
||
#
|
||
# Root check — watchdog operations require root
|
||
# acquire_lock — strict; no pile-up if prior cycle still active
|
||
# detect_hosts() — MY_ID in logs and notifications
|
||
# Array check — exits early if /mnt/user is not shfs-mounted
|
||
# Startup grace — WATCHDOG_STARTUP_GRACE respected before any checks
|
||
#
|
||
# ==============================================================================================
|
||
# CONFIGURATION
|
||
# ==============================================================================================
|
||
#
|
||
# master.conf
|
||
#
|
||
# WATCHDOG_ORCHESTRATOR_SCRIPTS — watchdogs to run, in order
|
||
# WATCHDOG_STARTUP_GRACE — seconds after boot before checks activate
|
||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT — periodic heartbeat log toggle
|
||
# WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS — heartbeat interval in hours
|
||
#
|
||
# ==============================================================================================
|
||
# RUNTIME MODES
|
||
# ==============================================================================================
|
||
#
|
||
# watchdog_orchestrator.sh
|
||
# Normal run (called by cron every 15 minutes).
|
||
#
|
||
# watchdog_orchestrator.sh --dry-run
|
||
# Pass --dry-run to all sub-scripts.
|
||
#
|
||
# watchdog_orchestrator.sh --status
|
||
# Show script paths and current grace state.
|
||
#
|
||
# watchdog_orchestrator.sh --log
|
||
# Verbose output from all sub-scripts.
|
||
#
|
||
# ==============================================================================================
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||
|
||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||
|
||
parse_args "$@"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Setup ━━━
|
||
# ==============================================================================================
|
||
if [[ "$EUID" -ne 0 ]]; then
|
||
error "Must be run as root"
|
||
exit 1
|
||
fi
|
||
|
||
# Skip immediately if another cycle is still running — no pile-up
|
||
acquire_lock
|
||
|
||
detect_hosts
|
||
|
||
log "$ICON_GEAR Config: grace=${WATCHDOG_STARTUP_GRACE}s heartbeat=${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}/${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr scripts=${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"
|
||
log "$ICON_WATCHDOG Order: $(for s in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do printf '%s ' "${s##*/}"; done)"
|
||
|
||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — passing --dry-run to all sub-scripts"
|
||
|
||
# Derive a display name from a script path: "resource_watchdog.sh" → "Resource Watchdog"
|
||
_watchdog_display_name() {
|
||
local path="$1"
|
||
local base="${path##*/}"
|
||
base="${base%.sh}"
|
||
base="${base//_/ }"
|
||
echo "$base" | awk '{for(i=1;i<=NF;i++) $i=toupper(substr($i,1,1)) substr($i,2); print}'
|
||
}
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Status ━━━
|
||
# ==============================================================================================
|
||
if [[ "$SHOW_STATUS" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY WATCHDOG ORCHESTRATOR STATUS ━━━━━"
|
||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||
echo ""
|
||
|
||
UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
|
||
if [[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||
warn "Within startup grace — $(format_duration $UPTIME_S) / $(format_duration $WATCHDOG_STARTUP_GRACE)"
|
||
else
|
||
echo "Past startup grace — $(format_duration $UPTIME_S) uptime"
|
||
fi
|
||
|
||
echo ""
|
||
echo "── Sub-scripts ──"
|
||
for entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||
local_path="$ECOSYSTEM_ROOT/$entry"
|
||
label="$(_watchdog_display_name "$entry")"
|
||
if [[ -f "$local_path" ]]; then
|
||
[[ -x "$local_path" ]] && icon="$ICON_DONE" || icon="$ICON_WARN"
|
||
echo " $icon $label — ${local_path##*/}"
|
||
else
|
||
echo " $ICON_ERROR $label — NOT FOUND: $local_path"
|
||
fi
|
||
done
|
||
|
||
echo ""
|
||
echo " Schedule: */15 * * * * (every 15 minutes)"
|
||
echo " Heartbeat: ${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true} / every ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1}hr"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||
exit 0
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Array Check ━━━
|
||
# ==============================================================================================
|
||
if ! platform_storage_healthy; then
|
||
echo "Array not started — skipping watchdog cycle"
|
||
exit 0
|
||
fi
|
||
log "$ICON_DISK Array: $(platform_storage_path) mounted ✅"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Startup Grace ━━━
|
||
# ==============================================================================================
|
||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||
echo "Startup grace — $(format_duration $UPTIME_SECONDS) / $(format_duration $WATCHDOG_STARTUP_GRACE) — skipping cycle"
|
||
exit 0
|
||
fi
|
||
log "Startup grace: past — uptime $(format_duration $UPTIME_SECONDS)"
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Run Watchdog Cycle ━━━
|
||
# ==============================================================================================
|
||
CYCLE_START=$(date +%s)
|
||
PASS=()
|
||
FAIL=()
|
||
|
||
run_watchdog() {
|
||
local name="$1" script="$2"
|
||
|
||
if [[ ! -f "$script" ]]; then
|
||
error "$name — not found: $script"
|
||
FAIL+=("$name:missing")
|
||
return 1
|
||
fi
|
||
|
||
[[ ! -x "$script" ]] && chmod +x "$script"
|
||
|
||
local extra_args=()
|
||
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
||
[[ "$ENABLE_LOGGING" == true ]] && extra_args+=("--log")
|
||
|
||
local _ws
|
||
_ws=$(date +%s)
|
||
log "$ICON_START $name"
|
||
if bash "$script" "${extra_args[@]}"; then
|
||
log "$ICON_DONE $name — done in $(format_duration $(( $(date +%s) - _ws )))"
|
||
PASS+=("$name")
|
||
return 0
|
||
else
|
||
error "$name — non-zero exit ($(format_duration $(( $(date +%s) - _ws ))))"
|
||
FAIL+=("$name")
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
for _entry in "${WATCHDOG_ORCHESTRATOR_SCRIPTS[@]}"; do
|
||
run_watchdog "$(_watchdog_display_name "$_entry")" "$ECOSYSTEM_ROOT/$_entry"
|
||
done
|
||
|
||
CYCLE_END=$(date +%s)
|
||
DURATION=$(( CYCLE_END - CYCLE_START ))
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Heartbeat ━━━
|
||
# ==============================================================================================
|
||
if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
|
||
HB_SECONDS=$(( ${WATCHDOG_ORCHESTRATOR_HEARTBEAT_HOURS:-1} * 3600 ))
|
||
HB_COUNT_FILE="${STATE_DIR:-/tmp}/watchdog_orch_hb.count"
|
||
HB_COUNT=$(cat "$HB_COUNT_FILE" 2>/dev/null || echo 0)
|
||
HB_COUNT=$(( HB_COUNT + 1 ))
|
||
echo "$HB_COUNT" > "$HB_COUNT_FILE"
|
||
# Each cron run = ~60s — use count × 60 as uptime approximation
|
||
HB_ELAPSED=$(( HB_COUNT * 60 ))
|
||
if [[ "$HB_SECONDS" -gt 0 ]] && (( HB_ELAPSED % HB_SECONDS < 60 )) && [[ "$HB_COUNT" -gt 1 ]]; then
|
||
HB_HR=$(( HB_ELAPSED / 3600 ))
|
||
warn "♥ watchdog_orchestrator alive — $MY_ID — ~${HB_HR}hr ($(date '+%H:%M:%S'))"
|
||
fi
|
||
fi
|
||
|
||
# ==============================================================================================
|
||
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
|
||
# ==============================================================================================
|
||
if [[ "${#FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
|
||
echo ""
|
||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||
for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done
|
||
for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
else
|
||
echo "$ICON_DONE Watchdog cycle — ${#PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
|
||
fi
|
||
|
||
if [[ "${#FAIL[@]}" -gt 0 ]]; then
|
||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||
"Watchdog Orchestrator" "warning"
|
||
exit 1
|
||
fi
|
||
|
||
exit 0
|