Gmer4Lfe
2a062e5140
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
251 lines
10 KiB
Bash
Executable File
251 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ================================= Array Start Orchestrator ===================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Single entry point for array start — fired by the Varaverk plugin's
|
|
# disks_mounted event hook (Plugin/unraid/event/disks_mounted/array_start_jobs).
|
|
# Launches everything configured in ARRAY_START_SCRIPTS in master.conf.
|
|
# This script exits after launching all scripts — the event hook sees it complete normally.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Configured in master.conf ARRAY_START_SCRIPTS — no changes to this script ever needed.
|
|
# Current order (order matters — see below):
|
|
#
|
|
# ONE-SHOT (run and exit naturally):
|
|
# System_Essentials/unraid_api_key_renew.sh — re-register Varaverk API key at boot
|
|
# System_Essentials/inotify_tuning.sh — raise inotify limits before containers start
|
|
# System_Essentials/docker_syslog_filter.sh — suppress veth log noise before logs fill
|
|
# System_Essentials/php_fpm_max_children.sh — WebGUI performance tuning
|
|
# Transcodes/ramdisk_setup.sh — create tmpfs + symlink before Emby starts
|
|
# Docker_Essentials/docker_network_connect.sh — ensure networks + container connections
|
|
#
|
|
# CONTINUOUS (run until array stops):
|
|
# Fallback/fallback.sh — mutual fallback monitor
|
|
#
|
|
# NOTE: watchdogs (docker, system, stability) are NOT launched here.
|
|
# They run via watchdog_orchestrator.sh every 15 min (cron), not as daemons.
|
|
#
|
|
# Script is launched in background with bash script.sh &
|
|
# After 1 second: if PID still alive → continuous (running in background)
|
|
# if PID dead + exit 0 → one-shot completed successfully
|
|
# if PID dead + exit N → failure
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Order Is Load-Bearing
|
|
# inotify limits must be raised before containers start — containers that
|
|
# start with low limits keep them. Ramdisk must exist before Emby starts.
|
|
# Docker networks must be connected before watchdogs check container states.
|
|
# fallback.sh goes last — it needs everything else stable to make decisions.
|
|
#
|
|
# Configuration Owns the List
|
|
# ARRAY_START_SCRIPTS in master.conf is the only place scripts are added or
|
|
# removed. This orchestrator never needs to be edited to change what runs —
|
|
# one-shot vs continuous behaviour is auto-detected from the PID after launch.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root check — all launched scripts require root
|
|
# acquire_lock — prevents duplicate array start launches
|
|
# detect_hosts() — MY_ID in notifications
|
|
# platform_require_cmd — notify validated before use
|
|
# chmod +x auto-fix — non-executable scripts fixed before launch
|
|
# Full path on failure — shows exact path for debugging
|
|
# notify on failures — alert if any script fails to launch
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# ARRAY_START_SCRIPTS — ordered list of scripts to launch at array start
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# array_started.sh
|
|
# Normal launch — called by Varaverk disks_mounted event hook.
|
|
#
|
|
# array_started.sh --dry-run
|
|
# Show what would be launched without launching.
|
|
#
|
|
# array_started.sh --status
|
|
# Show configured scripts and their current state.
|
|
#
|
|
# array_started.sh --log
|
|
# Verbose output per script.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
|
|
source "$ECOSYSTEM_ROOT/load_config.sh"
|
|
|
|
parse_args "$@"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
|
|
acquire_lock
|
|
|
|
detect_hosts
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — scripts will not be launched"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY ARRAY START STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_GEAR Scripts: ${#ARRAY_START_SCRIPTS[@]} configured"
|
|
echo ""
|
|
|
|
for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
|
[[ -z "$relative_path" ]] && continue
|
|
script_path="$ECOSYSTEM_ROOT/$relative_path"
|
|
script_name=$(basename "$script_path")
|
|
|
|
if [[ ! -f "$script_path" ]]; then
|
|
echo " $ICON_ERROR $script_name — FILE NOT FOUND"
|
|
echo " $script_path"
|
|
continue
|
|
fi
|
|
|
|
[[ ! -x "$script_path" ]] && flag=" (not executable — will auto-fix)" || flag=""
|
|
|
|
# Check if currently running
|
|
if pgrep -f "$script_path" >/dev/null 2>&1; then
|
|
RUN_PID=$(pgrep -f "$script_path" | head -1)
|
|
echo " $ICON_RUNNING $script_name — RUNNING (PID $RUN_PID)${flag}"
|
|
else
|
|
echo " $ICON_NOT_RUNNING $script_name — not running${flag}"
|
|
fi
|
|
done
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Launch Scripts ━━━
|
|
# ==============================================================================================
|
|
if [[ ${#ARRAY_START_SCRIPTS[@]} -eq 0 ]]; then
|
|
notify "Array started on $LOCAL_SERVER_NAME ($MY_ID) but ARRAY_START_SCRIPTS is empty — boot sequence skipped. Check master.conf." \
|
|
"Array Start" "alert"
|
|
error "ARRAY_START_SCRIPTS is empty — check master.conf"
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_GEAR Array Start — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
log "Ecosystem root: $ECOSYSTEM_ROOT"
|
|
log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..."
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
JOB_PASS=()
|
|
JOB_FAIL=()
|
|
|
|
for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
|
[[ -z "$relative_path" ]] && continue
|
|
|
|
SCRIPT_PATH="$ECOSYSTEM_ROOT/$relative_path"
|
|
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
|
|
|
|
# File existence check
|
|
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
|
error "$SCRIPT_NAME — not found"
|
|
error " Expected: $SCRIPT_PATH"
|
|
JOB_FAIL+=("$SCRIPT_NAME")
|
|
continue
|
|
fi
|
|
|
|
# Auto-fix permissions — chmod +x if needed
|
|
if [[ ! -x "$SCRIPT_PATH" ]]; then
|
|
warn "$SCRIPT_NAME — not executable, fixing..."
|
|
chmod +x "$SCRIPT_PATH" || {
|
|
error "$SCRIPT_NAME — chmod +x failed"
|
|
JOB_FAIL+=("$SCRIPT_NAME")
|
|
continue
|
|
}
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would launch: $SCRIPT_NAME"
|
|
JOB_PASS+=("$SCRIPT_NAME")
|
|
continue
|
|
fi
|
|
|
|
log "$ICON_START Launching $SCRIPT_NAME..."
|
|
bash "$SCRIPT_PATH" &
|
|
PID=$!
|
|
|
|
# Brief settle — 1s enough to detect immediate failures
|
|
sleep 1
|
|
|
|
if kill -0 "$PID" 2>/dev/null; then
|
|
# Still running → continuous script
|
|
log "$SCRIPT_NAME — running (PID $PID) ✅"
|
|
JOB_PASS+=("$SCRIPT_NAME")
|
|
else
|
|
# Exited — check if one-shot success or failure
|
|
wait "$PID"
|
|
EXIT_CODE=$?
|
|
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
|
echo "$SCRIPT_NAME — completed (one-shot) ✅"
|
|
JOB_PASS+=("$SCRIPT_NAME")
|
|
else
|
|
error "$SCRIPT_NAME — exited with code $EXIT_CODE"
|
|
error " Path: $SCRIPT_PATH"
|
|
JOB_FAIL+=("$SCRIPT_NAME")
|
|
fi
|
|
fi
|
|
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_SUCCESS Launched: ${#JOB_PASS[@]}"
|
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#JOB_FAIL[@]} — ${JOB_FAIL[*]}"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no scripts launched"
|
|
elif [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|
warn "Status: ${#JOB_FAIL[@]} script(s) failed — ${JOB_FAIL[*]}"
|
|
notify "Array start on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} script(s) failed: ${JOB_FAIL[*]}" \
|
|
"Array Start" "warning"
|
|
else
|
|
echo "$ICON_DONE Status: all ${#JOB_PASS[@]} script(s) launched ✅"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
|
exit 0 |