Standardize orchestrator child-script execution and logging
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
This commit is contained in:
Gmer4Lfe
@@ -5,14 +5,17 @@
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Runs transcode_cleanup.sh then transcode_manager.sh in the correct order.
|
||||
# Replaces individual cron entries for each — this is the single cron entry.
|
||||
# Schedule: */7 * * * * (every 7 minutes via User Scripts plugin)
|
||||
# Runs TRANSCODE_MANAGEMENT_SCRIPTS in order each cron cycle — this is the
|
||||
# single cron entry replacing individual entries for each child script.
|
||||
# Schedule: */7 * * * * (every 7 minutes)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Order driven by TRANSCODE_MANAGEMENT_SCRIPTS in master.conf.
|
||||
# Default: transcode_cleanup.sh → transcode_manager.sh
|
||||
#
|
||||
# transcode_cleanup.sh
|
||||
# Removes aged segment files not open by any process. Uses lsof for O(1)
|
||||
# per-file active check. Triggers flip-back to ramdisk after cleanup if
|
||||
@@ -36,6 +39,8 @@
|
||||
# Stale segment files from ended sessions inflate the ramdisk usage reading
|
||||
# and trigger unnecessary SSD flips even when active sessions would fit on
|
||||
# the ramdisk. Cleanup runs first so the manager measures real current usage.
|
||||
# Order is config-driven (TRANSCODE_MANAGEMENT_SCRIPTS) but this dependency
|
||||
# is real — reordering the array changes what the manager measures.
|
||||
#
|
||||
# Delegated Logging
|
||||
# This orchestrator does not write its own log — transcode_manager.sh owns
|
||||
@@ -48,8 +53,9 @@
|
||||
# Root check — mount and docker operations require root
|
||||
# acquire_lock — prevents concurrent 7-minute cycles overlapping
|
||||
# detect_hosts() — aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_WARN_GB per host
|
||||
# --dry-run — passed through to both child scripts
|
||||
# Exit code — worst exit code of both scripts returned to cron
|
||||
# --dry-run — passed through to every script in TRANSCODE_MANAGEMENT_SCRIPTS
|
||||
# Exit code — worst exit code across all scripts returned to cron
|
||||
# notify() — pushed on failure, skipped in --dry-run
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
@@ -57,6 +63,7 @@
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# TRANSCODE_MANAGEMENT_SCRIPTS — scripts to run, in order
|
||||
# TRANSCODE_DAILY_LOG — daily stats log (written by transcode_manager.sh)
|
||||
# TRANSCODE_LOG_RETENTION — days to keep (trimmed by manager)
|
||||
# TRANSCODE_STATE_FILE — current state (ramdisk_used, flip_count, etc.)
|
||||
@@ -70,13 +77,13 @@
|
||||
# Normal run (every 7 minutes via cron).
|
||||
#
|
||||
# transcode_management.sh --dry-run
|
||||
# Preview without changes (passed to both child scripts).
|
||||
# Preview without changes (passed to every script in TRANSCODE_MANAGEMENT_SCRIPTS).
|
||||
#
|
||||
# transcode_management.sh --status
|
||||
# Show configuration and current state.
|
||||
#
|
||||
# transcode_management.sh --log
|
||||
# Verbose output from both child scripts.
|
||||
# Verbose output from every child script.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
@@ -86,9 +93,6 @@ source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
CLEANUP_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_cleanup.sh"
|
||||
MANAGER_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_manager.sh"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
@@ -122,12 +126,15 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_TIME Schedule: every 7 minutes"
|
||||
echo ""
|
||||
echo "━━━ Child Scripts ━━━"
|
||||
[[ -f "$CLEANUP_SCRIPT" ]] && \
|
||||
echo " $ICON_SUCCESS transcode_cleanup.sh — found" || \
|
||||
echo " $ICON_ERROR transcode_cleanup.sh — NOT FOUND at $CLEANUP_SCRIPT"
|
||||
[[ -f "$MANAGER_SCRIPT" ]] && \
|
||||
echo " $ICON_SUCCESS transcode_manager.sh — found" || \
|
||||
echo " $ICON_ERROR transcode_manager.sh — NOT FOUND at $MANAGER_SCRIPT"
|
||||
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
|
||||
_script_path="$SCRIPT_DIR/../$_entry"
|
||||
_script_name=$(basename "$_entry")
|
||||
if [[ -f "$_script_path" ]]; then
|
||||
echo " $ICON_SUCCESS $_script_name — found"
|
||||
else
|
||||
echo " $ICON_ERROR $_script_name — NOT FOUND at $_script_path"
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
echo "━━━ Daily Log ━━━"
|
||||
if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]] && [[ -s "$TRANSCODE_DAILY_LOG" ]]; then
|
||||
@@ -154,15 +161,13 @@ fi
|
||||
# ==============================================================================================
|
||||
# ━━━ Validate Child Scripts ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ! -f "$CLEANUP_SCRIPT" ]]; then
|
||||
error "transcode_cleanup.sh not found: $CLEANUP_SCRIPT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$MANAGER_SCRIPT" ]]; then
|
||||
error "transcode_manager.sh not found: $MANAGER_SCRIPT"
|
||||
exit 1
|
||||
fi
|
||||
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
|
||||
_script_path="$SCRIPT_DIR/../$_entry"
|
||||
if [[ ! -f "$_script_path" ]]; then
|
||||
error "$(basename "$_entry") not found: $_script_path"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-run State Snapshot ━━━
|
||||
@@ -179,29 +184,46 @@ else
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Run Cleanup ━━━
|
||||
# ━━━ Run Scripts ━━━
|
||||
# ==============================================================================================
|
||||
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run — no flag here
|
||||
# suppresses that; it owns the log write for this cycle regardless of position ✅
|
||||
DRY_FLAG=""
|
||||
[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run"
|
||||
|
||||
_cleanup_start=$(date +%s)
|
||||
bash "$CLEANUP_SCRIPT" $DRY_FLAG
|
||||
CLEANUP_EXIT=$?
|
||||
log "cleanup: $(format_duration $(( $(date +%s) - _cleanup_start ))) (exit $CLEANUP_EXIT)"
|
||||
WORST_EXIT=0
|
||||
PASS_COUNT=0
|
||||
FAIL_NAMES=()
|
||||
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
|
||||
_script_path="$SCRIPT_DIR/../$_entry"
|
||||
_script_name=$(basename "$_entry" .sh)
|
||||
_start=$(date +%s)
|
||||
bash "$_script_path" $DRY_FLAG
|
||||
_exit=$?
|
||||
log "$_script_name: $(format_duration $(( $(date +%s) - _start ))) (exit $_exit)"
|
||||
if [[ "$_exit" -ne 0 ]]; then
|
||||
WORST_EXIT=1
|
||||
FAIL_NAMES+=("$_script_name")
|
||||
else
|
||||
PASS_COUNT=$(( PASS_COUNT + 1 ))
|
||||
fi
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Run Manager ━━━
|
||||
# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
|
||||
# ==============================================================================================
|
||||
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run
|
||||
# No --no-log flag here — manager owns the log write for this cycle ✅
|
||||
_manager_start=$(date +%s)
|
||||
bash "$MANAGER_SCRIPT" $DRY_FLAG
|
||||
MANAGER_EXIT=$?
|
||||
log "manager: $(format_duration $(( $(date +%s) - _manager_start ))) (exit $MANAGER_EXIT)"
|
||||
if [[ "$WORST_EXIT" -eq 0 ]]; then
|
||||
echo "$ICON_SUCCESS Transcode cycle — $PASS_COUNT/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
|
||||
else
|
||||
error "Transcode cycle — failed: ${FAIL_NAMES[*]}"
|
||||
if [[ "$DRY_RUN" != true ]]; then
|
||||
notify "Transcode management failure on $(hostname) ($MY_ID) — ${FAIL_NAMES[*]}" \
|
||||
"Transcode Management" "warning"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Exit ━━━
|
||||
# ==============================================================================================
|
||||
# Return worst exit code — caller knows if either script failed
|
||||
[[ "$CLEANUP_EXIT" -ne 0 || "$MANAGER_EXIT" -ne 0 ]] && exit 1
|
||||
exit 0
|
||||
# Return worst exit code — caller knows if any script failed
|
||||
exit "$WORST_EXIT"
|
||||
Reference in New Issue
Block a user