Standardize orchestrator child-script execution and logging

Every orchestrator invoked its children differently — four near-duplicate
run_job() copies, a differently-shaped run_watchdog(), or plain inline bash
calls, each with its own take on path resolution, pass/fail naming, and
dry-run threading. Extracted one shared run_orch_child() into common.sh so
there's a single place to fix or extend this behavior going forward.

Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were
checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log
silently did nothing beyond basic logging on those two. Fixed to
$ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no
trailing exit, so their exit codes reflected whatever the last command
happened to return rather than actual success/failure. transcode_management.sh
had no failure notification and no summary at all. Also made
transcode_management.sh's two-script pipeline config-driven
(TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room
to extend it later without editing the orchestrator itself.
This commit is contained in:
Gmer4Lfe
2026-07-03 10:57:52 -04:00
parent 6fd22ae4ee
commit 2a062e5140
14 changed files with 303 additions and 370 deletions
+10
View File
@@ -320,6 +320,16 @@
"Docker_Essentials/docker_container_stop.sh" # stop all containers last "Docker_Essentials/docker_container_stop.sh" # stop all containers last
) )
# ━━━ Transcode Management ━━━
# transcode_management.sh runs TRANSCODE_MANAGEMENT_SCRIPTS in order each cron cycle.
# Schedule: */7 * * * * (every 7 minutes)
# Order matters — cleanup first so the manager measures real current ramdisk usage,
# not usage inflated by stale segment files from ended sessions.
TRANSCODE_MANAGEMENT_SCRIPTS=(
"Transcodes/transcode_cleanup.sh" # remove aged segment files before usage is measured
"Transcodes/transcode_manager.sh" # flip ramdisk/SSD symlink, write daily log entry
)
# ━━━ Watchdog Orchestrator ━━━ # ━━━ Watchdog Orchestrator ━━━
# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. # watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
# Schedule: */15 * * * * (every 15 minutes) # Schedule: */15 * * * * (every 15 minutes)
+10
View File
@@ -317,6 +317,16 @@
"Docker_Essentials/docker_container_stop.sh" # stop all containers last "Docker_Essentials/docker_container_stop.sh" # stop all containers last
) )
# ━━━ Transcode Management ━━━
# transcode_management.sh runs TRANSCODE_MANAGEMENT_SCRIPTS in order each cron cycle.
# Schedule: */7 * * * * (every 7 minutes)
# Order matters — cleanup first so the manager measures real current ramdisk usage,
# not usage inflated by stale segment files from ended sessions.
TRANSCODE_MANAGEMENT_SCRIPTS=(
"Transcodes/transcode_cleanup.sh" # remove aged segment files before usage is measured
"Transcodes/transcode_manager.sh" # flip ramdisk/SSD symlink, write daily log entry
)
# ━━━ Watchdog Orchestrator ━━━ # ━━━ Watchdog Orchestrator ━━━
# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. # watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
# Schedule: */15 * * * * (every 15 minutes) # Schedule: */15 * * * * (every 15 minutes)
+21 -22
View File
@@ -942,41 +942,40 @@ array_started.sh
## ━━━ ADDING A NEW ORCHESTRATOR ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ## ━━━ ADDING A NEW ORCHESTRATOR ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
If you find yourself running 3+ related scripts on the same schedule, wrap them If you find yourself running 3+ related scripts on the same schedule, wrap them
in a new orchestrator. Model directly on `media_management.sh` which has the in a new orchestrator. Model directly on `daily_sync_maintenance.sh` which has the
complete pattern — dry-run passthrough, status display, pass/fail tracking, summary. complete pattern — dry-run passthrough, status display, pass/fail tracking, summary.
Child-script execution goes through the shared `run_orch_child()` helper in
`common.sh` — never hand-roll a per-file `run_job()` loop. It resolves the entry
against `$ECOSYSTEM_ROOT`, threads `--dry-run`/`--log` from `$DRY_RUN`/`$ENABLE_LOGGING`
automatically (never `$VERBOSE` — nothing in this codebase assigns it), and tracks
into `JOB_PASS`/`JOB_FAIL` arrays the caller declares.
```bash ```bash
# Minimal skeleton — the full pattern in its simplest form: # Minimal skeleton — the full pattern in its simplest form:
#!/bin/bash #!/bin/bash
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh" ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "$ECOSYSTEM_ROOT/load_config.sh"
parse_args "$@" parse_args "$@"
SCRIPTS_ROOT="$SCRIPT_DIR/.." JOB_PASS=()
PASS=() JOB_FAIL=()
FAIL=()
# Read job list from master.conf — never hardcode jobs in the orchestrator # Read job list from master.conf — never hardcode jobs in the orchestrator
for script_entry in "${MY_MAINTENANCE_JOBS[@]:-}"; do for script_entry in "${MY_MAINTENANCE_JOBS[@]:-}"; do
[[ -z "$script_entry" ]] && continue [[ -z "$script_entry" ]] && continue
run_orch_child "$script_entry"
read -r -a parts <<< "$script_entry"
script_path="$SCRIPTS_ROOT/${parts[0]}"
script_name=$(basename "${parts[0]}")
extra_args=("${parts[@]:1}")
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
if bash "$script_path" "${extra_args[@]}"; then
PASS+=("$script_name")
else
FAIL+=("$script_name")
fi
done done
# One summary — one notification # One summary — one notification. On a frequent (sub-daily) cadence, keep the
echo "Passed: ${#PASS[@]} Failed: ${#FAIL[@]}" # healthy path to a single line and reserve the full breakdown for failure/--log —
[[ ${#FAIL[@]} -gt 0 ]] && \ # see watchdog_orchestrator.sh or transcode_management.sh for that split.
notify "My maintenance failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \ echo "Passed: ${#JOB_PASS[@]} Failed: ${#JOB_FAIL[@]}"
if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
notify "My maintenance failed on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
"My Orchestrator" "warning" "My Orchestrator" "warning"
exit 1
fi
exit 0
``` ```
+17 -18
View File
@@ -163,9 +163,8 @@ log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..."
echo "" echo ""
START=$(date +%s) START=$(date +%s)
LAUNCHED=0 JOB_PASS=()
FAILED=0 JOB_FAIL=()
FAILED_SCRIPTS=()
for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
[[ -z "$relative_path" ]] && continue [[ -z "$relative_path" ]] && continue
@@ -177,8 +176,7 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
if [[ ! -f "$SCRIPT_PATH" ]]; then if [[ ! -f "$SCRIPT_PATH" ]]; then
error "$SCRIPT_NAME — not found" error "$SCRIPT_NAME — not found"
error " Expected: $SCRIPT_PATH" error " Expected: $SCRIPT_PATH"
(( FAILED++ )) JOB_FAIL+=("$SCRIPT_NAME")
FAILED_SCRIPTS+=("$SCRIPT_NAME")
continue continue
fi fi
@@ -187,15 +185,14 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
warn "$SCRIPT_NAME — not executable, fixing..." warn "$SCRIPT_NAME — not executable, fixing..."
chmod +x "$SCRIPT_PATH" || { chmod +x "$SCRIPT_PATH" || {
error "$SCRIPT_NAME — chmod +x failed" error "$SCRIPT_NAME — chmod +x failed"
(( FAILED++ )) JOB_FAIL+=("$SCRIPT_NAME")
FAILED_SCRIPTS+=("$SCRIPT_NAME")
continue continue
} }
fi fi
if [[ "$DRY_RUN" == true ]]; then if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would launch: $SCRIPT_NAME" warn "DRY RUN — would launch: $SCRIPT_NAME"
(( LAUNCHED++ )) JOB_PASS+=("$SCRIPT_NAME")
continue continue
fi fi
@@ -209,19 +206,18 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
if kill -0 "$PID" 2>/dev/null; then if kill -0 "$PID" 2>/dev/null; then
# Still running → continuous script # Still running → continuous script
log "$SCRIPT_NAME — running (PID $PID) ✅" log "$SCRIPT_NAME — running (PID $PID) ✅"
(( LAUNCHED++ )) JOB_PASS+=("$SCRIPT_NAME")
else else
# Exited — check if one-shot success or failure # Exited — check if one-shot success or failure
wait "$PID" wait "$PID"
EXIT_CODE=$? EXIT_CODE=$?
if [[ "$EXIT_CODE" -eq 0 ]]; then if [[ "$EXIT_CODE" -eq 0 ]]; then
echo "$SCRIPT_NAME — completed (one-shot) ✅" echo "$SCRIPT_NAME — completed (one-shot) ✅"
(( LAUNCHED++ )) JOB_PASS+=("$SCRIPT_NAME")
else else
error "$SCRIPT_NAME — exited with code $EXIT_CODE" error "$SCRIPT_NAME — exited with code $EXIT_CODE"
error " Path: $SCRIPT_PATH" error " Path: $SCRIPT_PATH"
(( FAILED++ )) JOB_FAIL+=("$SCRIPT_NAME")
FAILED_SCRIPTS+=("$SCRIPT_NAME")
fi fi
fi fi
@@ -235,18 +231,21 @@ END=$(date +%s)
echo "" echo ""
echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━" echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_SUCCESS Launched: $LAUNCHED" echo "$ICON_SUCCESS Launched: ${#JOB_PASS[@]}"
[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR Failed: $FAILED${FAILED_SCRIPTS[*]}" [[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#JOB_FAIL[@]}${JOB_FAIL[*]}"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
echo "" echo ""
if [[ "$DRY_RUN" == true ]]; then if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no scripts launched" warn "DRY RUN — no scripts launched"
elif [[ "$FAILED" -gt 0 ]]; then elif [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}" warn "Status: ${#JOB_FAIL[@]} script(s) failed — ${JOB_FAIL[*]}"
notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \ notify "Array start on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} script(s) failed: ${JOB_FAIL[*]}" \
"Array Start" "warning" "Array Start" "warning"
else else
echo "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅" echo "$ICON_DONE Status: all ${#JOB_PASS[@]} script(s) launched ✅"
fi fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
exit 0
+9 -42
View File
@@ -134,8 +134,8 @@ echo "$ICON_GEAR Running ${#ARRAY_STOP_SCRIPTS[@]} stop script(s) sequentially..
echo "" echo ""
START=$(date +%s) START=$(date +%s)
PASSED=() JOB_PASS=()
FAILED=() JOB_FAIL=()
STEP=0 STEP=0
for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
@@ -143,44 +143,11 @@ for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
(( STEP++ )) (( STEP++ ))
read -r -a parts <<< "$entry" read -r -a parts <<< "$entry"
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
script_name=$(basename "${parts[0]}") script_name=$(basename "${parts[0]}")
extra_args=("${parts[@]:1}") extra_args=("${parts[@]:1}")
echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━" echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
run_orch_child "$entry"
if [[ ! -f "$script_path" ]]; then
error "$script_name — not found at $script_path"
FAILED+=("$script_name")
echo ""
continue
fi
if [[ ! -x "$script_path" ]]; then
warn "$script_name — not executable, fixing..."
chmod +x "$script_path" || {
error "$script_name — chmod +x failed"
FAILED+=("$script_name")
echo ""
continue
}
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would run: $script_name ${extra_args[*]}"
PASSED+=("$script_name")
echo ""
continue
fi
if bash "$script_path" "${extra_args[@]}"; then
echo "$script_name — done ✅"
PASSED+=("$script_name")
else
warn "$script_name — failed (exit $?) — continuing to next step"
FAILED+=("$script_name")
fi
echo "" echo ""
done done
@@ -192,22 +159,22 @@ END=$(date +%s)
echo "━━━━━ $ICON_SUMMARY ARRAY STOP SUMMARY ━━━━━" echo "━━━━━ $ICON_SUMMARY ARRAY STOP SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}" [[ ${#JOB_PASS[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${JOB_PASS[*]}"
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" [[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
echo "" echo ""
if [[ "$DRY_RUN" == true ]]; then if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no changes made" warn "DRY RUN — no changes made"
elif [[ ${#FAILED[@]} -eq 0 ]]; then elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
echo "$ICON_DONE Status: all $STEP step(s) complete ✅" echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \ notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
"Array Stop" "normal" "Array Stop" "normal"
else else
warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}" warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
notify "Array stop on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \ notify "Array stop on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
"Array Stop" "warning" "Array Stop" "warning"
fi fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#FAILED[@]} -gt 0 ]] && exit 1 [[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
exit 0 exit 0
+12 -25
View File
@@ -80,6 +80,7 @@
# ============================================================================================== # ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "$SCRIPT_DIR/../load_config.sh" source "$SCRIPT_DIR/../load_config.sh"
@@ -199,29 +200,12 @@ fi
# ============================================================================================== # ==============================================================================================
# ━━━ Critical Maintenance Scripts ━━━ # ━━━ Critical Maintenance Scripts ━━━
# ============================================================================================== # ==============================================================================================
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then JOB_PASS=()
JOB_FAIL=()
for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
[[ -z "$script_entry" || "$script_entry" == \#* ]] && continue [[ -z "$script_entry" || "$script_entry" == \#* ]] && continue
run_orch_child "$script_entry"
SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
SCRIPT_ARGS="${script_entry#* }"
[[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
[[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run"
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
if [[ ! -f "$SCRIPT_PATH" ]]; then
warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
continue
fi
log "Running: $SCRIPT_NAME"
bash "$SCRIPT_PATH" $SCRIPT_ARGS
EXIT_CODE=$?
[[ "$EXIT_CODE" -ne 0 ]] && \
warn "$SCRIPT_NAME exited with code $EXIT_CODE"
done done
fi
# ============================================================================================== # ==============================================================================================
# ━━━ Partnership Check ━━━ # ━━━ Partnership Check ━━━
@@ -247,20 +231,23 @@ fi
END=$(date +%s) END=$(date +%s)
DURATION=$(format_duration $(( END - START ))) DURATION=$(format_duration $(( END - START )))
# Silent when healthy — only show summary if there were failures or notable events TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
if [[ ${#FAIL[@]} -gt 0 ]]; then
# Minimal one-liner when healthy — 30-min cadence, keep it quiet. Full detail on failure.
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
echo "" echo ""
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━" echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $DURATION" echo "$ICON_TIME Duration: $DURATION"
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}" [[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
echo "$ICON_ERROR Failed: ${FAIL[*]}" [[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed shares: ${FAIL[*]}"
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \ notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]} ${JOB_FAIL[*]}" \
"Critical Sync" "warning" "Critical Sync" "warning"
exit 1 exit 1
else else
echo "Critical sync complete — $MY_ID${DURATION}${#PASS[@]} share(s)" echo "Critical sync complete — $MY_ID${DURATION}${#PASS[@]} share(s), ${#JOB_PASS[@]} job(s)"
fi fi
exit 0 exit 0
+5 -34
View File
@@ -103,8 +103,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh" source "$SCRIPT_DIR/../load_config.sh"
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
SCRIPTS_ROOT="$SCRIPT_DIR/.." RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"
parse_args "$@" parse_args "$@"
@@ -184,35 +184,6 @@ for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
fi fi
done done
# Helper — run a maintenance script, track pass/fail
run_job() {
local script_entry="$1"
local extra_dry=""
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
read -r -a script_args <<< "$script_entry"
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
local script_name
script_name=$(basename "${script_args[0]}")
local extra_args=("${script_args[@]:1}")
if [[ ! -f "$script_path" ]]; then
error "$script_name — not found at $script_path"
JOB_FAIL+=("$script_name")
return 1
fi
log "Running: $script_name ${extra_args[*]}"
# shellcheck disable=SC2086
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
echo "$script_name — done ✅"
JOB_PASS+=("$script_name ${extra_args[*]}")
else
error "$script_name — failed (exit $?)"
JOB_FAIL+=("$script_name ${extra_args[*]}")
fi
}
WINDOW_START=$(date +%s) WINDOW_START=$(date +%s)
JOB_PASS=() JOB_PASS=()
JOB_FAIL=() JOB_FAIL=()
@@ -230,7 +201,7 @@ if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then
echo "" echo ""
echo "━━━ $ICON_GIT Pre-sync ━━━" echo "━━━ $ICON_GIT Pre-sync ━━━"
for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
run_job "$script_entry" run_orch_child "$script_entry"
done done
fi fi
@@ -240,7 +211,7 @@ fi
echo "" echo ""
echo "━━━ $ICON_SYNC Arr Sync ━━━" echo "━━━ $ICON_SYNC Arr Sync ━━━"
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh" ARR_SYNC_SCRIPT="$ECOSYSTEM_ROOT/Arrs_Stack/arr_sync.sh"
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
echo "ARR_SYNC_ENABLED=false — skipping" echo "ARR_SYNC_ENABLED=false — skipping"
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
@@ -336,7 +307,7 @@ if [[ ${#POST_SYNC_SCRIPTS[@]} -gt 0 ]]; then
echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━" echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━"
for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
echo "" echo ""
run_job "$script_entry" run_orch_child "$script_entry"
done done
fi fi
+19 -38
View File
@@ -50,7 +50,7 @@
# check_connectivity — verified before any rsync (skipped if no shares) # check_connectivity — verified before any rsync (skipped if no shares)
# check_remote_rootfs — aborts rsync if remote rootfs nearly full # check_remote_rootfs — aborts rsync if remote rootfs nearly full
# Non-fatal jobs — a failed arr_sync warns but does not block rsync or artwork fetch # Non-fatal jobs — a failed arr_sync warns but does not block rsync or artwork fetch
# Silent on success — runs 4x/day, only failures warrant notification # Minimal on success — runs 6x/day; full breakdown only on failure or --log
# #
# ============================================================================================== # ==============================================================================================
# CONFIGURATION # CONFIGURATION
@@ -88,8 +88,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh" source "$SCRIPT_DIR/../load_config.sh"
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
SCRIPTS_ROOT="$SCRIPT_DIR/.." RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"
parse_args "$@" parse_args "$@"
@@ -109,35 +109,6 @@ resolve_remote_ip
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
# ── Helper — run a maintenance job, track pass/fail ───────────────────────────────────────────
run_job() {
local script_entry="$1"
local extra_dry=""
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
read -r -a script_args <<< "$script_entry"
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
local script_name
script_name=$(basename "${script_args[0]}")
local extra_args=("${script_args[@]:1}")
if [[ ! -f "$script_path" ]]; then
error "$script_name — not found at $script_path"
JOB_FAIL+=("$script_name")
return 1
fi
log "Running: $script_name ${extra_args[*]}"
# shellcheck disable=SC2086
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
echo "$script_name — done ✅"
JOB_PASS+=("$script_name ${extra_args[*]}")
else
warn "$script_name — failed (exit $?)"
JOB_FAIL+=("$script_name ${extra_args[*]}")
fi
}
# ============================================================================================== # ==============================================================================================
# ━━━ Status ━━━ # ━━━ Status ━━━
# ============================================================================================== # ==============================================================================================
@@ -187,7 +158,7 @@ echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H
echo "" echo ""
echo "━━━ $ICON_GEAR Conf Pull ━━━" echo "━━━ $ICON_GEAR Conf Pull ━━━"
CONF_SYNC_SCRIPT="$SCRIPTS_ROOT/System_Essentials/conf_sync.sh" CONF_SYNC_SCRIPT="$ECOSYSTEM_ROOT/System_Essentials/conf_sync.sh"
if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
warn "conf_sync.sh not found — skipping partner conf refresh" warn "conf_sync.sh not found — skipping partner conf refresh"
else else
@@ -209,7 +180,7 @@ fi
echo "" echo ""
echo "━━━ $ICON_SYNC Arr Sync ━━━" echo "━━━ $ICON_SYNC Arr Sync ━━━"
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh" ARR_SYNC_SCRIPT="$ECOSYSTEM_ROOT/Arrs_Stack/arr_sync.sh"
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
echo "ARR_SYNC_ENABLED=false — skipping" echo "ARR_SYNC_ENABLED=false — skipping"
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
@@ -306,7 +277,7 @@ if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
for script_entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do for script_entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
[[ -z "$script_entry" ]] && continue [[ -z "$script_entry" ]] && continue
echo "" echo ""
run_job "$script_entry" run_orch_child "$script_entry"
done done
fi fi
@@ -315,6 +286,13 @@ WINDOW_END=$(date +%s)
# ============================================================================================== # ==============================================================================================
# ━━━ Summary ━━━ # ━━━ Summary ━━━
# ============================================================================================== # ==============================================================================================
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
# Full breakdown on failure or --log; minimal one-liner otherwise (4-hour cadence — keep it quiet).
SHOW_FULL=false
[[ "$TOTAL_FAIL" -gt 0 || "$ENABLE_LOGGING" == true ]] && SHOW_FULL=true
if [[ "$SHOW_FULL" == true ]]; then
echo "" echo ""
echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━" echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
@@ -343,19 +321,22 @@ if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
echo "" echo ""
fi fi
fi
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
if [[ "$DRY_RUN" == true ]]; then if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no changes made" warn "DRY RUN — no changes made"
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
if [[ "$SHOW_FULL" == true ]]; then
echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced" echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
else
echo "$ICON_DONE Intermediate sync — ${#JOB_PASS[@]} job(s), ${#PASS[@]}/$SHARE_COUNT share(s) ($(format_duration $(( WINDOW_END - WINDOW_START ))))"
fi
else else
warn "Status: $TOTAL_FAIL failure(s)" warn "Status: $TOTAL_FAIL failure(s)"
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \ notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
"Intermediate Sync" "warning" "Intermediate Sync" "warning"
fi fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" [[ "$SHOW_FULL" == true ]] && echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1 [[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
exit 0 exit 0
+9 -45
View File
@@ -234,8 +234,8 @@ echo "$ICON_GEAR Scripts: ${#MONTHLY_MAINTENANCE_SCRIPTS[@]}"
echo "" echo ""
START=$(date +%s) START=$(date +%s)
PASSED=() JOB_PASS=()
FAILED=() JOB_FAIL=()
STEP=0 STEP=0
# ============================================================================================== # ==============================================================================================
@@ -246,47 +246,11 @@ for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
(( STEP++ )) (( STEP++ ))
read -r -a parts <<< "$entry" read -r -a parts <<< "$entry"
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
script_name=$(basename "${parts[0]}") script_name=$(basename "${parts[0]}")
extra_args=("${parts[@]:1}") extra_args=("${parts[@]:1}")
echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━" echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
run_orch_child "$entry"
if [[ ! -f "$script_path" ]]; then
error "$script_name — not found at $script_path"
FAILED+=("$script_name")
echo ""
continue
fi
if [[ ! -x "$script_path" ]]; then
warn "$script_name — not executable, fixing..."
chmod +x "$script_path" || {
error "$script_name — chmod +x failed"
FAILED+=("$script_name")
echo ""
continue
}
fi
if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — would run: $script_name${extra_args:+ ${extra_args[*]}}"
PASSED+=("$script_name")
echo ""
continue
fi
local_args=()
[[ "$VERBOSE" == true ]] && local_args+=("--log")
if bash "$script_path" "${extra_args[@]}" "${local_args[@]}"; then
echo "$script_name — done ✅"
PASSED+=("$script_name")
else
warn "$script_name — failed (exit $?) — continuing to next step"
FAILED+=("$script_name")
fi
echo "" echo ""
done done
@@ -307,22 +271,22 @@ fi
echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━" echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━"
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}" [[ ${#JOB_PASS[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${JOB_PASS[*]}"
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" [[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
echo "" echo ""
if [[ "$DRY_RUN" == true ]]; then if [[ "$DRY_RUN" == true ]]; then
warn "DRY RUN — no changes made" warn "DRY RUN — no changes made"
elif [[ ${#FAILED[@]} -eq 0 ]]; then elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
echo "$ICON_DONE Status: all $STEP step(s) complete ✅" echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \ notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
"Monthly Maintenance" "normal" "Monthly Maintenance" "normal"
else else
warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}" warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \ notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
"Monthly Maintenance" "warning" "Monthly Maintenance" "warning"
fi fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
[[ ${#FAILED[@]} -gt 0 ]] && exit 1 [[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
exit 0 exit 0
+22 -33
View File
@@ -39,9 +39,12 @@
# OPERATIONAL SAFEGUARDS # OPERATIONAL SAFEGUARDS
# ============================================================================================== # ==============================================================================================
# #
# Root check — required for the docker/system reads used in the report
# acquire_lock — prevents overlapping weekly runs
# detect_hosts() — MY_ID in banner and summary # detect_hosts() — MY_ID in banner and summary
# Non-fatal steps — a failed script is logged; remaining scripts still run # Non-fatal steps — a failed script is logged; remaining scripts still run
# Flag pass-through — --dry-run and --log forwarded to all child scripts # Flag pass-through — --dry-run and --log forwarded to all child scripts
# notify() on failure — pushed only outside --dry-run, matching the runtime-mode contract below
# #
# ============================================================================================== # ==============================================================================================
# CONFIGURATION # CONFIGURATION
@@ -70,12 +73,22 @@
# ============================================================================================== # ==============================================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "$SCRIPT_DIR/../load_config.sh" source "$ECOSYSTEM_ROOT/load_config.sh"
SCRIPTS_ROOT="$SCRIPT_DIR/.."
parse_args "$@" parse_args "$@"
# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
error "Must be run as root"
exit 1
fi
acquire_lock
detect_hosts detect_hosts
# ============================================================================================== # ==============================================================================================
@@ -84,35 +97,6 @@ detect_hosts
JOB_PASS=() JOB_PASS=()
JOB_FAIL=() JOB_FAIL=()
run_job() {
local script_entry="$1"
local extra_dry=""
local extra_log=""
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
[[ "$ENABLE_LOGGING" == true ]] && extra_log="--log"
read -r -a script_args <<< "$script_entry"
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
local script_name
script_name=$(basename "${script_args[0]}")
local extra_args=("${script_args[@]:1}")
if [[ ! -f "$script_path" ]]; then
error "$script_name — not found at $script_path"
JOB_FAIL+=("$script_name")
return 1
fi
log "Running: $script_name ${extra_args[*]}"
if bash "$script_path" "${extra_args[@]}" $extra_dry $extra_log; then
echo "$script_name — done ✅"
JOB_PASS+=("$script_name")
else
error "$script_name — failed (exit $?)"
JOB_FAIL+=("$script_name")
fi
}
# ============================================================================================== # ==============================================================================================
# ━━━ Status ━━━ # ━━━ Status ━━━
# ============================================================================================== # ==============================================================================================
@@ -158,7 +142,7 @@ fi
for script_entry in "${COFFEE_REPORT_SCRIPTS[@]}"; do for script_entry in "${COFFEE_REPORT_SCRIPTS[@]}"; do
[[ -z "$script_entry" ]] && continue [[ -z "$script_entry" ]] && continue
echo "" echo ""
run_job "$script_entry" run_orch_child "$script_entry"
done done
# ============================================================================================== # ==============================================================================================
@@ -176,5 +160,10 @@ if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
echo "❌ Failed: ${JOB_FAIL[*]}" echo "❌ Failed: ${JOB_FAIL[*]}"
fi fi
if [[ ${#JOB_FAIL[@]} -gt 0 && "$DRY_RUN" != true ]]; then
notify "Sunday coffee report had failures on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
"Sunday Morning Coffee Report" "warning"
fi
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1 [[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
exit 0 exit 0
+60 -38
View File
@@ -5,14 +5,17 @@
# #
# PURPOSE # PURPOSE
# ───────────────────────────────────────────────────────────────────────────── # ─────────────────────────────────────────────────────────────────────────────
# Runs transcode_cleanup.sh then transcode_manager.sh in the correct order. # Runs TRANSCODE_MANAGEMENT_SCRIPTS in order each cron cycle — this is the
# Replaces individual cron entries for each — this is the single cron entry. # single cron entry replacing individual entries for each child script.
# Schedule: */7 * * * * (every 7 minutes via User Scripts plugin) # Schedule: */7 * * * * (every 7 minutes)
# #
# ============================================================================================== # ==============================================================================================
# OPERATIONAL MODEL # OPERATIONAL MODEL
# ============================================================================================== # ==============================================================================================
# #
# Order driven by TRANSCODE_MANAGEMENT_SCRIPTS in master.conf.
# Default: transcode_cleanup.sh → transcode_manager.sh
#
# transcode_cleanup.sh # transcode_cleanup.sh
# Removes aged segment files not open by any process. Uses lsof for O(1) # Removes aged segment files not open by any process. Uses lsof for O(1)
# per-file active check. Triggers flip-back to ramdisk after cleanup if # per-file active check. Triggers flip-back to ramdisk after cleanup if
@@ -36,6 +39,8 @@
# Stale segment files from ended sessions inflate the ramdisk usage reading # Stale segment files from ended sessions inflate the ramdisk usage reading
# and trigger unnecessary SSD flips even when active sessions would fit on # and trigger unnecessary SSD flips even when active sessions would fit on
# the ramdisk. Cleanup runs first so the manager measures real current usage. # the ramdisk. Cleanup runs first so the manager measures real current usage.
# Order is config-driven (TRANSCODE_MANAGEMENT_SCRIPTS) but this dependency
# is real — reordering the array changes what the manager measures.
# #
# Delegated Logging # Delegated Logging
# This orchestrator does not write its own log — transcode_manager.sh owns # This orchestrator does not write its own log — transcode_manager.sh owns
@@ -48,8 +53,9 @@
# Root check — mount and docker operations require root # Root check — mount and docker operations require root
# acquire_lock — prevents concurrent 7-minute cycles overlapping # acquire_lock — prevents concurrent 7-minute cycles overlapping
# detect_hosts() — aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_WARN_GB per host # detect_hosts() — aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_WARN_GB per host
# --dry-run — passed through to both child scripts # --dry-run — passed through to every script in TRANSCODE_MANAGEMENT_SCRIPTS
# Exit code — worst exit code of both scripts returned to cron # Exit code — worst exit code across all scripts returned to cron
# notify() — pushed on failure, skipped in --dry-run
# #
# ============================================================================================== # ==============================================================================================
# CONFIGURATION # CONFIGURATION
@@ -57,6 +63,7 @@
# #
# master.conf # master.conf
# #
# TRANSCODE_MANAGEMENT_SCRIPTS — scripts to run, in order
# TRANSCODE_DAILY_LOG — daily stats log (written by transcode_manager.sh) # TRANSCODE_DAILY_LOG — daily stats log (written by transcode_manager.sh)
# TRANSCODE_LOG_RETENTION — days to keep (trimmed by manager) # TRANSCODE_LOG_RETENTION — days to keep (trimmed by manager)
# TRANSCODE_STATE_FILE — current state (ramdisk_used, flip_count, etc.) # TRANSCODE_STATE_FILE — current state (ramdisk_used, flip_count, etc.)
@@ -70,13 +77,13 @@
# Normal run (every 7 minutes via cron). # Normal run (every 7 minutes via cron).
# #
# transcode_management.sh --dry-run # transcode_management.sh --dry-run
# Preview without changes (passed to both child scripts). # Preview without changes (passed to every script in TRANSCODE_MANAGEMENT_SCRIPTS).
# #
# transcode_management.sh --status # transcode_management.sh --status
# Show configuration and current state. # Show configuration and current state.
# #
# transcode_management.sh --log # transcode_management.sh --log
# Verbose output from both child scripts. # Verbose output from every child script.
# #
# ============================================================================================== # ==============================================================================================
@@ -86,9 +93,6 @@ source "$SCRIPT_DIR/../load_config.sh"
parse_args "$@" parse_args "$@"
CLEANUP_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_cleanup.sh"
MANAGER_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_manager.sh"
# ============================================================================================== # ==============================================================================================
# ━━━ Setup ━━━ # ━━━ Setup ━━━
# ============================================================================================== # ==============================================================================================
@@ -122,12 +126,15 @@ if [[ "$SHOW_STATUS" == true ]]; then
echo "$ICON_TIME Schedule: every 7 minutes" echo "$ICON_TIME Schedule: every 7 minutes"
echo "" echo ""
echo "━━━ Child Scripts ━━━" echo "━━━ Child Scripts ━━━"
[[ -f "$CLEANUP_SCRIPT" ]] && \ for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
echo " $ICON_SUCCESS transcode_cleanup.sh — found" || \ _script_path="$SCRIPT_DIR/../$_entry"
echo " $ICON_ERROR transcode_cleanup.sh — NOT FOUND at $CLEANUP_SCRIPT" _script_name=$(basename "$_entry")
[[ -f "$MANAGER_SCRIPT" ]] && \ if [[ -f "$_script_path" ]]; then
echo " $ICON_SUCCESS transcode_manager.sh — found" || \ echo " $ICON_SUCCESS $_script_name — found"
echo " $ICON_ERROR transcode_manager.sh — NOT FOUND at $MANAGER_SCRIPT" else
echo " $ICON_ERROR $_script_name — NOT FOUND at $_script_path"
fi
done
echo "" echo ""
echo "━━━ Daily Log ━━━" echo "━━━ Daily Log ━━━"
if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]] && [[ -s "$TRANSCODE_DAILY_LOG" ]]; then if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]] && [[ -s "$TRANSCODE_DAILY_LOG" ]]; then
@@ -154,15 +161,13 @@ fi
# ============================================================================================== # ==============================================================================================
# ━━━ Validate Child Scripts ━━━ # ━━━ Validate Child Scripts ━━━
# ============================================================================================== # ==============================================================================================
if [[ ! -f "$CLEANUP_SCRIPT" ]]; then for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
error "transcode_cleanup.sh not found: $CLEANUP_SCRIPT" _script_path="$SCRIPT_DIR/../$_entry"
exit 1 if [[ ! -f "$_script_path" ]]; then
fi error "$(basename "$_entry") not found: $_script_path"
if [[ ! -f "$MANAGER_SCRIPT" ]]; then
error "transcode_manager.sh not found: $MANAGER_SCRIPT"
exit 1 exit 1
fi fi
done
# ============================================================================================== # ==============================================================================================
# ━━━ Pre-run State Snapshot ━━━ # ━━━ Pre-run State Snapshot ━━━
@@ -179,29 +184,46 @@ else
fi fi
# ============================================================================================== # ==============================================================================================
# ━━━ Run Cleanup ━━━ # ━━━ Run Scripts ━━━
# ============================================================================================== # ==============================================================================================
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run — no flag here
# suppresses that; it owns the log write for this cycle regardless of position ✅
DRY_FLAG="" DRY_FLAG=""
[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run" [[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run"
_cleanup_start=$(date +%s) WORST_EXIT=0
bash "$CLEANUP_SCRIPT" $DRY_FLAG PASS_COUNT=0
CLEANUP_EXIT=$? FAIL_NAMES=()
log "cleanup: $(format_duration $(( $(date +%s) - _cleanup_start ))) (exit $CLEANUP_EXIT)" for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
_script_path="$SCRIPT_DIR/../$_entry"
_script_name=$(basename "$_entry" .sh)
_start=$(date +%s)
bash "$_script_path" $DRY_FLAG
_exit=$?
log "$_script_name: $(format_duration $(( $(date +%s) - _start ))) (exit $_exit)"
if [[ "$_exit" -ne 0 ]]; then
WORST_EXIT=1
FAIL_NAMES+=("$_script_name")
else
PASS_COUNT=$(( PASS_COUNT + 1 ))
fi
done
# ============================================================================================== # ==============================================================================================
# ━━━ Run Manager ━━━ # ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
# ============================================================================================== # ==============================================================================================
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run if [[ "$WORST_EXIT" -eq 0 ]]; then
# No --no-log flag here — manager owns the log write for this cycle ✅ echo "$ICON_SUCCESS Transcode cycle — $PASS_COUNT/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
_manager_start=$(date +%s) else
bash "$MANAGER_SCRIPT" $DRY_FLAG error "Transcode cycle — failed: ${FAIL_NAMES[*]}"
MANAGER_EXIT=$? if [[ "$DRY_RUN" != true ]]; then
log "manager: $(format_duration $(( $(date +%s) - _manager_start ))) (exit $MANAGER_EXIT)" notify "Transcode management failure on $(hostname) ($MY_ID) — ${FAIL_NAMES[*]}" \
"Transcode Management" "warning"
fi
fi
# ============================================================================================== # ==============================================================================================
# ━━━ Exit ━━━ # ━━━ Exit ━━━
# ============================================================================================== # ==============================================================================================
# Return worst exit code — caller knows if either script failed # Return worst exit code — caller knows if any script failed
[[ "$CLEANUP_EXIT" -ne 0 || "$MANAGER_EXIT" -ne 0 ]] && exit 1 exit "$WORST_EXIT"
exit 0
+9 -4
View File
@@ -193,7 +193,7 @@ run_watchdog() {
local extra_args=() local extra_args=()
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run") [[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
[[ "$VERBOSE" == true ]] && extra_args+=("--log") [[ "$ENABLE_LOGGING" == true ]] && extra_args+=("--log")
local _ws local _ws
_ws=$(date +%s) _ws=$(date +%s)
@@ -234,18 +234,23 @@ if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
fi fi
# ============================================================================================== # ==============================================================================================
# ━━━ Summary — only shown on failures or --log ━━━ # ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
# ============================================================================================== # ==============================================================================================
if [[ "${#FAIL[@]}" -gt 0 || "$VERBOSE" == true ]]; then if [[ "${#FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
echo "" echo ""
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID$(date '+%H:%M:%S') ━━━━━" echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID$(date '+%H:%M:%S') ━━━━━"
for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done
for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done
echo "$ICON_TIME Duration: $(format_duration $DURATION)" echo "$ICON_TIME Duration: $(format_duration $DURATION)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
else
echo "$ICON_DONE Watchdog cycle — ${#PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
fi
if [[ "${#FAIL[@]}" -gt 0 ]]; then if [[ "${#FAIL[@]}" -gt 0 ]]; then
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \ notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
"Watchdog Orchestrator" "warning" "Watchdog Orchestrator" "warning"
exit 1
fi fi
fi
exit 0
+3 -32
View File
@@ -84,8 +84,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../load_config.sh" source "$SCRIPT_DIR/../load_config.sh"
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
SCRIPTS_ROOT="$SCRIPT_DIR/.." RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"
parse_args "$@" parse_args "$@"
@@ -123,35 +123,6 @@ read -r -a MAINTENANCE_CONTAINERS <<< \
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates" [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"
# ── Helper — run a post-sync maintenance script ────────────────────────────────────────────────
run_job() {
local script_entry="$1"
local extra_dry=""
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
read -r -a script_args <<< "$script_entry"
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
local script_name
script_name=$(basename "${script_args[0]}")
local extra_args=("${script_args[@]:1}")
if [[ ! -f "$script_path" ]]; then
error "$script_name — not found at $script_path"
JOB_FAIL+=("$script_name")
return 1
fi
log "Running: $script_name ${extra_args[*]}"
# shellcheck disable=SC2086
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
echo "$script_name — done ✅"
JOB_PASS+=("$script_name ${extra_args[*]}")
else
error "$script_name — failed (exit $?)"
JOB_FAIL+=("$script_name ${extra_args[*]}")
fi
}
# ============================================================================================== # ==============================================================================================
# ━━━ Status ━━━ # ━━━ Status ━━━
# ============================================================================================== # ==============================================================================================
@@ -390,7 +361,7 @@ if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
[[ -z "$script_entry" ]] && continue [[ -z "$script_entry" ]] && continue
echo "" echo ""
run_job "$script_entry" run_orch_child "$script_entry"
done done
fi fi
+58
View File
@@ -1537,6 +1537,64 @@ _release_rsync_on_exit() {
fi fi
} }
# ==============================================================================================
# ── ORCHESTRATOR CHILD EXECUTION ──────────────────────────────────────────────────────────────
# ==============================================================================================
# Shared child-script runner for Orchestrators/*.sh. Replaces the run_job()/run_watchdog()
# copies that used to be hand-duplicated (with small inconsistencies) into each orchestrator —
# one implementation now, one place to fix a bug or extend behavior (e.g. per-child logging).
#
# Usage — caller declares its own tracking arrays before the loop:
# JOB_PASS=()
# JOB_FAIL=()
# for entry in "${SOME_SCRIPTS[@]}"; do
# run_orch_child "$entry"
# done
#
# $entry is "relative/path.sh [extra args...]" — the same format every master.conf
# script-list array already uses. Resolved against $ECOSYSTEM_ROOT, which the caller
# must set before calling this (the absolute repo root — "$(cd "$SCRIPT_DIR/.." && pwd)").
#
# --dry-run / --log are threaded down automatically from $DRY_RUN / $ENABLE_LOGGING —
# never $VERBOSE, which nothing in this codebase ever assigns.
run_orch_child() {
local entry="$1"
local script_args script_path script_name label extra_args run_args
read -r -a script_args <<< "$entry"
script_path="$ECOSYSTEM_ROOT/${script_args[0]}"
script_name=$(basename "${script_args[0]}")
extra_args=("${script_args[@]:1}")
label="$script_name"
[[ -n "${extra_args[*]}" ]] && label="$script_name ${extra_args[*]}"
if [[ ! -f "$script_path" ]]; then
error "$script_name — not found at $script_path"
JOB_FAIL+=("$label")
return 1
fi
[[ ! -x "$script_path" ]] && chmod +x "$script_path"
run_args=("${extra_args[@]}")
[[ "$DRY_RUN" == true ]] && run_args+=("--dry-run")
[[ "$ENABLE_LOGGING" == true ]] && run_args+=("--log")
local _start _ec
_start=$(date +%s)
log "Running: $label"
if bash "$script_path" "${run_args[@]}"; then
log "$script_name — done in $(format_duration $(( $(date +%s) - _start )))"
JOB_PASS+=("$label")
return 0
else
_ec=$?
error "$label — failed (exit $_ec, $(format_duration $(( $(date +%s) - _start ))))"
JOB_FAIL+=("$label")
return 1
fi
}
# ============================================================================================== # ==============================================================================================
# ── PATH TRANSLATION ────────────────────────────────────────────────────────────────────────── # ── PATH TRANSLATION ──────────────────────────────────────────────────────────────────────────
# ============================================================================================== # ==============================================================================================