From 2a062e51409b24a036cc9033dfb75aa41dd6d86e Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Fri, 3 Jul 2026 10:57:52 -0400 Subject: [PATCH] Standardize orchestrator child-script execution and logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself. --- Configurations/master.conf | 10 ++ Deployment/master.conf.template | 10 ++ Orchestrators/README-Orchestrators.md | 43 ++++--- Orchestrators/array_started.sh | 37 +++--- Orchestrators/array_stopping.sh | 51 ++------ Orchestrators/critical_sync_maintenance.sh | 45 +++---- Orchestrators/daily_sync_maintenance.sh | 39 +----- .../intermediate_sync_maintenance.sh | 113 ++++++++---------- Orchestrators/monthly_maintenance.sh | 54 ++------- Orchestrators/sunday_morning_coffee_report.sh | 55 ++++----- Orchestrators/transcode_management.sh | 102 +++++++++------- Orchestrators/watchdog_orchestrator.sh | 21 ++-- Orchestrators/weekly_sync_maintenance.sh | 35 +----- common.sh | 58 +++++++++ 14 files changed, 303 insertions(+), 370 deletions(-) diff --git a/Configurations/master.conf b/Configurations/master.conf index ec48b61..2bbc8ce 100644 --- a/Configurations/master.conf +++ b/Configurations/master.conf @@ -320,6 +320,16 @@ "Docker_Essentials/docker_container_stop.sh" # stop all containers last ) +# ━━━ Transcode Management ━━━ +# transcode_management.sh runs TRANSCODE_MANAGEMENT_SCRIPTS in order each cron cycle. +# Schedule: */7 * * * * (every 7 minutes) +# Order matters — cleanup first so the manager measures real current ramdisk usage, +# not usage inflated by stale segment files from ended sessions. + TRANSCODE_MANAGEMENT_SCRIPTS=( + "Transcodes/transcode_cleanup.sh" # remove aged segment files before usage is measured + "Transcodes/transcode_manager.sh" # flip ramdisk/SSD symlink, write daily log entry + ) + # ━━━ Watchdog Orchestrator ━━━ # watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. # Schedule: */15 * * * * (every 15 minutes) diff --git a/Deployment/master.conf.template b/Deployment/master.conf.template index 2bdcfc1..a365a2a 100644 --- a/Deployment/master.conf.template +++ b/Deployment/master.conf.template @@ -317,6 +317,16 @@ "Docker_Essentials/docker_container_stop.sh" # stop all containers last ) +# ━━━ Transcode Management ━━━ +# transcode_management.sh runs TRANSCODE_MANAGEMENT_SCRIPTS in order each cron cycle. +# Schedule: */7 * * * * (every 7 minutes) +# Order matters — cleanup first so the manager measures real current ramdisk usage, +# not usage inflated by stale segment files from ended sessions. + TRANSCODE_MANAGEMENT_SCRIPTS=( + "Transcodes/transcode_cleanup.sh" # remove aged segment files before usage is measured + "Transcodes/transcode_manager.sh" # flip ramdisk/SSD symlink, write daily log entry + ) + # ━━━ Watchdog Orchestrator ━━━ # watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle. # Schedule: */15 * * * * (every 15 minutes) diff --git a/Orchestrators/README-Orchestrators.md b/Orchestrators/README-Orchestrators.md index ebb94f4..abd732c 100644 --- a/Orchestrators/README-Orchestrators.md +++ b/Orchestrators/README-Orchestrators.md @@ -942,41 +942,40 @@ array_started.sh ## ━━━ ADDING A NEW ORCHESTRATOR ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ If you find yourself running 3+ related scripts on the same schedule, wrap them -in a new orchestrator. Model directly on `media_management.sh` which has the +in a new orchestrator. Model directly on `daily_sync_maintenance.sh` which has the complete pattern — dry-run passthrough, status display, pass/fail tracking, summary. +Child-script execution goes through the shared `run_orch_child()` helper in +`common.sh` — never hand-roll a per-file `run_job()` loop. It resolves the entry +against `$ECOSYSTEM_ROOT`, threads `--dry-run`/`--log` from `$DRY_RUN`/`$ENABLE_LOGGING` +automatically (never `$VERBOSE` — nothing in this codebase assigns it), and tracks +into `JOB_PASS`/`JOB_FAIL` arrays the caller declares. + ```bash # Minimal skeleton — the full pattern in its simplest form: #!/bin/bash SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/../load_config.sh" +ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +source "$ECOSYSTEM_ROOT/load_config.sh" parse_args "$@" -SCRIPTS_ROOT="$SCRIPT_DIR/.." -PASS=() -FAIL=() +JOB_PASS=() +JOB_FAIL=() # Read job list from master.conf — never hardcode jobs in the orchestrator for script_entry in "${MY_MAINTENANCE_JOBS[@]:-}"; do [[ -z "$script_entry" ]] && continue - - read -r -a parts <<< "$script_entry" - script_path="$SCRIPTS_ROOT/${parts[0]}" - script_name=$(basename "${parts[0]}") - extra_args=("${parts[@]:1}") - - [[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run") - - if bash "$script_path" "${extra_args[@]}"; then - PASS+=("$script_name") - else - FAIL+=("$script_name") - fi + run_orch_child "$script_entry" done -# One summary — one notification -echo "Passed: ${#PASS[@]} Failed: ${#FAIL[@]}" -[[ ${#FAIL[@]} -gt 0 ]] && \ - notify "My maintenance failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \ +# One summary — one notification. On a frequent (sub-daily) cadence, keep the +# healthy path to a single line and reserve the full breakdown for failure/--log — +# see watchdog_orchestrator.sh or transcode_management.sh for that split. +echo "Passed: ${#JOB_PASS[@]} Failed: ${#JOB_FAIL[@]}" +if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then + notify "My maintenance failed on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \ "My Orchestrator" "warning" + exit 1 +fi +exit 0 ``` \ No newline at end of file diff --git a/Orchestrators/array_started.sh b/Orchestrators/array_started.sh index 8e92e26..6ca352c 100755 --- a/Orchestrators/array_started.sh +++ b/Orchestrators/array_started.sh @@ -163,9 +163,8 @@ log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..." echo "" START=$(date +%s) -LAUNCHED=0 -FAILED=0 -FAILED_SCRIPTS=() +JOB_PASS=() +JOB_FAIL=() for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do [[ -z "$relative_path" ]] && continue @@ -177,8 +176,7 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do if [[ ! -f "$SCRIPT_PATH" ]]; then error "$SCRIPT_NAME — not found" error " Expected: $SCRIPT_PATH" - (( FAILED++ )) - FAILED_SCRIPTS+=("$SCRIPT_NAME") + JOB_FAIL+=("$SCRIPT_NAME") continue fi @@ -187,15 +185,14 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do warn "$SCRIPT_NAME — not executable, fixing..." chmod +x "$SCRIPT_PATH" || { error "$SCRIPT_NAME — chmod +x failed" - (( FAILED++ )) - FAILED_SCRIPTS+=("$SCRIPT_NAME") + JOB_FAIL+=("$SCRIPT_NAME") continue } fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would launch: $SCRIPT_NAME" - (( LAUNCHED++ )) + JOB_PASS+=("$SCRIPT_NAME") continue fi @@ -209,19 +206,18 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do if kill -0 "$PID" 2>/dev/null; then # Still running → continuous script log "$SCRIPT_NAME — running (PID $PID) ✅" - (( LAUNCHED++ )) + JOB_PASS+=("$SCRIPT_NAME") else # Exited — check if one-shot success or failure wait "$PID" EXIT_CODE=$? if [[ "$EXIT_CODE" -eq 0 ]]; then echo "$SCRIPT_NAME — completed (one-shot) ✅" - (( LAUNCHED++ )) + JOB_PASS+=("$SCRIPT_NAME") else error "$SCRIPT_NAME — exited with code $EXIT_CODE" error " Path: $SCRIPT_PATH" - (( FAILED++ )) - FAILED_SCRIPTS+=("$SCRIPT_NAME") + JOB_FAIL+=("$SCRIPT_NAME") fi fi @@ -235,18 +231,21 @@ END=$(date +%s) echo "" echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" -echo "$ICON_SUCCESS Launched: $LAUNCHED" -[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR Failed: $FAILED — ${FAILED_SCRIPTS[*]}" +echo "$ICON_SUCCESS Launched: ${#JOB_PASS[@]}" +[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#JOB_FAIL[@]} — ${JOB_FAIL[*]}" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no scripts launched" -elif [[ "$FAILED" -gt 0 ]]; then - warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}" - notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \ +elif [[ ${#JOB_FAIL[@]} -gt 0 ]]; then + warn "Status: ${#JOB_FAIL[@]} script(s) failed — ${JOB_FAIL[*]}" + notify "Array start on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} script(s) failed: ${JOB_FAIL[*]}" \ "Array Start" "warning" else - echo "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅" + echo "$ICON_DONE Status: all ${#JOB_PASS[@]} script(s) launched ✅" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Orchestrators/array_stopping.sh b/Orchestrators/array_stopping.sh index a892234..e4f3f67 100755 --- a/Orchestrators/array_stopping.sh +++ b/Orchestrators/array_stopping.sh @@ -134,8 +134,8 @@ echo "$ICON_GEAR Running ${#ARRAY_STOP_SCRIPTS[@]} stop script(s) sequentially.. echo "" START=$(date +%s) -PASSED=() -FAILED=() +JOB_PASS=() +JOB_FAIL=() STEP=0 for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do @@ -143,44 +143,11 @@ for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do (( STEP++ )) read -r -a parts <<< "$entry" - script_path="$ECOSYSTEM_ROOT/${parts[0]}" script_name=$(basename "${parts[0]}") extra_args=("${parts[@]:1}") echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━" - - if [[ ! -f "$script_path" ]]; then - error "$script_name — not found at $script_path" - FAILED+=("$script_name") - echo "" - continue - fi - - if [[ ! -x "$script_path" ]]; then - warn "$script_name — not executable, fixing..." - chmod +x "$script_path" || { - error "$script_name — chmod +x failed" - FAILED+=("$script_name") - echo "" - continue - } - fi - - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would run: $script_name ${extra_args[*]}" - PASSED+=("$script_name") - echo "" - continue - fi - - if bash "$script_path" "${extra_args[@]}"; then - echo "$script_name — done ✅" - PASSED+=("$script_name") - else - warn "$script_name — failed (exit $?) — continuing to next step" - FAILED+=("$script_name") - fi - + run_orch_child "$entry" echo "" done @@ -192,22 +159,22 @@ END=$(date +%s) echo "━━━━━ $ICON_SUMMARY ARRAY STOP SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" -[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}" -[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" +[[ ${#JOB_PASS[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${JOB_PASS[*]}" +[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}" echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" -elif [[ ${#FAILED[@]} -eq 0 ]]; then +elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then echo "$ICON_DONE Status: all $STEP step(s) complete ✅" notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \ "Array Stop" "normal" else - warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}" - notify "Array stop on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \ + warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}" + notify "Array stop on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \ "Array Stop" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1 exit 0 diff --git a/Orchestrators/critical_sync_maintenance.sh b/Orchestrators/critical_sync_maintenance.sh index 68b3833..e8bde8b 100755 --- a/Orchestrators/critical_sync_maintenance.sh +++ b/Orchestrators/critical_sync_maintenance.sh @@ -80,6 +80,7 @@ # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "$SCRIPT_DIR/../load_config.sh" @@ -199,29 +200,12 @@ fi # ============================================================================================== # ━━━ Critical Maintenance Scripts ━━━ # ============================================================================================== -if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then - for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do - [[ -z "$script_entry" || "$script_entry" == \#* ]] && continue - - SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}" - SCRIPT_ARGS="${script_entry#* }" - [[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS="" - [[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run" - - SCRIPT_NAME=$(basename "$SCRIPT_PATH") - - if [[ ! -f "$SCRIPT_PATH" ]]; then - warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping" - continue - fi - - log "Running: $SCRIPT_NAME" - bash "$SCRIPT_PATH" $SCRIPT_ARGS - EXIT_CODE=$? - [[ "$EXIT_CODE" -ne 0 ]] && \ - warn "$SCRIPT_NAME exited with code $EXIT_CODE" - done -fi +JOB_PASS=() +JOB_FAIL=() +for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do + [[ -z "$script_entry" || "$script_entry" == \#* ]] && continue + run_orch_child "$script_entry" +done # ============================================================================================== # ━━━ Partnership Check ━━━ @@ -247,20 +231,23 @@ fi END=$(date +%s) DURATION=$(format_duration $(( END - START ))) -# Silent when healthy — only show summary if there were failures or notable events -if [[ ${#FAIL[@]} -gt 0 ]]; then +TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} )) + +# Minimal one-liner when healthy — 30-min cadence, keep it quiet. Full detail on failure. +if [[ "$TOTAL_FAIL" -gt 0 ]]; then echo "" echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $DURATION" - [[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}" - echo "$ICON_ERROR Failed: ${FAIL[*]}" + [[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}" + [[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed shares: ${FAIL[*]}" + [[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \ + notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]} ${JOB_FAIL[*]}" \ "Critical Sync" "warning" exit 1 else - echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)" + echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s), ${#JOB_PASS[@]} job(s)" fi exit 0 \ No newline at end of file diff --git a/Orchestrators/daily_sync_maintenance.sh b/Orchestrators/daily_sync_maintenance.sh index 0eead69..edc888f 100755 --- a/Orchestrators/daily_sync_maintenance.sh +++ b/Orchestrators/daily_sync_maintenance.sh @@ -103,8 +103,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" -RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" -SCRIPTS_ROOT="$SCRIPT_DIR/.." +ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh" parse_args "$@" @@ -184,35 +184,6 @@ for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do fi done -# Helper — run a maintenance script, track pass/fail -run_job() { - local script_entry="$1" - local extra_dry="" - [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run" - - read -r -a script_args <<< "$script_entry" - local script_path="$SCRIPTS_ROOT/${script_args[0]}" - local script_name - script_name=$(basename "${script_args[0]}") - local extra_args=("${script_args[@]:1}") - - if [[ ! -f "$script_path" ]]; then - error "$script_name — not found at $script_path" - JOB_FAIL+=("$script_name") - return 1 - fi - - log "Running: $script_name ${extra_args[*]}" - # shellcheck disable=SC2086 - if bash "$script_path" "${extra_args[@]}" $extra_dry; then - echo "$script_name — done ✅" - JOB_PASS+=("$script_name ${extra_args[*]}") - else - error "$script_name — failed (exit $?)" - JOB_FAIL+=("$script_name ${extra_args[*]}") - fi -} - WINDOW_START=$(date +%s) JOB_PASS=() JOB_FAIL=() @@ -230,7 +201,7 @@ if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then echo "" echo "━━━ $ICON_GIT Pre-sync ━━━" for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do - run_job "$script_entry" + run_orch_child "$script_entry" done fi @@ -240,7 +211,7 @@ fi echo "" echo "━━━ $ICON_SYNC Arr Sync ━━━" -ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh" +ARR_SYNC_SCRIPT="$ECOSYSTEM_ROOT/Arrs_Stack/arr_sync.sh" if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then echo "ARR_SYNC_ENABLED=false — skipping" elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then @@ -336,7 +307,7 @@ if [[ ${#POST_SYNC_SCRIPTS[@]} -gt 0 ]]; then echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━" for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do echo "" - run_job "$script_entry" + run_orch_child "$script_entry" done fi diff --git a/Orchestrators/intermediate_sync_maintenance.sh b/Orchestrators/intermediate_sync_maintenance.sh index 2d71b9b..70bad72 100755 --- a/Orchestrators/intermediate_sync_maintenance.sh +++ b/Orchestrators/intermediate_sync_maintenance.sh @@ -50,7 +50,7 @@ # check_connectivity — verified before any rsync (skipped if no shares) # check_remote_rootfs — aborts rsync if remote rootfs nearly full # Non-fatal jobs — a failed arr_sync warns but does not block rsync or artwork fetch -# Silent on success — runs 4x/day, only failures warrant notification +# Minimal on success — runs 6x/day; full breakdown only on failure or --log # # ============================================================================================== # CONFIGURATION @@ -88,8 +88,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" -RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" -SCRIPTS_ROOT="$SCRIPT_DIR/.." +ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh" parse_args "$@" @@ -109,35 +109,6 @@ resolve_remote_ip [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" -# ── Helper — run a maintenance job, track pass/fail ─────────────────────────────────────────── -run_job() { - local script_entry="$1" - local extra_dry="" - [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run" - - read -r -a script_args <<< "$script_entry" - local script_path="$SCRIPTS_ROOT/${script_args[0]}" - local script_name - script_name=$(basename "${script_args[0]}") - local extra_args=("${script_args[@]:1}") - - if [[ ! -f "$script_path" ]]; then - error "$script_name — not found at $script_path" - JOB_FAIL+=("$script_name") - return 1 - fi - - log "Running: $script_name ${extra_args[*]}" - # shellcheck disable=SC2086 - if bash "$script_path" "${extra_args[@]}" $extra_dry; then - echo "$script_name — done ✅" - JOB_PASS+=("$script_name ${extra_args[*]}") - else - warn "$script_name — failed (exit $?)" - JOB_FAIL+=("$script_name ${extra_args[*]}") - fi -} - # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== @@ -187,7 +158,7 @@ echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H echo "" echo "━━━ $ICON_GEAR Conf Pull ━━━" -CONF_SYNC_SCRIPT="$SCRIPTS_ROOT/System_Essentials/conf_sync.sh" +CONF_SYNC_SCRIPT="$ECOSYSTEM_ROOT/System_Essentials/conf_sync.sh" if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then warn "conf_sync.sh not found — skipping partner conf refresh" else @@ -209,7 +180,7 @@ fi echo "" echo "━━━ $ICON_SYNC Arr Sync ━━━" -ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh" +ARR_SYNC_SCRIPT="$ECOSYSTEM_ROOT/Arrs_Stack/arr_sync.sh" if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then echo "ARR_SYNC_ENABLED=false — skipping" elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then @@ -306,7 +277,7 @@ if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then for script_entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do [[ -z "$script_entry" ]] && continue echo "" - run_job "$script_entry" + run_orch_child "$script_entry" done fi @@ -315,47 +286,57 @@ WINDOW_END=$(date +%s) # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== -echo "" -echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━" -echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" -echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')" -echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))" -echo "" - -if [[ "$SHARE_COUNT" -gt 0 ]]; then - echo "$ICON_SYNC Shares ($SHARE_COUNT):" - for entry in "${SHARE_TIMES[@]}"; do - sname="${entry%%:*}" - sdur="${entry##*:}" - if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then - echo " $ICON_ERROR $sname — $(format_duration "$sdur")" - else - echo " $ICON_DONE $sname — $(format_duration "$sdur")" - fi - done - echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT" - echo "" -fi - -if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then - echo "$ICON_GEAR Jobs:" - for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done - for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done - echo "" -fi - TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} )) +# Full breakdown on failure or --log; minimal one-liner otherwise (4-hour cadence — keep it quiet). +SHOW_FULL=false +[[ "$TOTAL_FAIL" -gt 0 || "$ENABLE_LOGGING" == true ]] && SHOW_FULL=true + +if [[ "$SHOW_FULL" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')" + echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))" + echo "" + + if [[ "$SHARE_COUNT" -gt 0 ]]; then + echo "$ICON_SYNC Shares ($SHARE_COUNT):" + for entry in "${SHARE_TIMES[@]}"; do + sname="${entry%%:*}" + sdur="${entry##*:}" + if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then + echo " $ICON_ERROR $sname — $(format_duration "$sdur")" + else + echo " $ICON_DONE $sname — $(format_duration "$sdur")" + fi + done + echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT" + echo "" + fi + + if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then + echo "$ICON_GEAR Jobs:" + for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done + for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done + echo "" + fi +fi + if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ "$TOTAL_FAIL" -eq 0 ]]; then - echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced" + if [[ "$SHOW_FULL" == true ]]; then + echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced" + else + echo "$ICON_DONE Intermediate sync — ${#JOB_PASS[@]} job(s), ${#PASS[@]}/$SHARE_COUNT share(s) ($(format_duration $(( WINDOW_END - WINDOW_START ))))" + fi else warn "Status: $TOTAL_FAIL failure(s)" notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \ "Intermediate Sync" "warning" fi -echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +[[ "$SHOW_FULL" == true ]] && echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" [[ "$TOTAL_FAIL" -gt 0 ]] && exit 1 exit 0 diff --git a/Orchestrators/monthly_maintenance.sh b/Orchestrators/monthly_maintenance.sh index ed15f5d..d2a7d04 100755 --- a/Orchestrators/monthly_maintenance.sh +++ b/Orchestrators/monthly_maintenance.sh @@ -234,8 +234,8 @@ echo "$ICON_GEAR Scripts: ${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" echo "" START=$(date +%s) -PASSED=() -FAILED=() +JOB_PASS=() +JOB_FAIL=() STEP=0 # ============================================================================================== @@ -246,47 +246,11 @@ for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do (( STEP++ )) read -r -a parts <<< "$entry" - script_path="$ECOSYSTEM_ROOT/${parts[0]}" script_name=$(basename "${parts[0]}") extra_args=("${parts[@]:1}") echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━" - - if [[ ! -f "$script_path" ]]; then - error "$script_name — not found at $script_path" - FAILED+=("$script_name") - echo "" - continue - fi - - if [[ ! -x "$script_path" ]]; then - warn "$script_name — not executable, fixing..." - chmod +x "$script_path" || { - error "$script_name — chmod +x failed" - FAILED+=("$script_name") - echo "" - continue - } - fi - - if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — would run: $script_name${extra_args:+ ${extra_args[*]}}" - PASSED+=("$script_name") - echo "" - continue - fi - - local_args=() - [[ "$VERBOSE" == true ]] && local_args+=("--log") - - if bash "$script_path" "${extra_args[@]}" "${local_args[@]}"; then - echo "$script_name — done ✅" - PASSED+=("$script_name") - else - warn "$script_name — failed (exit $?) — continuing to next step" - FAILED+=("$script_name") - fi - + run_orch_child "$entry" echo "" done @@ -307,22 +271,22 @@ fi echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" -[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}" -[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" +[[ ${#JOB_PASS[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${JOB_PASS[*]}" +[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}" echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" -elif [[ ${#FAILED[@]} -eq 0 ]]; then +elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then echo "$ICON_DONE Status: all $STEP step(s) complete ✅" notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \ "Monthly Maintenance" "normal" else - warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}" - notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \ + warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}" + notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \ "Monthly Maintenance" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1 exit 0 diff --git a/Orchestrators/sunday_morning_coffee_report.sh b/Orchestrators/sunday_morning_coffee_report.sh index 7a8b7c8..e0726ab 100755 --- a/Orchestrators/sunday_morning_coffee_report.sh +++ b/Orchestrators/sunday_morning_coffee_report.sh @@ -39,9 +39,12 @@ # OPERATIONAL SAFEGUARDS # ============================================================================================== # +# Root check — required for the docker/system reads used in the report +# acquire_lock — prevents overlapping weekly runs # detect_hosts() — MY_ID in banner and summary # Non-fatal steps — a failed script is logged; remaining scripts still run # Flag pass-through — --dry-run and --log forwarded to all child scripts +# notify() on failure — pushed only outside --dry-run, matching the runtime-mode contract below # # ============================================================================================== # CONFIGURATION @@ -70,12 +73,22 @@ # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -source "$SCRIPT_DIR/../load_config.sh" - -SCRIPTS_ROOT="$SCRIPT_DIR/.." +source "$ECOSYSTEM_ROOT/load_config.sh" parse_args "$@" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +acquire_lock + detect_hosts # ============================================================================================== @@ -84,35 +97,6 @@ detect_hosts JOB_PASS=() JOB_FAIL=() -run_job() { - local script_entry="$1" - local extra_dry="" - local extra_log="" - [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run" - [[ "$ENABLE_LOGGING" == true ]] && extra_log="--log" - - read -r -a script_args <<< "$script_entry" - local script_path="$SCRIPTS_ROOT/${script_args[0]}" - local script_name - script_name=$(basename "${script_args[0]}") - local extra_args=("${script_args[@]:1}") - - if [[ ! -f "$script_path" ]]; then - error "$script_name — not found at $script_path" - JOB_FAIL+=("$script_name") - return 1 - fi - - log "Running: $script_name ${extra_args[*]}" - if bash "$script_path" "${extra_args[@]}" $extra_dry $extra_log; then - echo "$script_name — done ✅" - JOB_PASS+=("$script_name") - else - error "$script_name — failed (exit $?)" - JOB_FAIL+=("$script_name") - fi -} - # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== @@ -158,7 +142,7 @@ fi for script_entry in "${COFFEE_REPORT_SCRIPTS[@]}"; do [[ -z "$script_entry" ]] && continue echo "" - run_job "$script_entry" + run_orch_child "$script_entry" done # ============================================================================================== @@ -176,5 +160,10 @@ if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then echo "❌ Failed: ${JOB_FAIL[*]}" fi +if [[ ${#JOB_FAIL[@]} -gt 0 && "$DRY_RUN" != true ]]; then + notify "Sunday coffee report had failures on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \ + "Sunday Morning Coffee Report" "warning" +fi + [[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1 exit 0 diff --git a/Orchestrators/transcode_management.sh b/Orchestrators/transcode_management.sh index dd7907e..78a6542 100755 --- a/Orchestrators/transcode_management.sh +++ b/Orchestrators/transcode_management.sh @@ -5,14 +5,17 @@ # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── -# Runs transcode_cleanup.sh then transcode_manager.sh in the correct order. -# Replaces individual cron entries for each — this is the single cron entry. -# Schedule: */7 * * * * (every 7 minutes via User Scripts plugin) +# Runs TRANSCODE_MANAGEMENT_SCRIPTS in order each cron cycle — this is the +# single cron entry replacing individual entries for each child script. +# Schedule: */7 * * * * (every 7 minutes) # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # +# Order driven by TRANSCODE_MANAGEMENT_SCRIPTS in master.conf. +# Default: transcode_cleanup.sh → transcode_manager.sh +# # transcode_cleanup.sh # Removes aged segment files not open by any process. Uses lsof for O(1) # per-file active check. Triggers flip-back to ramdisk after cleanup if @@ -36,6 +39,8 @@ # Stale segment files from ended sessions inflate the ramdisk usage reading # and trigger unnecessary SSD flips even when active sessions would fit on # the ramdisk. Cleanup runs first so the manager measures real current usage. +# Order is config-driven (TRANSCODE_MANAGEMENT_SCRIPTS) but this dependency +# is real — reordering the array changes what the manager measures. # # Delegated Logging # This orchestrator does not write its own log — transcode_manager.sh owns @@ -48,8 +53,9 @@ # Root check — mount and docker operations require root # acquire_lock — prevents concurrent 7-minute cycles overlapping # detect_hosts() — aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_WARN_GB per host -# --dry-run — passed through to both child scripts -# Exit code — worst exit code of both scripts returned to cron +# --dry-run — passed through to every script in TRANSCODE_MANAGEMENT_SCRIPTS +# Exit code — worst exit code across all scripts returned to cron +# notify() — pushed on failure, skipped in --dry-run # # ============================================================================================== # CONFIGURATION @@ -57,6 +63,7 @@ # # master.conf # +# TRANSCODE_MANAGEMENT_SCRIPTS — scripts to run, in order # TRANSCODE_DAILY_LOG — daily stats log (written by transcode_manager.sh) # TRANSCODE_LOG_RETENTION — days to keep (trimmed by manager) # TRANSCODE_STATE_FILE — current state (ramdisk_used, flip_count, etc.) @@ -70,13 +77,13 @@ # Normal run (every 7 minutes via cron). # # transcode_management.sh --dry-run -# Preview without changes (passed to both child scripts). +# Preview without changes (passed to every script in TRANSCODE_MANAGEMENT_SCRIPTS). # # transcode_management.sh --status # Show configuration and current state. # # transcode_management.sh --log -# Verbose output from both child scripts. +# Verbose output from every child script. # # ============================================================================================== @@ -86,9 +93,6 @@ source "$SCRIPT_DIR/../load_config.sh" parse_args "$@" -CLEANUP_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_cleanup.sh" -MANAGER_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_manager.sh" - # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== @@ -122,12 +126,15 @@ if [[ "$SHOW_STATUS" == true ]]; then echo "$ICON_TIME Schedule: every 7 minutes" echo "" echo "━━━ Child Scripts ━━━" - [[ -f "$CLEANUP_SCRIPT" ]] && \ - echo " $ICON_SUCCESS transcode_cleanup.sh — found" || \ - echo " $ICON_ERROR transcode_cleanup.sh — NOT FOUND at $CLEANUP_SCRIPT" - [[ -f "$MANAGER_SCRIPT" ]] && \ - echo " $ICON_SUCCESS transcode_manager.sh — found" || \ - echo " $ICON_ERROR transcode_manager.sh — NOT FOUND at $MANAGER_SCRIPT" + for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do + _script_path="$SCRIPT_DIR/../$_entry" + _script_name=$(basename "$_entry") + if [[ -f "$_script_path" ]]; then + echo " $ICON_SUCCESS $_script_name — found" + else + echo " $ICON_ERROR $_script_name — NOT FOUND at $_script_path" + fi + done echo "" echo "━━━ Daily Log ━━━" if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]] && [[ -s "$TRANSCODE_DAILY_LOG" ]]; then @@ -154,15 +161,13 @@ fi # ============================================================================================== # ━━━ Validate Child Scripts ━━━ # ============================================================================================== -if [[ ! -f "$CLEANUP_SCRIPT" ]]; then - error "transcode_cleanup.sh not found: $CLEANUP_SCRIPT" - exit 1 -fi - -if [[ ! -f "$MANAGER_SCRIPT" ]]; then - error "transcode_manager.sh not found: $MANAGER_SCRIPT" - exit 1 -fi +for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do + _script_path="$SCRIPT_DIR/../$_entry" + if [[ ! -f "$_script_path" ]]; then + error "$(basename "$_entry") not found: $_script_path" + exit 1 + fi +done # ============================================================================================== # ━━━ Pre-run State Snapshot ━━━ @@ -179,29 +184,46 @@ else fi # ============================================================================================== -# ━━━ Run Cleanup ━━━ +# ━━━ Run Scripts ━━━ # ============================================================================================== +# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run — no flag here +# suppresses that; it owns the log write for this cycle regardless of position ✅ DRY_FLAG="" [[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run" -_cleanup_start=$(date +%s) -bash "$CLEANUP_SCRIPT" $DRY_FLAG -CLEANUP_EXIT=$? -log "cleanup: $(format_duration $(( $(date +%s) - _cleanup_start ))) (exit $CLEANUP_EXIT)" +WORST_EXIT=0 +PASS_COUNT=0 +FAIL_NAMES=() +for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do + _script_path="$SCRIPT_DIR/../$_entry" + _script_name=$(basename "$_entry" .sh) + _start=$(date +%s) + bash "$_script_path" $DRY_FLAG + _exit=$? + log "$_script_name: $(format_duration $(( $(date +%s) - _start ))) (exit $_exit)" + if [[ "$_exit" -ne 0 ]]; then + WORST_EXIT=1 + FAIL_NAMES+=("$_script_name") + else + PASS_COUNT=$(( PASS_COUNT + 1 )) + fi +done # ============================================================================================== -# ━━━ Run Manager ━━━ +# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━ # ============================================================================================== -# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run -# No --no-log flag here — manager owns the log write for this cycle ✅ -_manager_start=$(date +%s) -bash "$MANAGER_SCRIPT" $DRY_FLAG -MANAGER_EXIT=$? -log "manager: $(format_duration $(( $(date +%s) - _manager_start ))) (exit $MANAGER_EXIT)" +if [[ "$WORST_EXIT" -eq 0 ]]; then + echo "$ICON_SUCCESS Transcode cycle — $PASS_COUNT/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed" +else + error "Transcode cycle — failed: ${FAIL_NAMES[*]}" + if [[ "$DRY_RUN" != true ]]; then + notify "Transcode management failure on $(hostname) ($MY_ID) — ${FAIL_NAMES[*]}" \ + "Transcode Management" "warning" + fi +fi # ============================================================================================== # ━━━ Exit ━━━ # ============================================================================================== -# Return worst exit code — caller knows if either script failed -[[ "$CLEANUP_EXIT" -ne 0 || "$MANAGER_EXIT" -ne 0 ]] && exit 1 -exit 0 \ No newline at end of file +# Return worst exit code — caller knows if any script failed +exit "$WORST_EXIT" \ No newline at end of file diff --git a/Orchestrators/watchdog_orchestrator.sh b/Orchestrators/watchdog_orchestrator.sh index aa891b5..de3e431 100755 --- a/Orchestrators/watchdog_orchestrator.sh +++ b/Orchestrators/watchdog_orchestrator.sh @@ -193,7 +193,7 @@ run_watchdog() { local extra_args=() [[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run") - [[ "$VERBOSE" == true ]] && extra_args+=("--log") + [[ "$ENABLE_LOGGING" == true ]] && extra_args+=("--log") local _ws _ws=$(date +%s) @@ -234,18 +234,23 @@ if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then fi # ============================================================================================== -# ━━━ Summary — only shown on failures or --log ━━━ +# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━ # ============================================================================================== -if [[ "${#FAIL[@]}" -gt 0 || "$VERBOSE" == true ]]; then +if [[ "${#FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━" for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done echo "$ICON_TIME Duration: $(format_duration $DURATION)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - - if [[ "${#FAIL[@]}" -gt 0 ]]; then - notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \ - "Watchdog Orchestrator" "warning" - fi +else + echo "$ICON_DONE Watchdog cycle — ${#PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))" fi + +if [[ "${#FAIL[@]}" -gt 0 ]]; then + notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \ + "Watchdog Orchestrator" "warning" + exit 1 +fi + +exit 0 diff --git a/Orchestrators/weekly_sync_maintenance.sh b/Orchestrators/weekly_sync_maintenance.sh index 895c811..726a7ba 100755 --- a/Orchestrators/weekly_sync_maintenance.sh +++ b/Orchestrators/weekly_sync_maintenance.sh @@ -84,8 +84,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" -RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh" -SCRIPTS_ROOT="$SCRIPT_DIR/.." +ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh" parse_args "$@" @@ -123,35 +123,6 @@ read -r -a MAINTENANCE_CONTAINERS <<< \ [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates" -# ── Helper — run a post-sync maintenance script ──────────────────────────────────────────────── -run_job() { - local script_entry="$1" - local extra_dry="" - [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run" - - read -r -a script_args <<< "$script_entry" - local script_path="$SCRIPTS_ROOT/${script_args[0]}" - local script_name - script_name=$(basename "${script_args[0]}") - local extra_args=("${script_args[@]:1}") - - if [[ ! -f "$script_path" ]]; then - error "$script_name — not found at $script_path" - JOB_FAIL+=("$script_name") - return 1 - fi - - log "Running: $script_name ${extra_args[*]}" - # shellcheck disable=SC2086 - if bash "$script_path" "${extra_args[@]}" $extra_dry; then - echo "$script_name — done ✅" - JOB_PASS+=("$script_name ${extra_args[*]}") - else - error "$script_name — failed (exit $?)" - JOB_FAIL+=("$script_name ${extra_args[*]}") - fi -} - # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== @@ -390,7 +361,7 @@ if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do [[ -z "$script_entry" ]] && continue echo "" - run_job "$script_entry" + run_orch_child "$script_entry" done fi diff --git a/common.sh b/common.sh index de92f5b..c279ad4 100755 --- a/common.sh +++ b/common.sh @@ -1537,6 +1537,64 @@ _release_rsync_on_exit() { fi } +# ============================================================================================== +# ── ORCHESTRATOR CHILD EXECUTION ────────────────────────────────────────────────────────────── +# ============================================================================================== +# Shared child-script runner for Orchestrators/*.sh. Replaces the run_job()/run_watchdog() +# copies that used to be hand-duplicated (with small inconsistencies) into each orchestrator — +# one implementation now, one place to fix a bug or extend behavior (e.g. per-child logging). +# +# Usage — caller declares its own tracking arrays before the loop: +# JOB_PASS=() +# JOB_FAIL=() +# for entry in "${SOME_SCRIPTS[@]}"; do +# run_orch_child "$entry" +# done +# +# $entry is "relative/path.sh [extra args...]" — the same format every master.conf +# script-list array already uses. Resolved against $ECOSYSTEM_ROOT, which the caller +# must set before calling this (the absolute repo root — "$(cd "$SCRIPT_DIR/.." && pwd)"). +# +# --dry-run / --log are threaded down automatically from $DRY_RUN / $ENABLE_LOGGING — +# never $VERBOSE, which nothing in this codebase ever assigns. + +run_orch_child() { + local entry="$1" + local script_args script_path script_name label extra_args run_args + + read -r -a script_args <<< "$entry" + script_path="$ECOSYSTEM_ROOT/${script_args[0]}" + script_name=$(basename "${script_args[0]}") + extra_args=("${script_args[@]:1}") + label="$script_name" + [[ -n "${extra_args[*]}" ]] && label="$script_name ${extra_args[*]}" + + if [[ ! -f "$script_path" ]]; then + error "$script_name — not found at $script_path" + JOB_FAIL+=("$label") + return 1 + fi + [[ ! -x "$script_path" ]] && chmod +x "$script_path" + + run_args=("${extra_args[@]}") + [[ "$DRY_RUN" == true ]] && run_args+=("--dry-run") + [[ "$ENABLE_LOGGING" == true ]] && run_args+=("--log") + + local _start _ec + _start=$(date +%s) + log "Running: $label" + if bash "$script_path" "${run_args[@]}"; then + log "$script_name — done in $(format_duration $(( $(date +%s) - _start )))" + JOB_PASS+=("$label") + return 0 + else + _ec=$? + error "$label — failed (exit $_ec, $(format_duration $(( $(date +%s) - _start ))))" + JOB_FAIL+=("$label") + return 1 + fi +} + # ============================================================================================== # ── PATH TRANSLATION ────────────────────────────────────────────────────────────────────────── # ==============================================================================================