Standardize orchestrator child-script execution and logging
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
This commit is contained in:
Gmer4Lfe
@@ -320,6 +320,16 @@
|
||||
"Docker_Essentials/docker_container_stop.sh" # stop all containers last
|
||||
)
|
||||
|
||||
# ━━━ Transcode Management ━━━
|
||||
# transcode_management.sh runs TRANSCODE_MANAGEMENT_SCRIPTS in order each cron cycle.
|
||||
# Schedule: */7 * * * * (every 7 minutes)
|
||||
# Order matters — cleanup first so the manager measures real current ramdisk usage,
|
||||
# not usage inflated by stale segment files from ended sessions.
|
||||
TRANSCODE_MANAGEMENT_SCRIPTS=(
|
||||
"Transcodes/transcode_cleanup.sh" # remove aged segment files before usage is measured
|
||||
"Transcodes/transcode_manager.sh" # flip ramdisk/SSD symlink, write daily log entry
|
||||
)
|
||||
|
||||
# ━━━ Watchdog Orchestrator ━━━
|
||||
# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
|
||||
# Schedule: */15 * * * * (every 15 minutes)
|
||||
|
||||
@@ -317,6 +317,16 @@
|
||||
"Docker_Essentials/docker_container_stop.sh" # stop all containers last
|
||||
)
|
||||
|
||||
# ━━━ Transcode Management ━━━
|
||||
# transcode_management.sh runs TRANSCODE_MANAGEMENT_SCRIPTS in order each cron cycle.
|
||||
# Schedule: */7 * * * * (every 7 minutes)
|
||||
# Order matters — cleanup first so the manager measures real current ramdisk usage,
|
||||
# not usage inflated by stale segment files from ended sessions.
|
||||
TRANSCODE_MANAGEMENT_SCRIPTS=(
|
||||
"Transcodes/transcode_cleanup.sh" # remove aged segment files before usage is measured
|
||||
"Transcodes/transcode_manager.sh" # flip ramdisk/SSD symlink, write daily log entry
|
||||
)
|
||||
|
||||
# ━━━ Watchdog Orchestrator ━━━
|
||||
# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
|
||||
# Schedule: */15 * * * * (every 15 minutes)
|
||||
|
||||
@@ -942,41 +942,40 @@ array_started.sh
|
||||
## ━━━ ADDING A NEW ORCHESTRATOR ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
|
||||
If you find yourself running 3+ related scripts on the same schedule, wrap them
|
||||
in a new orchestrator. Model directly on `media_management.sh` which has the
|
||||
in a new orchestrator. Model directly on `daily_sync_maintenance.sh` which has the
|
||||
complete pattern — dry-run passthrough, status display, pass/fail tracking, summary.
|
||||
|
||||
Child-script execution goes through the shared `run_orch_child()` helper in
|
||||
`common.sh` — never hand-roll a per-file `run_job()` loop. It resolves the entry
|
||||
against `$ECOSYSTEM_ROOT`, threads `--dry-run`/`--log` from `$DRY_RUN`/`$ENABLE_LOGGING`
|
||||
automatically (never `$VERBOSE` — nothing in this codebase assigns it), and tracks
|
||||
into `JOB_PASS`/`JOB_FAIL` arrays the caller declares.
|
||||
|
||||
```bash
|
||||
# Minimal skeleton — the full pattern in its simplest form:
|
||||
#!/bin/bash
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||
parse_args "$@"
|
||||
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
PASS=()
|
||||
FAIL=()
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
|
||||
# Read job list from master.conf — never hardcode jobs in the orchestrator
|
||||
for script_entry in "${MY_MAINTENANCE_JOBS[@]:-}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
|
||||
read -r -a parts <<< "$script_entry"
|
||||
script_path="$SCRIPTS_ROOT/${parts[0]}"
|
||||
script_name=$(basename "${parts[0]}")
|
||||
extra_args=("${parts[@]:1}")
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
||||
|
||||
if bash "$script_path" "${extra_args[@]}"; then
|
||||
PASS+=("$script_name")
|
||||
else
|
||||
FAIL+=("$script_name")
|
||||
fi
|
||||
run_orch_child "$script_entry"
|
||||
done
|
||||
|
||||
# One summary — one notification
|
||||
echo "Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
||||
[[ ${#FAIL[@]} -gt 0 ]] && \
|
||||
notify "My maintenance failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||||
# One summary — one notification. On a frequent (sub-daily) cadence, keep the
|
||||
# healthy path to a single line and reserve the full breakdown for failure/--log —
|
||||
# see watchdog_orchestrator.sh or transcode_management.sh for that split.
|
||||
echo "Passed: ${#JOB_PASS[@]} Failed: ${#JOB_FAIL[@]}"
|
||||
if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
notify "My maintenance failed on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||
"My Orchestrator" "warning"
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
```
|
||||
@@ -163,9 +163,8 @@ log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..."
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
LAUNCHED=0
|
||||
FAILED=0
|
||||
FAILED_SCRIPTS=()
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
|
||||
for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
||||
[[ -z "$relative_path" ]] && continue
|
||||
@@ -177,8 +176,7 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
||||
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
||||
error "$SCRIPT_NAME — not found"
|
||||
error " Expected: $SCRIPT_PATH"
|
||||
(( FAILED++ ))
|
||||
FAILED_SCRIPTS+=("$SCRIPT_NAME")
|
||||
JOB_FAIL+=("$SCRIPT_NAME")
|
||||
continue
|
||||
fi
|
||||
|
||||
@@ -187,15 +185,14 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
||||
warn "$SCRIPT_NAME — not executable, fixing..."
|
||||
chmod +x "$SCRIPT_PATH" || {
|
||||
error "$SCRIPT_NAME — chmod +x failed"
|
||||
(( FAILED++ ))
|
||||
FAILED_SCRIPTS+=("$SCRIPT_NAME")
|
||||
JOB_FAIL+=("$SCRIPT_NAME")
|
||||
continue
|
||||
}
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would launch: $SCRIPT_NAME"
|
||||
(( LAUNCHED++ ))
|
||||
JOB_PASS+=("$SCRIPT_NAME")
|
||||
continue
|
||||
fi
|
||||
|
||||
@@ -209,19 +206,18 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
||||
if kill -0 "$PID" 2>/dev/null; then
|
||||
# Still running → continuous script
|
||||
log "$SCRIPT_NAME — running (PID $PID) ✅"
|
||||
(( LAUNCHED++ ))
|
||||
JOB_PASS+=("$SCRIPT_NAME")
|
||||
else
|
||||
# Exited — check if one-shot success or failure
|
||||
wait "$PID"
|
||||
EXIT_CODE=$?
|
||||
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
||||
echo "$SCRIPT_NAME — completed (one-shot) ✅"
|
||||
(( LAUNCHED++ ))
|
||||
JOB_PASS+=("$SCRIPT_NAME")
|
||||
else
|
||||
error "$SCRIPT_NAME — exited with code $EXIT_CODE"
|
||||
error " Path: $SCRIPT_PATH"
|
||||
(( FAILED++ ))
|
||||
FAILED_SCRIPTS+=("$SCRIPT_NAME")
|
||||
JOB_FAIL+=("$SCRIPT_NAME")
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -235,18 +231,21 @@ END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_SUCCESS Launched: $LAUNCHED"
|
||||
[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR Failed: $FAILED — ${FAILED_SCRIPTS[*]}"
|
||||
echo "$ICON_SUCCESS Launched: ${#JOB_PASS[@]}"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#JOB_FAIL[@]} — ${JOB_FAIL[*]}"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no scripts launched"
|
||||
elif [[ "$FAILED" -gt 0 ]]; then
|
||||
warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}"
|
||||
notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
|
||||
elif [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
warn "Status: ${#JOB_FAIL[@]} script(s) failed — ${JOB_FAIL[*]}"
|
||||
notify "Array start on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} script(s) failed: ${JOB_FAIL[*]}" \
|
||||
"Array Start" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
|
||||
echo "$ICON_DONE Status: all ${#JOB_PASS[@]} script(s) launched ✅"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -134,8 +134,8 @@ echo "$ICON_GEAR Running ${#ARRAY_STOP_SCRIPTS[@]} stop script(s) sequentially..
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
PASSED=()
|
||||
FAILED=()
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
STEP=0
|
||||
|
||||
for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
|
||||
@@ -143,44 +143,11 @@ for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
|
||||
(( STEP++ ))
|
||||
|
||||
read -r -a parts <<< "$entry"
|
||||
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
|
||||
script_name=$(basename "${parts[0]}")
|
||||
extra_args=("${parts[@]:1}")
|
||||
|
||||
echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
FAILED+=("$script_name")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ ! -x "$script_path" ]]; then
|
||||
warn "$script_name — not executable, fixing..."
|
||||
chmod +x "$script_path" || {
|
||||
error "$script_name — chmod +x failed"
|
||||
FAILED+=("$script_name")
|
||||
echo ""
|
||||
continue
|
||||
}
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would run: $script_name ${extra_args[*]}"
|
||||
PASSED+=("$script_name")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
if bash "$script_path" "${extra_args[@]}"; then
|
||||
echo "$script_name — done ✅"
|
||||
PASSED+=("$script_name")
|
||||
else
|
||||
warn "$script_name — failed (exit $?) — continuing to next step"
|
||||
FAILED+=("$script_name")
|
||||
fi
|
||||
|
||||
run_orch_child "$entry"
|
||||
echo ""
|
||||
done
|
||||
|
||||
@@ -192,22 +159,22 @@ END=$(date +%s)
|
||||
echo "━━━━━ $ICON_SUMMARY ARRAY STOP SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
[[ ${#JOB_PASS[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${JOB_PASS[*]}"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
||||
notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
||||
"Array Stop" "normal"
|
||||
else
|
||||
warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}"
|
||||
notify "Array stop on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \
|
||||
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
||||
notify "Array stop on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
||||
"Array Stop" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
|
||||
@@ -80,6 +80,7 @@
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
@@ -199,29 +200,12 @@ fi
|
||||
# ==============================================================================================
|
||||
# ━━━ Critical Maintenance Scripts ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" || "$script_entry" == \#* ]] && continue
|
||||
|
||||
SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
|
||||
SCRIPT_ARGS="${script_entry#* }"
|
||||
[[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
|
||||
[[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run"
|
||||
|
||||
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
|
||||
|
||||
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
||||
warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Running: $SCRIPT_NAME"
|
||||
bash "$SCRIPT_PATH" $SCRIPT_ARGS
|
||||
EXIT_CODE=$?
|
||||
[[ "$EXIT_CODE" -ne 0 ]] && \
|
||||
warn "$SCRIPT_NAME exited with code $EXIT_CODE"
|
||||
run_orch_child "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Partnership Check ━━━
|
||||
@@ -247,20 +231,23 @@ fi
|
||||
END=$(date +%s)
|
||||
DURATION=$(format_duration $(( END - START )))
|
||||
|
||||
# Silent when healthy — only show summary if there were failures or notable events
|
||||
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
# Minimal one-liner when healthy — 30-min cadence, keep it quiet. Full detail on failure.
|
||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $DURATION"
|
||||
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
||||
echo "$ICON_ERROR Failed: ${FAIL[*]}"
|
||||
[[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed shares: ${FAIL[*]}"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||||
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]} ${JOB_FAIL[*]}" \
|
||||
"Critical Sync" "warning"
|
||||
exit 1
|
||||
else
|
||||
echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)"
|
||||
echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s), ${#JOB_PASS[@]} job(s)"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
@@ -103,8 +103,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
@@ -184,35 +184,6 @@ for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
fi
|
||||
done
|
||||
|
||||
# Helper — run a maintenance script, track pass/fail
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
# shellcheck disable=SC2086
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
||||
echo "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
}
|
||||
|
||||
WINDOW_START=$(date +%s)
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
@@ -230,7 +201,7 @@ if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "━━━ $ICON_GIT Pre-sync ━━━"
|
||||
for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
|
||||
run_job "$script_entry"
|
||||
run_orch_child "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
@@ -240,7 +211,7 @@ fi
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
||||
|
||||
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh"
|
||||
ARR_SYNC_SCRIPT="$ECOSYSTEM_ROOT/Arrs_Stack/arr_sync.sh"
|
||||
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
||||
echo "ARR_SYNC_ENABLED=false — skipping"
|
||||
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
||||
@@ -336,7 +307,7 @@ if [[ ${#POST_SYNC_SCRIPTS[@]} -gt 0 ]]; then
|
||||
echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━"
|
||||
for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
|
||||
echo ""
|
||||
run_job "$script_entry"
|
||||
run_orch_child "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
|
||||
@@ -50,7 +50,7 @@
|
||||
# check_connectivity — verified before any rsync (skipped if no shares)
|
||||
# check_remote_rootfs — aborts rsync if remote rootfs nearly full
|
||||
# Non-fatal jobs — a failed arr_sync warns but does not block rsync or artwork fetch
|
||||
# Silent on success — runs 4x/day, only failures warrant notification
|
||||
# Minimal on success — runs 6x/day; full breakdown only on failure or --log
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
@@ -88,8 +88,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
@@ -109,35 +109,6 @@ resolve_remote_ip
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# ── Helper — run a maintenance job, track pass/fail ───────────────────────────────────────────
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
# shellcheck disable=SC2086
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
||||
echo "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
warn "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
@@ -187,7 +158,7 @@ echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Conf Pull ━━━"
|
||||
|
||||
CONF_SYNC_SCRIPT="$SCRIPTS_ROOT/System_Essentials/conf_sync.sh"
|
||||
CONF_SYNC_SCRIPT="$ECOSYSTEM_ROOT/System_Essentials/conf_sync.sh"
|
||||
if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
|
||||
warn "conf_sync.sh not found — skipping partner conf refresh"
|
||||
else
|
||||
@@ -209,7 +180,7 @@ fi
|
||||
echo ""
|
||||
echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
||||
|
||||
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh"
|
||||
ARR_SYNC_SCRIPT="$ECOSYSTEM_ROOT/Arrs_Stack/arr_sync.sh"
|
||||
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
||||
echo "ARR_SYNC_ENABLED=false — skipping"
|
||||
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
||||
@@ -306,7 +277,7 @@ if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
for script_entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
echo ""
|
||||
run_job "$script_entry"
|
||||
run_orch_child "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
@@ -315,6 +286,13 @@ WINDOW_END=$(date +%s)
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
|
||||
# Full breakdown on failure or --log; minimal one-liner otherwise (4-hour cadence — keep it quiet).
|
||||
SHOW_FULL=false
|
||||
[[ "$TOTAL_FAIL" -gt 0 || "$ENABLE_LOGGING" == true ]] && SHOW_FULL=true
|
||||
|
||||
if [[ "$SHOW_FULL" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
@@ -343,19 +321,22 @@ if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||
echo ""
|
||||
fi
|
||||
|
||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||
if [[ "$SHOW_FULL" == true ]]; then
|
||||
echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
|
||||
else
|
||||
echo "$ICON_DONE Intermediate sync — ${#JOB_PASS[@]} job(s), ${#PASS[@]}/$SHARE_COUNT share(s) ($(format_duration $(( WINDOW_END - WINDOW_START ))))"
|
||||
fi
|
||||
else
|
||||
warn "Status: $TOTAL_FAIL failure(s)"
|
||||
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||
"Intermediate Sync" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
[[ "$SHOW_FULL" == true ]] && echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
|
||||
@@ -234,8 +234,8 @@ echo "$ICON_GEAR Scripts: ${#MONTHLY_MAINTENANCE_SCRIPTS[@]}"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
PASSED=()
|
||||
FAILED=()
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
STEP=0
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -246,47 +246,11 @@ for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
(( STEP++ ))
|
||||
|
||||
read -r -a parts <<< "$entry"
|
||||
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
|
||||
script_name=$(basename "${parts[0]}")
|
||||
extra_args=("${parts[@]:1}")
|
||||
|
||||
echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
FAILED+=("$script_name")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ ! -x "$script_path" ]]; then
|
||||
warn "$script_name — not executable, fixing..."
|
||||
chmod +x "$script_path" || {
|
||||
error "$script_name — chmod +x failed"
|
||||
FAILED+=("$script_name")
|
||||
echo ""
|
||||
continue
|
||||
}
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would run: $script_name${extra_args:+ ${extra_args[*]}}"
|
||||
PASSED+=("$script_name")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
local_args=()
|
||||
[[ "$VERBOSE" == true ]] && local_args+=("--log")
|
||||
|
||||
if bash "$script_path" "${extra_args[@]}" "${local_args[@]}"; then
|
||||
echo "$script_name — done ✅"
|
||||
PASSED+=("$script_name")
|
||||
else
|
||||
warn "$script_name — failed (exit $?) — continuing to next step"
|
||||
FAILED+=("$script_name")
|
||||
fi
|
||||
|
||||
run_orch_child "$entry"
|
||||
echo ""
|
||||
done
|
||||
|
||||
@@ -307,22 +271,22 @@ fi
|
||||
echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
[[ ${#JOB_PASS[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${JOB_PASS[*]}"
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
||||
notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
||||
"Monthly Maintenance" "normal"
|
||||
else
|
||||
warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}"
|
||||
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \
|
||||
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
||||
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
||||
"Monthly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
|
||||
@@ -39,9 +39,12 @@
|
||||
# OPERATIONAL SAFEGUARDS
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Root check — required for the docker/system reads used in the report
|
||||
# acquire_lock — prevents overlapping weekly runs
|
||||
# detect_hosts() — MY_ID in banner and summary
|
||||
# Non-fatal steps — a failed script is logged; remaining scripts still run
|
||||
# Flag pass-through — --dry-run and --log forwarded to all child scripts
|
||||
# notify() on failure — pushed only outside --dry-run, matching the runtime-mode contract below
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
@@ -70,12 +73,22 @@
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -84,35 +97,6 @@ detect_hosts
|
||||
JOB_PASS=()
|
||||
JOB_FAIL=()
|
||||
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
local extra_log=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
[[ "$ENABLE_LOGGING" == true ]] && extra_log="--log"
|
||||
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry $extra_log; then
|
||||
echo "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name")
|
||||
else
|
||||
error "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name")
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
@@ -158,7 +142,7 @@ fi
|
||||
for script_entry in "${COFFEE_REPORT_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
echo ""
|
||||
run_job "$script_entry"
|
||||
run_orch_child "$script_entry"
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
@@ -176,5 +160,10 @@ if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||
echo "❌ Failed: ${JOB_FAIL[*]}"
|
||||
fi
|
||||
|
||||
if [[ ${#JOB_FAIL[@]} -gt 0 && "$DRY_RUN" != true ]]; then
|
||||
notify "Sunday coffee report had failures on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||
"Sunday Morning Coffee Report" "warning"
|
||||
fi
|
||||
|
||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
|
||||
@@ -5,14 +5,17 @@
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Runs transcode_cleanup.sh then transcode_manager.sh in the correct order.
|
||||
# Replaces individual cron entries for each — this is the single cron entry.
|
||||
# Schedule: */7 * * * * (every 7 minutes via User Scripts plugin)
|
||||
# Runs TRANSCODE_MANAGEMENT_SCRIPTS in order each cron cycle — this is the
|
||||
# single cron entry replacing individual entries for each child script.
|
||||
# Schedule: */7 * * * * (every 7 minutes)
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Order driven by TRANSCODE_MANAGEMENT_SCRIPTS in master.conf.
|
||||
# Default: transcode_cleanup.sh → transcode_manager.sh
|
||||
#
|
||||
# transcode_cleanup.sh
|
||||
# Removes aged segment files not open by any process. Uses lsof for O(1)
|
||||
# per-file active check. Triggers flip-back to ramdisk after cleanup if
|
||||
@@ -36,6 +39,8 @@
|
||||
# Stale segment files from ended sessions inflate the ramdisk usage reading
|
||||
# and trigger unnecessary SSD flips even when active sessions would fit on
|
||||
# the ramdisk. Cleanup runs first so the manager measures real current usage.
|
||||
# Order is config-driven (TRANSCODE_MANAGEMENT_SCRIPTS) but this dependency
|
||||
# is real — reordering the array changes what the manager measures.
|
||||
#
|
||||
# Delegated Logging
|
||||
# This orchestrator does not write its own log — transcode_manager.sh owns
|
||||
@@ -48,8 +53,9 @@
|
||||
# Root check — mount and docker operations require root
|
||||
# acquire_lock — prevents concurrent 7-minute cycles overlapping
|
||||
# detect_hosts() — aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_WARN_GB per host
|
||||
# --dry-run — passed through to both child scripts
|
||||
# Exit code — worst exit code of both scripts returned to cron
|
||||
# --dry-run — passed through to every script in TRANSCODE_MANAGEMENT_SCRIPTS
|
||||
# Exit code — worst exit code across all scripts returned to cron
|
||||
# notify() — pushed on failure, skipped in --dry-run
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
@@ -57,6 +63,7 @@
|
||||
#
|
||||
# master.conf
|
||||
#
|
||||
# TRANSCODE_MANAGEMENT_SCRIPTS — scripts to run, in order
|
||||
# TRANSCODE_DAILY_LOG — daily stats log (written by transcode_manager.sh)
|
||||
# TRANSCODE_LOG_RETENTION — days to keep (trimmed by manager)
|
||||
# TRANSCODE_STATE_FILE — current state (ramdisk_used, flip_count, etc.)
|
||||
@@ -70,13 +77,13 @@
|
||||
# Normal run (every 7 minutes via cron).
|
||||
#
|
||||
# transcode_management.sh --dry-run
|
||||
# Preview without changes (passed to both child scripts).
|
||||
# Preview without changes (passed to every script in TRANSCODE_MANAGEMENT_SCRIPTS).
|
||||
#
|
||||
# transcode_management.sh --status
|
||||
# Show configuration and current state.
|
||||
#
|
||||
# transcode_management.sh --log
|
||||
# Verbose output from both child scripts.
|
||||
# Verbose output from every child script.
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
@@ -86,9 +93,6 @@ source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
CLEANUP_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_cleanup.sh"
|
||||
MANAGER_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_manager.sh"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
@@ -122,12 +126,15 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo "$ICON_TIME Schedule: every 7 minutes"
|
||||
echo ""
|
||||
echo "━━━ Child Scripts ━━━"
|
||||
[[ -f "$CLEANUP_SCRIPT" ]] && \
|
||||
echo " $ICON_SUCCESS transcode_cleanup.sh — found" || \
|
||||
echo " $ICON_ERROR transcode_cleanup.sh — NOT FOUND at $CLEANUP_SCRIPT"
|
||||
[[ -f "$MANAGER_SCRIPT" ]] && \
|
||||
echo " $ICON_SUCCESS transcode_manager.sh — found" || \
|
||||
echo " $ICON_ERROR transcode_manager.sh — NOT FOUND at $MANAGER_SCRIPT"
|
||||
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
|
||||
_script_path="$SCRIPT_DIR/../$_entry"
|
||||
_script_name=$(basename "$_entry")
|
||||
if [[ -f "$_script_path" ]]; then
|
||||
echo " $ICON_SUCCESS $_script_name — found"
|
||||
else
|
||||
echo " $ICON_ERROR $_script_name — NOT FOUND at $_script_path"
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
echo "━━━ Daily Log ━━━"
|
||||
if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]] && [[ -s "$TRANSCODE_DAILY_LOG" ]]; then
|
||||
@@ -154,15 +161,13 @@ fi
|
||||
# ==============================================================================================
|
||||
# ━━━ Validate Child Scripts ━━━
|
||||
# ==============================================================================================
|
||||
if [[ ! -f "$CLEANUP_SCRIPT" ]]; then
|
||||
error "transcode_cleanup.sh not found: $CLEANUP_SCRIPT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$MANAGER_SCRIPT" ]]; then
|
||||
error "transcode_manager.sh not found: $MANAGER_SCRIPT"
|
||||
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
|
||||
_script_path="$SCRIPT_DIR/../$_entry"
|
||||
if [[ ! -f "$_script_path" ]]; then
|
||||
error "$(basename "$_entry") not found: $_script_path"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-run State Snapshot ━━━
|
||||
@@ -179,29 +184,46 @@ else
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Run Cleanup ━━━
|
||||
# ━━━ Run Scripts ━━━
|
||||
# ==============================================================================================
|
||||
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run — no flag here
|
||||
# suppresses that; it owns the log write for this cycle regardless of position ✅
|
||||
DRY_FLAG=""
|
||||
[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run"
|
||||
|
||||
_cleanup_start=$(date +%s)
|
||||
bash "$CLEANUP_SCRIPT" $DRY_FLAG
|
||||
CLEANUP_EXIT=$?
|
||||
log "cleanup: $(format_duration $(( $(date +%s) - _cleanup_start ))) (exit $CLEANUP_EXIT)"
|
||||
WORST_EXIT=0
|
||||
PASS_COUNT=0
|
||||
FAIL_NAMES=()
|
||||
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
|
||||
_script_path="$SCRIPT_DIR/../$_entry"
|
||||
_script_name=$(basename "$_entry" .sh)
|
||||
_start=$(date +%s)
|
||||
bash "$_script_path" $DRY_FLAG
|
||||
_exit=$?
|
||||
log "$_script_name: $(format_duration $(( $(date +%s) - _start ))) (exit $_exit)"
|
||||
if [[ "$_exit" -ne 0 ]]; then
|
||||
WORST_EXIT=1
|
||||
FAIL_NAMES+=("$_script_name")
|
||||
else
|
||||
PASS_COUNT=$(( PASS_COUNT + 1 ))
|
||||
fi
|
||||
done
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Run Manager ━━━
|
||||
# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
|
||||
# ==============================================================================================
|
||||
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run
|
||||
# No --no-log flag here — manager owns the log write for this cycle ✅
|
||||
_manager_start=$(date +%s)
|
||||
bash "$MANAGER_SCRIPT" $DRY_FLAG
|
||||
MANAGER_EXIT=$?
|
||||
log "manager: $(format_duration $(( $(date +%s) - _manager_start ))) (exit $MANAGER_EXIT)"
|
||||
if [[ "$WORST_EXIT" -eq 0 ]]; then
|
||||
echo "$ICON_SUCCESS Transcode cycle — $PASS_COUNT/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
|
||||
else
|
||||
error "Transcode cycle — failed: ${FAIL_NAMES[*]}"
|
||||
if [[ "$DRY_RUN" != true ]]; then
|
||||
notify "Transcode management failure on $(hostname) ($MY_ID) — ${FAIL_NAMES[*]}" \
|
||||
"Transcode Management" "warning"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Exit ━━━
|
||||
# ==============================================================================================
|
||||
# Return worst exit code — caller knows if either script failed
|
||||
[[ "$CLEANUP_EXIT" -ne 0 || "$MANAGER_EXIT" -ne 0 ]] && exit 1
|
||||
exit 0
|
||||
# Return worst exit code — caller knows if any script failed
|
||||
exit "$WORST_EXIT"
|
||||
@@ -193,7 +193,7 @@ run_watchdog() {
|
||||
|
||||
local extra_args=()
|
||||
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
||||
[[ "$VERBOSE" == true ]] && extra_args+=("--log")
|
||||
[[ "$ENABLE_LOGGING" == true ]] && extra_args+=("--log")
|
||||
|
||||
local _ws
|
||||
_ws=$(date +%s)
|
||||
@@ -234,18 +234,23 @@ if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary — only shown on failures or --log ━━━
|
||||
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${#FAIL[@]}" -gt 0 || "$VERBOSE" == true ]]; then
|
||||
if [[ "${#FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||||
for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done
|
||||
for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
echo "$ICON_DONE Watchdog cycle — ${#PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
|
||||
fi
|
||||
|
||||
if [[ "${#FAIL[@]}" -gt 0 ]]; then
|
||||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||||
"Watchdog Orchestrator" "warning"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
exit 0
|
||||
|
||||
@@ -84,8 +84,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
||||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
@@ -123,35 +123,6 @@ read -r -a MAINTENANCE_CONTAINERS <<< \
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"
|
||||
|
||||
# ── Helper — run a post-sync maintenance script ────────────────────────────────────────────────
|
||||
run_job() {
|
||||
local script_entry="$1"
|
||||
local extra_dry=""
|
||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
||||
|
||||
read -r -a script_args <<< "$script_entry"
|
||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
||||
local script_name
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
local extra_args=("${script_args[@]:1}")
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$script_name")
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Running: $script_name ${extra_args[*]}"
|
||||
# shellcheck disable=SC2086
|
||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
||||
echo "$script_name — done ✅"
|
||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
||||
else
|
||||
error "$script_name — failed (exit $?)"
|
||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
@@ -390,7 +361,7 @@ if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
||||
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$script_entry" ]] && continue
|
||||
echo ""
|
||||
run_job "$script_entry"
|
||||
run_orch_child "$script_entry"
|
||||
done
|
||||
fi
|
||||
|
||||
|
||||
@@ -1537,6 +1537,64 @@ _release_rsync_on_exit() {
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── ORCHESTRATOR CHILD EXECUTION ──────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Shared child-script runner for Orchestrators/*.sh. Replaces the run_job()/run_watchdog()
|
||||
# copies that used to be hand-duplicated (with small inconsistencies) into each orchestrator —
|
||||
# one implementation now, one place to fix a bug or extend behavior (e.g. per-child logging).
|
||||
#
|
||||
# Usage — caller declares its own tracking arrays before the loop:
|
||||
# JOB_PASS=()
|
||||
# JOB_FAIL=()
|
||||
# for entry in "${SOME_SCRIPTS[@]}"; do
|
||||
# run_orch_child "$entry"
|
||||
# done
|
||||
#
|
||||
# $entry is "relative/path.sh [extra args...]" — the same format every master.conf
|
||||
# script-list array already uses. Resolved against $ECOSYSTEM_ROOT, which the caller
|
||||
# must set before calling this (the absolute repo root — "$(cd "$SCRIPT_DIR/.." && pwd)").
|
||||
#
|
||||
# --dry-run / --log are threaded down automatically from $DRY_RUN / $ENABLE_LOGGING —
|
||||
# never $VERBOSE, which nothing in this codebase ever assigns.
|
||||
|
||||
run_orch_child() {
|
||||
local entry="$1"
|
||||
local script_args script_path script_name label extra_args run_args
|
||||
|
||||
read -r -a script_args <<< "$entry"
|
||||
script_path="$ECOSYSTEM_ROOT/${script_args[0]}"
|
||||
script_name=$(basename "${script_args[0]}")
|
||||
extra_args=("${script_args[@]:1}")
|
||||
label="$script_name"
|
||||
[[ -n "${extra_args[*]}" ]] && label="$script_name ${extra_args[*]}"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
JOB_FAIL+=("$label")
|
||||
return 1
|
||||
fi
|
||||
[[ ! -x "$script_path" ]] && chmod +x "$script_path"
|
||||
|
||||
run_args=("${extra_args[@]}")
|
||||
[[ "$DRY_RUN" == true ]] && run_args+=("--dry-run")
|
||||
[[ "$ENABLE_LOGGING" == true ]] && run_args+=("--log")
|
||||
|
||||
local _start _ec
|
||||
_start=$(date +%s)
|
||||
log "Running: $label"
|
||||
if bash "$script_path" "${run_args[@]}"; then
|
||||
log "$script_name — done in $(format_duration $(( $(date +%s) - _start )))"
|
||||
JOB_PASS+=("$label")
|
||||
return 0
|
||||
else
|
||||
_ec=$?
|
||||
error "$label — failed (exit $_ec, $(format_duration $(( $(date +%s) - _start ))))"
|
||||
JOB_FAIL+=("$label")
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── PATH TRANSLATION ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
Reference in New Issue
Block a user