Standardize orchestrator child-script execution and logging
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
This commit is contained in:
@@ -320,6 +320,16 @@
|
|||||||
"Docker_Essentials/docker_container_stop.sh" # stop all containers last
|
"Docker_Essentials/docker_container_stop.sh" # stop all containers last
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# ━━━ Transcode Management ━━━
|
||||||
|
# transcode_management.sh runs TRANSCODE_MANAGEMENT_SCRIPTS in order each cron cycle.
|
||||||
|
# Schedule: */7 * * * * (every 7 minutes)
|
||||||
|
# Order matters — cleanup first so the manager measures real current ramdisk usage,
|
||||||
|
# not usage inflated by stale segment files from ended sessions.
|
||||||
|
TRANSCODE_MANAGEMENT_SCRIPTS=(
|
||||||
|
"Transcodes/transcode_cleanup.sh" # remove aged segment files before usage is measured
|
||||||
|
"Transcodes/transcode_manager.sh" # flip ramdisk/SSD symlink, write daily log entry
|
||||||
|
)
|
||||||
|
|
||||||
# ━━━ Watchdog Orchestrator ━━━
|
# ━━━ Watchdog Orchestrator ━━━
|
||||||
# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
|
# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
|
||||||
# Schedule: */15 * * * * (every 15 minutes)
|
# Schedule: */15 * * * * (every 15 minutes)
|
||||||
|
|||||||
@@ -317,6 +317,16 @@
|
|||||||
"Docker_Essentials/docker_container_stop.sh" # stop all containers last
|
"Docker_Essentials/docker_container_stop.sh" # stop all containers last
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# ━━━ Transcode Management ━━━
|
||||||
|
# transcode_management.sh runs TRANSCODE_MANAGEMENT_SCRIPTS in order each cron cycle.
|
||||||
|
# Schedule: */7 * * * * (every 7 minutes)
|
||||||
|
# Order matters — cleanup first so the manager measures real current ramdisk usage,
|
||||||
|
# not usage inflated by stale segment files from ended sessions.
|
||||||
|
TRANSCODE_MANAGEMENT_SCRIPTS=(
|
||||||
|
"Transcodes/transcode_cleanup.sh" # remove aged segment files before usage is measured
|
||||||
|
"Transcodes/transcode_manager.sh" # flip ramdisk/SSD symlink, write daily log entry
|
||||||
|
)
|
||||||
|
|
||||||
# ━━━ Watchdog Orchestrator ━━━
|
# ━━━ Watchdog Orchestrator ━━━
|
||||||
# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
|
# watchdog_orchestrator.sh runs WATCHDOG_ORCHESTRATOR_SCRIPTS in order each cron cycle.
|
||||||
# Schedule: */15 * * * * (every 15 minutes)
|
# Schedule: */15 * * * * (every 15 minutes)
|
||||||
|
|||||||
@@ -942,41 +942,40 @@ array_started.sh
|
|||||||
## ━━━ ADDING A NEW ORCHESTRATOR ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
## ━━━ ADDING A NEW ORCHESTRATOR ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||||
|
|
||||||
If you find yourself running 3+ related scripts on the same schedule, wrap them
|
If you find yourself running 3+ related scripts on the same schedule, wrap them
|
||||||
in a new orchestrator. Model directly on `media_management.sh` which has the
|
in a new orchestrator. Model directly on `daily_sync_maintenance.sh` which has the
|
||||||
complete pattern — dry-run passthrough, status display, pass/fail tracking, summary.
|
complete pattern — dry-run passthrough, status display, pass/fail tracking, summary.
|
||||||
|
|
||||||
|
Child-script execution goes through the shared `run_orch_child()` helper in
|
||||||
|
`common.sh` — never hand-roll a per-file `run_job()` loop. It resolves the entry
|
||||||
|
against `$ECOSYSTEM_ROOT`, threads `--dry-run`/`--log` from `$DRY_RUN`/`$ENABLE_LOGGING`
|
||||||
|
automatically (never `$VERBOSE` — nothing in this codebase assigns it), and tracks
|
||||||
|
into `JOB_PASS`/`JOB_FAIL` arrays the caller declares.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Minimal skeleton — the full pattern in its simplest form:
|
# Minimal skeleton — the full pattern in its simplest form:
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
source "$SCRIPT_DIR/../load_config.sh"
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
JOB_PASS=()
|
||||||
PASS=()
|
JOB_FAIL=()
|
||||||
FAIL=()
|
|
||||||
|
|
||||||
# Read job list from master.conf — never hardcode jobs in the orchestrator
|
# Read job list from master.conf — never hardcode jobs in the orchestrator
|
||||||
for script_entry in "${MY_MAINTENANCE_JOBS[@]:-}"; do
|
for script_entry in "${MY_MAINTENANCE_JOBS[@]:-}"; do
|
||||||
[[ -z "$script_entry" ]] && continue
|
[[ -z "$script_entry" ]] && continue
|
||||||
|
run_orch_child "$script_entry"
|
||||||
read -r -a parts <<< "$script_entry"
|
|
||||||
script_path="$SCRIPTS_ROOT/${parts[0]}"
|
|
||||||
script_name=$(basename "${parts[0]}")
|
|
||||||
extra_args=("${parts[@]:1}")
|
|
||||||
|
|
||||||
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
|
||||||
|
|
||||||
if bash "$script_path" "${extra_args[@]}"; then
|
|
||||||
PASS+=("$script_name")
|
|
||||||
else
|
|
||||||
FAIL+=("$script_name")
|
|
||||||
fi
|
|
||||||
done
|
done
|
||||||
|
|
||||||
# One summary — one notification
|
# One summary — one notification. On a frequent (sub-daily) cadence, keep the
|
||||||
echo "Passed: ${#PASS[@]} Failed: ${#FAIL[@]}"
|
# healthy path to a single line and reserve the full breakdown for failure/--log —
|
||||||
[[ ${#FAIL[@]} -gt 0 ]] && \
|
# see watchdog_orchestrator.sh or transcode_management.sh for that split.
|
||||||
notify "My maintenance failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
echo "Passed: ${#JOB_PASS[@]} Failed: ${#JOB_FAIL[@]}"
|
||||||
|
if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||||
|
notify "My maintenance failed on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||||
"My Orchestrator" "warning"
|
"My Orchestrator" "warning"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
```
|
```
|
||||||
@@ -163,9 +163,8 @@ log "Launching ${#ARRAY_START_SCRIPTS[@]} script(s)..."
|
|||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
START=$(date +%s)
|
START=$(date +%s)
|
||||||
LAUNCHED=0
|
JOB_PASS=()
|
||||||
FAILED=0
|
JOB_FAIL=()
|
||||||
FAILED_SCRIPTS=()
|
|
||||||
|
|
||||||
for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
||||||
[[ -z "$relative_path" ]] && continue
|
[[ -z "$relative_path" ]] && continue
|
||||||
@@ -177,8 +176,7 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
|||||||
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
||||||
error "$SCRIPT_NAME — not found"
|
error "$SCRIPT_NAME — not found"
|
||||||
error " Expected: $SCRIPT_PATH"
|
error " Expected: $SCRIPT_PATH"
|
||||||
(( FAILED++ ))
|
JOB_FAIL+=("$SCRIPT_NAME")
|
||||||
FAILED_SCRIPTS+=("$SCRIPT_NAME")
|
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -187,15 +185,14 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
|||||||
warn "$SCRIPT_NAME — not executable, fixing..."
|
warn "$SCRIPT_NAME — not executable, fixing..."
|
||||||
chmod +x "$SCRIPT_PATH" || {
|
chmod +x "$SCRIPT_PATH" || {
|
||||||
error "$SCRIPT_NAME — chmod +x failed"
|
error "$SCRIPT_NAME — chmod +x failed"
|
||||||
(( FAILED++ ))
|
JOB_FAIL+=("$SCRIPT_NAME")
|
||||||
FAILED_SCRIPTS+=("$SCRIPT_NAME")
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would launch: $SCRIPT_NAME"
|
warn "DRY RUN — would launch: $SCRIPT_NAME"
|
||||||
(( LAUNCHED++ ))
|
JOB_PASS+=("$SCRIPT_NAME")
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -209,19 +206,18 @@ for relative_path in "${ARRAY_START_SCRIPTS[@]}"; do
|
|||||||
if kill -0 "$PID" 2>/dev/null; then
|
if kill -0 "$PID" 2>/dev/null; then
|
||||||
# Still running → continuous script
|
# Still running → continuous script
|
||||||
log "$SCRIPT_NAME — running (PID $PID) ✅"
|
log "$SCRIPT_NAME — running (PID $PID) ✅"
|
||||||
(( LAUNCHED++ ))
|
JOB_PASS+=("$SCRIPT_NAME")
|
||||||
else
|
else
|
||||||
# Exited — check if one-shot success or failure
|
# Exited — check if one-shot success or failure
|
||||||
wait "$PID"
|
wait "$PID"
|
||||||
EXIT_CODE=$?
|
EXIT_CODE=$?
|
||||||
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
if [[ "$EXIT_CODE" -eq 0 ]]; then
|
||||||
echo "$SCRIPT_NAME — completed (one-shot) ✅"
|
echo "$SCRIPT_NAME — completed (one-shot) ✅"
|
||||||
(( LAUNCHED++ ))
|
JOB_PASS+=("$SCRIPT_NAME")
|
||||||
else
|
else
|
||||||
error "$SCRIPT_NAME — exited with code $EXIT_CODE"
|
error "$SCRIPT_NAME — exited with code $EXIT_CODE"
|
||||||
error " Path: $SCRIPT_PATH"
|
error " Path: $SCRIPT_PATH"
|
||||||
(( FAILED++ ))
|
JOB_FAIL+=("$SCRIPT_NAME")
|
||||||
FAILED_SCRIPTS+=("$SCRIPT_NAME")
|
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -235,18 +231,21 @@ END=$(date +%s)
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
|
echo "━━━━━ $ICON_SUMMARY ARRAY START SUMMARY ━━━━━"
|
||||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||||
echo "$ICON_SUCCESS Launched: $LAUNCHED"
|
echo "$ICON_SUCCESS Launched: ${#JOB_PASS[@]}"
|
||||||
[[ "$FAILED" -gt 0 ]] && echo "$ICON_ERROR Failed: $FAILED — ${FAILED_SCRIPTS[*]}"
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${#JOB_FAIL[@]} — ${JOB_FAIL[*]}"
|
||||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — no scripts launched"
|
warn "DRY RUN — no scripts launched"
|
||||||
elif [[ "$FAILED" -gt 0 ]]; then
|
elif [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||||
warn "Status: $FAILED script(s) failed — ${FAILED_SCRIPTS[*]}"
|
warn "Status: ${#JOB_FAIL[@]} script(s) failed — ${JOB_FAIL[*]}"
|
||||||
notify "Array start on $(hostname) ($MY_ID) — $FAILED script(s) failed: ${FAILED_SCRIPTS[*]}" \
|
notify "Array start on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} script(s) failed: ${JOB_FAIL[*]}" \
|
||||||
"Array Start" "warning"
|
"Array Start" "warning"
|
||||||
else
|
else
|
||||||
echo "$ICON_DONE Status: all $LAUNCHED script(s) launched ✅"
|
echo "$ICON_DONE Status: all ${#JOB_PASS[@]} script(s) launched ✅"
|
||||||
fi
|
fi
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||||
|
exit 0
|
||||||
@@ -134,8 +134,8 @@ echo "$ICON_GEAR Running ${#ARRAY_STOP_SCRIPTS[@]} stop script(s) sequentially..
|
|||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
START=$(date +%s)
|
START=$(date +%s)
|
||||||
PASSED=()
|
JOB_PASS=()
|
||||||
FAILED=()
|
JOB_FAIL=()
|
||||||
STEP=0
|
STEP=0
|
||||||
|
|
||||||
for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
|
for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
|
||||||
@@ -143,44 +143,11 @@ for entry in "${ARRAY_STOP_SCRIPTS[@]}"; do
|
|||||||
(( STEP++ ))
|
(( STEP++ ))
|
||||||
|
|
||||||
read -r -a parts <<< "$entry"
|
read -r -a parts <<< "$entry"
|
||||||
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
|
|
||||||
script_name=$(basename "${parts[0]}")
|
script_name=$(basename "${parts[0]}")
|
||||||
extra_args=("${parts[@]:1}")
|
extra_args=("${parts[@]:1}")
|
||||||
|
|
||||||
echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
|
echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
|
||||||
|
run_orch_child "$entry"
|
||||||
if [[ ! -f "$script_path" ]]; then
|
|
||||||
error "$script_name — not found at $script_path"
|
|
||||||
FAILED+=("$script_name")
|
|
||||||
echo ""
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ ! -x "$script_path" ]]; then
|
|
||||||
warn "$script_name — not executable, fixing..."
|
|
||||||
chmod +x "$script_path" || {
|
|
||||||
error "$script_name — chmod +x failed"
|
|
||||||
FAILED+=("$script_name")
|
|
||||||
echo ""
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
|
||||||
warn "DRY RUN — would run: $script_name ${extra_args[*]}"
|
|
||||||
PASSED+=("$script_name")
|
|
||||||
echo ""
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
if bash "$script_path" "${extra_args[@]}"; then
|
|
||||||
echo "$script_name — done ✅"
|
|
||||||
PASSED+=("$script_name")
|
|
||||||
else
|
|
||||||
warn "$script_name — failed (exit $?) — continuing to next step"
|
|
||||||
FAILED+=("$script_name")
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
done
|
done
|
||||||
|
|
||||||
@@ -192,22 +159,22 @@ END=$(date +%s)
|
|||||||
echo "━━━━━ $ICON_SUMMARY ARRAY STOP SUMMARY ━━━━━"
|
echo "━━━━━ $ICON_SUMMARY ARRAY STOP SUMMARY ━━━━━"
|
||||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||||
[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}"
|
[[ ${#JOB_PASS[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${JOB_PASS[*]}"
|
||||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — no changes made"
|
warn "DRY RUN — no changes made"
|
||||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
||||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
||||||
notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
notify "Array stop complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
||||||
"Array Stop" "normal"
|
"Array Stop" "normal"
|
||||||
else
|
else
|
||||||
warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}"
|
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
||||||
notify "Array stop on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \
|
notify "Array stop on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
||||||
"Array Stop" "warning"
|
"Array Stop" "warning"
|
||||||
fi
|
fi
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||||
exit 0
|
exit 0
|
||||||
|
|||||||
@@ -80,6 +80,7 @@
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
|
||||||
source "$SCRIPT_DIR/../load_config.sh"
|
source "$SCRIPT_DIR/../load_config.sh"
|
||||||
|
|
||||||
@@ -199,29 +200,12 @@ fi
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Critical Maintenance Scripts ━━━
|
# ━━━ Critical Maintenance Scripts ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
if [[ ${#CRITICAL_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
JOB_PASS=()
|
||||||
for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
JOB_FAIL=()
|
||||||
[[ -z "$script_entry" || "$script_entry" == \#* ]] && continue
|
for script_entry in "${CRITICAL_MAINTENANCE_SCRIPTS[@]}"; do
|
||||||
|
[[ -z "$script_entry" || "$script_entry" == \#* ]] && continue
|
||||||
SCRIPT_PATH="$SCRIPT_DIR/../${script_entry%% *}"
|
run_orch_child "$script_entry"
|
||||||
SCRIPT_ARGS="${script_entry#* }"
|
done
|
||||||
[[ "$SCRIPT_ARGS" == "$script_entry" ]] && SCRIPT_ARGS=""
|
|
||||||
[[ "$DRY_RUN" == true ]] && SCRIPT_ARGS="$SCRIPT_ARGS --dry-run"
|
|
||||||
|
|
||||||
SCRIPT_NAME=$(basename "$SCRIPT_PATH")
|
|
||||||
|
|
||||||
if [[ ! -f "$SCRIPT_PATH" ]]; then
|
|
||||||
warn "$SCRIPT_NAME not found at $SCRIPT_PATH — skipping"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "Running: $SCRIPT_NAME"
|
|
||||||
bash "$SCRIPT_PATH" $SCRIPT_ARGS
|
|
||||||
EXIT_CODE=$?
|
|
||||||
[[ "$EXIT_CODE" -ne 0 ]] && \
|
|
||||||
warn "$SCRIPT_NAME exited with code $EXIT_CODE"
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Partnership Check ━━━
|
# ━━━ Partnership Check ━━━
|
||||||
@@ -247,20 +231,23 @@ fi
|
|||||||
END=$(date +%s)
|
END=$(date +%s)
|
||||||
DURATION=$(format_duration $(( END - START )))
|
DURATION=$(format_duration $(( END - START )))
|
||||||
|
|
||||||
# Silent when healthy — only show summary if there were failures or notable events
|
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||||
if [[ ${#FAIL[@]} -gt 0 ]]; then
|
|
||||||
|
# Minimal one-liner when healthy — 30-min cadence, keep it quiet. Full detail on failure.
|
||||||
|
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
echo "━━━━━ $ICON_SUMMARY CRITICAL SYNC SUMMARY ━━━━━"
|
||||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||||
echo "$ICON_TIME Duration: $DURATION"
|
echo "$ICON_TIME Duration: $DURATION"
|
||||||
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
[[ ${#PASS[@]} -gt 0 ]] && echo "Synced: ${PASS[*]}"
|
||||||
echo "$ICON_ERROR Failed: ${FAIL[*]}"
|
[[ ${#FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed shares: ${FAIL[*]}"
|
||||||
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed jobs: ${JOB_FAIL[*]}"
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
notify "Critical sync failed on $(hostname) ($MY_ID) — ${FAIL[*]} ${JOB_FAIL[*]}" \
|
||||||
"Critical Sync" "warning"
|
"Critical Sync" "warning"
|
||||||
exit 1
|
exit 1
|
||||||
else
|
else
|
||||||
echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s)"
|
echo "Critical sync complete — $MY_ID — ${DURATION} — ${#PASS[@]} share(s), ${#JOB_PASS[@]} job(s)"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
exit 0
|
exit 0
|
||||||
@@ -103,8 +103,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|||||||
|
|
||||||
source "$SCRIPT_DIR/../load_config.sh"
|
source "$SCRIPT_DIR/../load_config.sh"
|
||||||
|
|
||||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"
|
||||||
|
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
@@ -184,35 +184,6 @@ for script_entry in "${DAILY_MAINTENANCE_SCRIPTS[@]}"; do
|
|||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
# Helper — run a maintenance script, track pass/fail
|
|
||||||
run_job() {
|
|
||||||
local script_entry="$1"
|
|
||||||
local extra_dry=""
|
|
||||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
|
||||||
|
|
||||||
read -r -a script_args <<< "$script_entry"
|
|
||||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
|
||||||
local script_name
|
|
||||||
script_name=$(basename "${script_args[0]}")
|
|
||||||
local extra_args=("${script_args[@]:1}")
|
|
||||||
|
|
||||||
if [[ ! -f "$script_path" ]]; then
|
|
||||||
error "$script_name — not found at $script_path"
|
|
||||||
JOB_FAIL+=("$script_name")
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "Running: $script_name ${extra_args[*]}"
|
|
||||||
# shellcheck disable=SC2086
|
|
||||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
|
||||||
echo "$script_name — done ✅"
|
|
||||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
|
||||||
else
|
|
||||||
error "$script_name — failed (exit $?)"
|
|
||||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
WINDOW_START=$(date +%s)
|
WINDOW_START=$(date +%s)
|
||||||
JOB_PASS=()
|
JOB_PASS=()
|
||||||
JOB_FAIL=()
|
JOB_FAIL=()
|
||||||
@@ -230,7 +201,7 @@ if [[ ${#PRE_SYNC_SCRIPTS[@]} -gt 0 ]]; then
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_GIT Pre-sync ━━━"
|
echo "━━━ $ICON_GIT Pre-sync ━━━"
|
||||||
for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
|
for script_entry in "${PRE_SYNC_SCRIPTS[@]}"; do
|
||||||
run_job "$script_entry"
|
run_orch_child "$script_entry"
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -240,7 +211,7 @@ fi
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
||||||
|
|
||||||
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh"
|
ARR_SYNC_SCRIPT="$ECOSYSTEM_ROOT/Arrs_Stack/arr_sync.sh"
|
||||||
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
||||||
echo "ARR_SYNC_ENABLED=false — skipping"
|
echo "ARR_SYNC_ENABLED=false — skipping"
|
||||||
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
||||||
@@ -336,7 +307,7 @@ if [[ ${#POST_SYNC_SCRIPTS[@]} -gt 0 ]]; then
|
|||||||
echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━"
|
echo "━━━ $ICON_CLEAN Post-sync Maintenance ━━━"
|
||||||
for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
|
for script_entry in "${POST_SYNC_SCRIPTS[@]}"; do
|
||||||
echo ""
|
echo ""
|
||||||
run_job "$script_entry"
|
run_orch_child "$script_entry"
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
@@ -50,7 +50,7 @@
|
|||||||
# check_connectivity — verified before any rsync (skipped if no shares)
|
# check_connectivity — verified before any rsync (skipped if no shares)
|
||||||
# check_remote_rootfs — aborts rsync if remote rootfs nearly full
|
# check_remote_rootfs — aborts rsync if remote rootfs nearly full
|
||||||
# Non-fatal jobs — a failed arr_sync warns but does not block rsync or artwork fetch
|
# Non-fatal jobs — a failed arr_sync warns but does not block rsync or artwork fetch
|
||||||
# Silent on success — runs 4x/day, only failures warrant notification
|
# Minimal on success — runs 6x/day; full breakdown only on failure or --log
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# CONFIGURATION
|
# CONFIGURATION
|
||||||
@@ -88,8 +88,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|||||||
|
|
||||||
source "$SCRIPT_DIR/../load_config.sh"
|
source "$SCRIPT_DIR/../load_config.sh"
|
||||||
|
|
||||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"
|
||||||
|
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
@@ -109,35 +109,6 @@ resolve_remote_ip
|
|||||||
|
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||||
|
|
||||||
# ── Helper — run a maintenance job, track pass/fail ───────────────────────────────────────────
|
|
||||||
run_job() {
|
|
||||||
local script_entry="$1"
|
|
||||||
local extra_dry=""
|
|
||||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
|
||||||
|
|
||||||
read -r -a script_args <<< "$script_entry"
|
|
||||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
|
||||||
local script_name
|
|
||||||
script_name=$(basename "${script_args[0]}")
|
|
||||||
local extra_args=("${script_args[@]:1}")
|
|
||||||
|
|
||||||
if [[ ! -f "$script_path" ]]; then
|
|
||||||
error "$script_name — not found at $script_path"
|
|
||||||
JOB_FAIL+=("$script_name")
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "Running: $script_name ${extra_args[*]}"
|
|
||||||
# shellcheck disable=SC2086
|
|
||||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
|
||||||
echo "$script_name — done ✅"
|
|
||||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
|
||||||
else
|
|
||||||
warn "$script_name — failed (exit $?)"
|
|
||||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Status ━━━
|
# ━━━ Status ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -187,7 +158,7 @@ echo "━━━ $ICON_GEAR Intermediate Sync — $MY_ID — $(date '+%Y-%m-%d %H
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_GEAR Conf Pull ━━━"
|
echo "━━━ $ICON_GEAR Conf Pull ━━━"
|
||||||
|
|
||||||
CONF_SYNC_SCRIPT="$SCRIPTS_ROOT/System_Essentials/conf_sync.sh"
|
CONF_SYNC_SCRIPT="$ECOSYSTEM_ROOT/System_Essentials/conf_sync.sh"
|
||||||
if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
|
if [[ ! -f "$CONF_SYNC_SCRIPT" ]]; then
|
||||||
warn "conf_sync.sh not found — skipping partner conf refresh"
|
warn "conf_sync.sh not found — skipping partner conf refresh"
|
||||||
else
|
else
|
||||||
@@ -209,7 +180,7 @@ fi
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
echo "━━━ $ICON_SYNC Arr Sync ━━━"
|
||||||
|
|
||||||
ARR_SYNC_SCRIPT="$SCRIPTS_ROOT/Arrs_Stack/arr_sync.sh"
|
ARR_SYNC_SCRIPT="$ECOSYSTEM_ROOT/Arrs_Stack/arr_sync.sh"
|
||||||
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
if [[ "${ARR_SYNC_ENABLED:-true}" != "true" ]]; then
|
||||||
echo "ARR_SYNC_ENABLED=false — skipping"
|
echo "ARR_SYNC_ENABLED=false — skipping"
|
||||||
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
elif [[ ! -f "$ARR_SYNC_SCRIPT" ]]; then
|
||||||
@@ -306,7 +277,7 @@ if [[ ${#INTERMEDIATE_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
|||||||
for script_entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
|
for script_entry in "${INTERMEDIATE_MAINTENANCE_SCRIPTS[@]}"; do
|
||||||
[[ -z "$script_entry" ]] && continue
|
[[ -z "$script_entry" ]] && continue
|
||||||
echo ""
|
echo ""
|
||||||
run_job "$script_entry"
|
run_orch_child "$script_entry"
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -315,47 +286,57 @@ WINDOW_END=$(date +%s)
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Summary ━━━
|
# ━━━ Summary ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
echo ""
|
|
||||||
echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━"
|
|
||||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
||||||
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
|
||||||
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
|
||||||
echo ""
|
|
||||||
|
|
||||||
if [[ "$SHARE_COUNT" -gt 0 ]]; then
|
|
||||||
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
|
||||||
for entry in "${SHARE_TIMES[@]}"; do
|
|
||||||
sname="${entry%%:*}"
|
|
||||||
sdur="${entry##*:}"
|
|
||||||
if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
|
|
||||||
echo " $ICON_ERROR $sname — $(format_duration "$sdur")"
|
|
||||||
else
|
|
||||||
echo " $ICON_DONE $sname — $(format_duration "$sdur")"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
|
||||||
echo ""
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|
||||||
echo "$ICON_GEAR Jobs:"
|
|
||||||
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
|
||||||
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
|
||||||
echo ""
|
|
||||||
fi
|
|
||||||
|
|
||||||
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
TOTAL_FAIL=$(( ${#FAIL[@]} + ${#JOB_FAIL[@]} ))
|
||||||
|
|
||||||
|
# Full breakdown on failure or --log; minimal one-liner otherwise (4-hour cadence — keep it quiet).
|
||||||
|
SHOW_FULL=false
|
||||||
|
[[ "$TOTAL_FAIL" -gt 0 || "$ENABLE_LOGGING" == true ]] && SHOW_FULL=true
|
||||||
|
|
||||||
|
if [[ "$SHOW_FULL" == true ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SUMMARY INTERMEDIATE SYNC SUMMARY ━━━━━"
|
||||||
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||||
|
echo "$ICON_TIME Window: $(date -d @"$WINDOW_START" '+%Y-%m-%d %H:%M:%S') → $(date -d @"$WINDOW_END" '+%H:%M:%S')"
|
||||||
|
echo "$ICON_TIME Duration: $(format_duration $(( WINDOW_END - WINDOW_START )))"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
if [[ "$SHARE_COUNT" -gt 0 ]]; then
|
||||||
|
echo "$ICON_SYNC Shares ($SHARE_COUNT):"
|
||||||
|
for entry in "${SHARE_TIMES[@]}"; do
|
||||||
|
sname="${entry%%:*}"
|
||||||
|
sdur="${entry##*:}"
|
||||||
|
if printf '%s\n' "${FAIL[@]}" | grep -q "^${sname}"; then
|
||||||
|
echo " $ICON_ERROR $sname — $(format_duration "$sdur")"
|
||||||
|
else
|
||||||
|
echo " $ICON_DONE $sname — $(format_duration "$sdur")"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo " Passed: ${#PASS[@]}/$SHARE_COUNT Failed: ${#FAIL[@]}/$SHARE_COUNT"
|
||||||
|
echo ""
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ${#JOB_PASS[@]} -gt 0 || ${#JOB_FAIL[@]} -gt 0 ]]; then
|
||||||
|
echo "$ICON_GEAR Jobs:"
|
||||||
|
for job in "${JOB_PASS[@]}"; do echo " $ICON_DONE $job"; done
|
||||||
|
for job in "${JOB_FAIL[@]}"; do echo " $ICON_ERROR $job"; done
|
||||||
|
echo ""
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — no changes made"
|
warn "DRY RUN — no changes made"
|
||||||
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
elif [[ "$TOTAL_FAIL" -eq 0 ]]; then
|
||||||
echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
|
if [[ "$SHOW_FULL" == true ]]; then
|
||||||
|
echo "$ICON_DONE Status: all complete ✅ — ${#JOB_PASS[@]} job(s) run, ${#PASS[@]}/$SHARE_COUNT share(s) synced"
|
||||||
|
else
|
||||||
|
echo "$ICON_DONE Intermediate sync — ${#JOB_PASS[@]} job(s), ${#PASS[@]}/$SHARE_COUNT share(s) ($(format_duration $(( WINDOW_END - WINDOW_START ))))"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
warn "Status: $TOTAL_FAIL failure(s)"
|
warn "Status: $TOTAL_FAIL failure(s)"
|
||||||
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
notify "Intermediate sync failed on $(hostname) ($MY_ID) — shares: ${#FAIL[@]}/$SHARE_COUNT failed, jobs: ${#JOB_FAIL[@]} failed" \
|
||||||
"Intermediate Sync" "warning"
|
"Intermediate Sync" "warning"
|
||||||
fi
|
fi
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
[[ "$SHOW_FULL" == true ]] && echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
[[ "$TOTAL_FAIL" -gt 0 ]] && exit 1
|
||||||
exit 0
|
exit 0
|
||||||
|
|||||||
@@ -234,8 +234,8 @@ echo "$ICON_GEAR Scripts: ${#MONTHLY_MAINTENANCE_SCRIPTS[@]}"
|
|||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
START=$(date +%s)
|
START=$(date +%s)
|
||||||
PASSED=()
|
JOB_PASS=()
|
||||||
FAILED=()
|
JOB_FAIL=()
|
||||||
STEP=0
|
STEP=0
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -246,47 +246,11 @@ for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
|
|||||||
(( STEP++ ))
|
(( STEP++ ))
|
||||||
|
|
||||||
read -r -a parts <<< "$entry"
|
read -r -a parts <<< "$entry"
|
||||||
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
|
|
||||||
script_name=$(basename "${parts[0]}")
|
script_name=$(basename "${parts[0]}")
|
||||||
extra_args=("${parts[@]:1}")
|
extra_args=("${parts[@]:1}")
|
||||||
|
|
||||||
echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
|
echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
|
||||||
|
run_orch_child "$entry"
|
||||||
if [[ ! -f "$script_path" ]]; then
|
|
||||||
error "$script_name — not found at $script_path"
|
|
||||||
FAILED+=("$script_name")
|
|
||||||
echo ""
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ ! -x "$script_path" ]]; then
|
|
||||||
warn "$script_name — not executable, fixing..."
|
|
||||||
chmod +x "$script_path" || {
|
|
||||||
error "$script_name — chmod +x failed"
|
|
||||||
FAILED+=("$script_name")
|
|
||||||
echo ""
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
|
||||||
warn "DRY RUN — would run: $script_name${extra_args:+ ${extra_args[*]}}"
|
|
||||||
PASSED+=("$script_name")
|
|
||||||
echo ""
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
|
|
||||||
local_args=()
|
|
||||||
[[ "$VERBOSE" == true ]] && local_args+=("--log")
|
|
||||||
|
|
||||||
if bash "$script_path" "${extra_args[@]}" "${local_args[@]}"; then
|
|
||||||
echo "$script_name — done ✅"
|
|
||||||
PASSED+=("$script_name")
|
|
||||||
else
|
|
||||||
warn "$script_name — failed (exit $?) — continuing to next step"
|
|
||||||
FAILED+=("$script_name")
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
done
|
done
|
||||||
|
|
||||||
@@ -307,22 +271,22 @@ fi
|
|||||||
echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━"
|
echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━"
|
||||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||||
[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}"
|
[[ ${#JOB_PASS[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${JOB_PASS[*]}"
|
||||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — no changes made"
|
warn "DRY RUN — no changes made"
|
||||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
||||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
||||||
notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
||||||
"Monthly Maintenance" "normal"
|
"Monthly Maintenance" "normal"
|
||||||
else
|
else
|
||||||
warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}"
|
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
||||||
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \
|
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
||||||
"Monthly Maintenance" "warning"
|
"Monthly Maintenance" "warning"
|
||||||
fi
|
fi
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||||
exit 0
|
exit 0
|
||||||
|
|||||||
@@ -39,9 +39,12 @@
|
|||||||
# OPERATIONAL SAFEGUARDS
|
# OPERATIONAL SAFEGUARDS
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
|
# Root check — required for the docker/system reads used in the report
|
||||||
|
# acquire_lock — prevents overlapping weekly runs
|
||||||
# detect_hosts() — MY_ID in banner and summary
|
# detect_hosts() — MY_ID in banner and summary
|
||||||
# Non-fatal steps — a failed script is logged; remaining scripts still run
|
# Non-fatal steps — a failed script is logged; remaining scripts still run
|
||||||
# Flag pass-through — --dry-run and --log forwarded to all child scripts
|
# Flag pass-through — --dry-run and --log forwarded to all child scripts
|
||||||
|
# notify() on failure — pushed only outside --dry-run, matching the runtime-mode contract below
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# CONFIGURATION
|
# CONFIGURATION
|
||||||
@@ -70,12 +73,22 @@
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
|
||||||
source "$SCRIPT_DIR/../load_config.sh"
|
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||||
|
|
||||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
|
||||||
|
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ━━━ Setup ━━━
|
||||||
|
# ==============================================================================================
|
||||||
|
if [[ "$EUID" -ne 0 ]]; then
|
||||||
|
error "Must be run as root"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
acquire_lock
|
||||||
|
|
||||||
detect_hosts
|
detect_hosts
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -84,35 +97,6 @@ detect_hosts
|
|||||||
JOB_PASS=()
|
JOB_PASS=()
|
||||||
JOB_FAIL=()
|
JOB_FAIL=()
|
||||||
|
|
||||||
run_job() {
|
|
||||||
local script_entry="$1"
|
|
||||||
local extra_dry=""
|
|
||||||
local extra_log=""
|
|
||||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
|
||||||
[[ "$ENABLE_LOGGING" == true ]] && extra_log="--log"
|
|
||||||
|
|
||||||
read -r -a script_args <<< "$script_entry"
|
|
||||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
|
||||||
local script_name
|
|
||||||
script_name=$(basename "${script_args[0]}")
|
|
||||||
local extra_args=("${script_args[@]:1}")
|
|
||||||
|
|
||||||
if [[ ! -f "$script_path" ]]; then
|
|
||||||
error "$script_name — not found at $script_path"
|
|
||||||
JOB_FAIL+=("$script_name")
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "Running: $script_name ${extra_args[*]}"
|
|
||||||
if bash "$script_path" "${extra_args[@]}" $extra_dry $extra_log; then
|
|
||||||
echo "$script_name — done ✅"
|
|
||||||
JOB_PASS+=("$script_name")
|
|
||||||
else
|
|
||||||
error "$script_name — failed (exit $?)"
|
|
||||||
JOB_FAIL+=("$script_name")
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Status ━━━
|
# ━━━ Status ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -158,7 +142,7 @@ fi
|
|||||||
for script_entry in "${COFFEE_REPORT_SCRIPTS[@]}"; do
|
for script_entry in "${COFFEE_REPORT_SCRIPTS[@]}"; do
|
||||||
[[ -z "$script_entry" ]] && continue
|
[[ -z "$script_entry" ]] && continue
|
||||||
echo ""
|
echo ""
|
||||||
run_job "$script_entry"
|
run_orch_child "$script_entry"
|
||||||
done
|
done
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -176,5 +160,10 @@ if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
|
|||||||
echo "❌ Failed: ${JOB_FAIL[*]}"
|
echo "❌ Failed: ${JOB_FAIL[*]}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [[ ${#JOB_FAIL[@]} -gt 0 && "$DRY_RUN" != true ]]; then
|
||||||
|
notify "Sunday coffee report had failures on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
|
||||||
|
"Sunday Morning Coffee Report" "warning"
|
||||||
|
fi
|
||||||
|
|
||||||
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
||||||
exit 0
|
exit 0
|
||||||
|
|||||||
@@ -5,14 +5,17 @@
|
|||||||
#
|
#
|
||||||
# PURPOSE
|
# PURPOSE
|
||||||
# ─────────────────────────────────────────────────────────────────────────────
|
# ─────────────────────────────────────────────────────────────────────────────
|
||||||
# Runs transcode_cleanup.sh then transcode_manager.sh in the correct order.
|
# Runs TRANSCODE_MANAGEMENT_SCRIPTS in order each cron cycle — this is the
|
||||||
# Replaces individual cron entries for each — this is the single cron entry.
|
# single cron entry replacing individual entries for each child script.
|
||||||
# Schedule: */7 * * * * (every 7 minutes via User Scripts plugin)
|
# Schedule: */7 * * * * (every 7 minutes)
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# OPERATIONAL MODEL
|
# OPERATIONAL MODEL
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
#
|
#
|
||||||
|
# Order driven by TRANSCODE_MANAGEMENT_SCRIPTS in master.conf.
|
||||||
|
# Default: transcode_cleanup.sh → transcode_manager.sh
|
||||||
|
#
|
||||||
# transcode_cleanup.sh
|
# transcode_cleanup.sh
|
||||||
# Removes aged segment files not open by any process. Uses lsof for O(1)
|
# Removes aged segment files not open by any process. Uses lsof for O(1)
|
||||||
# per-file active check. Triggers flip-back to ramdisk after cleanup if
|
# per-file active check. Triggers flip-back to ramdisk after cleanup if
|
||||||
@@ -36,6 +39,8 @@
|
|||||||
# Stale segment files from ended sessions inflate the ramdisk usage reading
|
# Stale segment files from ended sessions inflate the ramdisk usage reading
|
||||||
# and trigger unnecessary SSD flips even when active sessions would fit on
|
# and trigger unnecessary SSD flips even when active sessions would fit on
|
||||||
# the ramdisk. Cleanup runs first so the manager measures real current usage.
|
# the ramdisk. Cleanup runs first so the manager measures real current usage.
|
||||||
|
# Order is config-driven (TRANSCODE_MANAGEMENT_SCRIPTS) but this dependency
|
||||||
|
# is real — reordering the array changes what the manager measures.
|
||||||
#
|
#
|
||||||
# Delegated Logging
|
# Delegated Logging
|
||||||
# This orchestrator does not write its own log — transcode_manager.sh owns
|
# This orchestrator does not write its own log — transcode_manager.sh owns
|
||||||
@@ -48,8 +53,9 @@
|
|||||||
# Root check — mount and docker operations require root
|
# Root check — mount and docker operations require root
|
||||||
# acquire_lock — prevents concurrent 7-minute cycles overlapping
|
# acquire_lock — prevents concurrent 7-minute cycles overlapping
|
||||||
# detect_hosts() — aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_WARN_GB per host
|
# detect_hosts() — aliases RAMDISK_PATH, TRANSCODE_SSD, RAMDISK_WARN_GB per host
|
||||||
# --dry-run — passed through to both child scripts
|
# --dry-run — passed through to every script in TRANSCODE_MANAGEMENT_SCRIPTS
|
||||||
# Exit code — worst exit code of both scripts returned to cron
|
# Exit code — worst exit code across all scripts returned to cron
|
||||||
|
# notify() — pushed on failure, skipped in --dry-run
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# CONFIGURATION
|
# CONFIGURATION
|
||||||
@@ -57,6 +63,7 @@
|
|||||||
#
|
#
|
||||||
# master.conf
|
# master.conf
|
||||||
#
|
#
|
||||||
|
# TRANSCODE_MANAGEMENT_SCRIPTS — scripts to run, in order
|
||||||
# TRANSCODE_DAILY_LOG — daily stats log (written by transcode_manager.sh)
|
# TRANSCODE_DAILY_LOG — daily stats log (written by transcode_manager.sh)
|
||||||
# TRANSCODE_LOG_RETENTION — days to keep (trimmed by manager)
|
# TRANSCODE_LOG_RETENTION — days to keep (trimmed by manager)
|
||||||
# TRANSCODE_STATE_FILE — current state (ramdisk_used, flip_count, etc.)
|
# TRANSCODE_STATE_FILE — current state (ramdisk_used, flip_count, etc.)
|
||||||
@@ -70,13 +77,13 @@
|
|||||||
# Normal run (every 7 minutes via cron).
|
# Normal run (every 7 minutes via cron).
|
||||||
#
|
#
|
||||||
# transcode_management.sh --dry-run
|
# transcode_management.sh --dry-run
|
||||||
# Preview without changes (passed to both child scripts).
|
# Preview without changes (passed to every script in TRANSCODE_MANAGEMENT_SCRIPTS).
|
||||||
#
|
#
|
||||||
# transcode_management.sh --status
|
# transcode_management.sh --status
|
||||||
# Show configuration and current state.
|
# Show configuration and current state.
|
||||||
#
|
#
|
||||||
# transcode_management.sh --log
|
# transcode_management.sh --log
|
||||||
# Verbose output from both child scripts.
|
# Verbose output from every child script.
|
||||||
#
|
#
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|
||||||
@@ -86,9 +93,6 @@ source "$SCRIPT_DIR/../load_config.sh"
|
|||||||
|
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
CLEANUP_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_cleanup.sh"
|
|
||||||
MANAGER_SCRIPT="$SCRIPT_DIR/../Transcodes/transcode_manager.sh"
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Setup ━━━
|
# ━━━ Setup ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -122,12 +126,15 @@ if [[ "$SHOW_STATUS" == true ]]; then
|
|||||||
echo "$ICON_TIME Schedule: every 7 minutes"
|
echo "$ICON_TIME Schedule: every 7 minutes"
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ Child Scripts ━━━"
|
echo "━━━ Child Scripts ━━━"
|
||||||
[[ -f "$CLEANUP_SCRIPT" ]] && \
|
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
|
||||||
echo " $ICON_SUCCESS transcode_cleanup.sh — found" || \
|
_script_path="$SCRIPT_DIR/../$_entry"
|
||||||
echo " $ICON_ERROR transcode_cleanup.sh — NOT FOUND at $CLEANUP_SCRIPT"
|
_script_name=$(basename "$_entry")
|
||||||
[[ -f "$MANAGER_SCRIPT" ]] && \
|
if [[ -f "$_script_path" ]]; then
|
||||||
echo " $ICON_SUCCESS transcode_manager.sh — found" || \
|
echo " $ICON_SUCCESS $_script_name — found"
|
||||||
echo " $ICON_ERROR transcode_manager.sh — NOT FOUND at $MANAGER_SCRIPT"
|
else
|
||||||
|
echo " $ICON_ERROR $_script_name — NOT FOUND at $_script_path"
|
||||||
|
fi
|
||||||
|
done
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ Daily Log ━━━"
|
echo "━━━ Daily Log ━━━"
|
||||||
if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]] && [[ -s "$TRANSCODE_DAILY_LOG" ]]; then
|
if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]] && [[ -s "$TRANSCODE_DAILY_LOG" ]]; then
|
||||||
@@ -154,15 +161,13 @@ fi
|
|||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Validate Child Scripts ━━━
|
# ━━━ Validate Child Scripts ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
if [[ ! -f "$CLEANUP_SCRIPT" ]]; then
|
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
|
||||||
error "transcode_cleanup.sh not found: $CLEANUP_SCRIPT"
|
_script_path="$SCRIPT_DIR/../$_entry"
|
||||||
exit 1
|
if [[ ! -f "$_script_path" ]]; then
|
||||||
fi
|
error "$(basename "$_entry") not found: $_script_path"
|
||||||
|
exit 1
|
||||||
if [[ ! -f "$MANAGER_SCRIPT" ]]; then
|
fi
|
||||||
error "transcode_manager.sh not found: $MANAGER_SCRIPT"
|
done
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Pre-run State Snapshot ━━━
|
# ━━━ Pre-run State Snapshot ━━━
|
||||||
@@ -179,29 +184,46 @@ else
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Run Cleanup ━━━
|
# ━━━ Run Scripts ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run — no flag here
|
||||||
|
# suppresses that; it owns the log write for this cycle regardless of position ✅
|
||||||
DRY_FLAG=""
|
DRY_FLAG=""
|
||||||
[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run"
|
[[ "$DRY_RUN" == true ]] && DRY_FLAG="--dry-run"
|
||||||
|
|
||||||
_cleanup_start=$(date +%s)
|
WORST_EXIT=0
|
||||||
bash "$CLEANUP_SCRIPT" $DRY_FLAG
|
PASS_COUNT=0
|
||||||
CLEANUP_EXIT=$?
|
FAIL_NAMES=()
|
||||||
log "cleanup: $(format_duration $(( $(date +%s) - _cleanup_start ))) (exit $CLEANUP_EXIT)"
|
for _entry in "${TRANSCODE_MANAGEMENT_SCRIPTS[@]}"; do
|
||||||
|
_script_path="$SCRIPT_DIR/../$_entry"
|
||||||
|
_script_name=$(basename "$_entry" .sh)
|
||||||
|
_start=$(date +%s)
|
||||||
|
bash "$_script_path" $DRY_FLAG
|
||||||
|
_exit=$?
|
||||||
|
log "$_script_name: $(format_duration $(( $(date +%s) - _start ))) (exit $_exit)"
|
||||||
|
if [[ "$_exit" -ne 0 ]]; then
|
||||||
|
WORST_EXIT=1
|
||||||
|
FAIL_NAMES+=("$_script_name")
|
||||||
|
else
|
||||||
|
PASS_COUNT=$(( PASS_COUNT + 1 ))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Run Manager ━━━
|
# ━━━ Summary — minimal one-liner by default (7-min cadence — keep it quiet when healthy) ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# transcode_manager.sh writes to TRANSCODE_DAILY_LOG after each run
|
if [[ "$WORST_EXIT" -eq 0 ]]; then
|
||||||
# No --no-log flag here — manager owns the log write for this cycle ✅
|
echo "$ICON_SUCCESS Transcode cycle — $PASS_COUNT/${#TRANSCODE_MANAGEMENT_SCRIPTS[@]} passed"
|
||||||
_manager_start=$(date +%s)
|
else
|
||||||
bash "$MANAGER_SCRIPT" $DRY_FLAG
|
error "Transcode cycle — failed: ${FAIL_NAMES[*]}"
|
||||||
MANAGER_EXIT=$?
|
if [[ "$DRY_RUN" != true ]]; then
|
||||||
log "manager: $(format_duration $(( $(date +%s) - _manager_start ))) (exit $MANAGER_EXIT)"
|
notify "Transcode management failure on $(hostname) ($MY_ID) — ${FAIL_NAMES[*]}" \
|
||||||
|
"Transcode Management" "warning"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Exit ━━━
|
# ━━━ Exit ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# Return worst exit code — caller knows if either script failed
|
# Return worst exit code — caller knows if any script failed
|
||||||
[[ "$CLEANUP_EXIT" -ne 0 || "$MANAGER_EXIT" -ne 0 ]] && exit 1
|
exit "$WORST_EXIT"
|
||||||
exit 0
|
|
||||||
@@ -193,7 +193,7 @@ run_watchdog() {
|
|||||||
|
|
||||||
local extra_args=()
|
local extra_args=()
|
||||||
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
[[ "$DRY_RUN" == true ]] && extra_args+=("--dry-run")
|
||||||
[[ "$VERBOSE" == true ]] && extra_args+=("--log")
|
[[ "$ENABLE_LOGGING" == true ]] && extra_args+=("--log")
|
||||||
|
|
||||||
local _ws
|
local _ws
|
||||||
_ws=$(date +%s)
|
_ws=$(date +%s)
|
||||||
@@ -234,18 +234,23 @@ if [[ "${WATCHDOG_ORCHESTRATOR_HEARTBEAT:-true}" == true ]]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Summary — only shown on failures or --log ━━━
|
# ━━━ Summary — minimal one-liner by default, full breakdown on failure or --log ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
if [[ "${#FAIL[@]}" -gt 0 || "$VERBOSE" == true ]]; then
|
if [[ "${#FAIL[@]}" -gt 0 || "$ENABLE_LOGGING" == true ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
echo "━━━━━ $ICON_SUMMARY WATCHDOG CYCLE — $MY_ID — $(date '+%H:%M:%S') ━━━━━"
|
||||||
for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done
|
for p in "${PASS[@]}"; do log " $ICON_DONE $p"; done
|
||||||
for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
for f in "${FAIL[@]}"; do error " $ICON_ERROR $f"; done
|
||||||
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
echo "$ICON_TIME Duration: $(format_duration $DURATION)"
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
else
|
||||||
if [[ "${#FAIL[@]}" -gt 0 ]]; then
|
echo "$ICON_DONE Watchdog cycle — ${#PASS[@]}/${#WATCHDOG_ORCHESTRATOR_SCRIPTS[@]} passed ($(format_duration $DURATION))"
|
||||||
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
|
||||||
"Watchdog Orchestrator" "warning"
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [[ "${#FAIL[@]}" -gt 0 ]]; then
|
||||||
|
notify "Watchdog cycle failure on $(hostname) ($MY_ID) — ${FAIL[*]}" \
|
||||||
|
"Watchdog Orchestrator" "warning"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit 0
|
||||||
|
|||||||
@@ -84,8 +84,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|||||||
|
|
||||||
source "$SCRIPT_DIR/../load_config.sh"
|
source "$SCRIPT_DIR/../load_config.sh"
|
||||||
|
|
||||||
RSYNC_SCRIPT="$SCRIPT_DIR/../Rsync/rsync.sh"
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
SCRIPTS_ROOT="$SCRIPT_DIR/.."
|
RSYNC_SCRIPT="$ECOSYSTEM_ROOT/Rsync/rsync.sh"
|
||||||
|
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
@@ -123,35 +123,6 @@ read -r -a MAINTENANCE_CONTAINERS <<< \
|
|||||||
|
|
||||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be stopped, no sync, no updates"
|
||||||
|
|
||||||
# ── Helper — run a post-sync maintenance script ────────────────────────────────────────────────
|
|
||||||
run_job() {
|
|
||||||
local script_entry="$1"
|
|
||||||
local extra_dry=""
|
|
||||||
[[ "$DRY_RUN" == true ]] && extra_dry="--dry-run"
|
|
||||||
|
|
||||||
read -r -a script_args <<< "$script_entry"
|
|
||||||
local script_path="$SCRIPTS_ROOT/${script_args[0]}"
|
|
||||||
local script_name
|
|
||||||
script_name=$(basename "${script_args[0]}")
|
|
||||||
local extra_args=("${script_args[@]:1}")
|
|
||||||
|
|
||||||
if [[ ! -f "$script_path" ]]; then
|
|
||||||
error "$script_name — not found at $script_path"
|
|
||||||
JOB_FAIL+=("$script_name")
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "Running: $script_name ${extra_args[*]}"
|
|
||||||
# shellcheck disable=SC2086
|
|
||||||
if bash "$script_path" "${extra_args[@]}" $extra_dry; then
|
|
||||||
echo "$script_name — done ✅"
|
|
||||||
JOB_PASS+=("$script_name ${extra_args[*]}")
|
|
||||||
else
|
|
||||||
error "$script_name — failed (exit $?)"
|
|
||||||
JOB_FAIL+=("$script_name ${extra_args[*]}")
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ━━━ Status ━━━
|
# ━━━ Status ━━━
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
@@ -390,7 +361,7 @@ if [[ ${#WEEKLY_MAINTENANCE_SCRIPTS[@]} -gt 0 ]]; then
|
|||||||
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
for script_entry in "${WEEKLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||||
[[ -z "$script_entry" ]] && continue
|
[[ -z "$script_entry" ]] && continue
|
||||||
echo ""
|
echo ""
|
||||||
run_job "$script_entry"
|
run_orch_child "$script_entry"
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|||||||
@@ -1537,6 +1537,64 @@ _release_rsync_on_exit() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── ORCHESTRATOR CHILD EXECUTION ──────────────────────────────────────────────────────────────
|
||||||
|
# ==============================================================================================
|
||||||
|
# Shared child-script runner for Orchestrators/*.sh. Replaces the run_job()/run_watchdog()
|
||||||
|
# copies that used to be hand-duplicated (with small inconsistencies) into each orchestrator —
|
||||||
|
# one implementation now, one place to fix a bug or extend behavior (e.g. per-child logging).
|
||||||
|
#
|
||||||
|
# Usage — caller declares its own tracking arrays before the loop:
|
||||||
|
# JOB_PASS=()
|
||||||
|
# JOB_FAIL=()
|
||||||
|
# for entry in "${SOME_SCRIPTS[@]}"; do
|
||||||
|
# run_orch_child "$entry"
|
||||||
|
# done
|
||||||
|
#
|
||||||
|
# $entry is "relative/path.sh [extra args...]" — the same format every master.conf
|
||||||
|
# script-list array already uses. Resolved against $ECOSYSTEM_ROOT, which the caller
|
||||||
|
# must set before calling this (the absolute repo root — "$(cd "$SCRIPT_DIR/.." && pwd)").
|
||||||
|
#
|
||||||
|
# --dry-run / --log are threaded down automatically from $DRY_RUN / $ENABLE_LOGGING —
|
||||||
|
# never $VERBOSE, which nothing in this codebase ever assigns.
|
||||||
|
|
||||||
|
run_orch_child() {
|
||||||
|
local entry="$1"
|
||||||
|
local script_args script_path script_name label extra_args run_args
|
||||||
|
|
||||||
|
read -r -a script_args <<< "$entry"
|
||||||
|
script_path="$ECOSYSTEM_ROOT/${script_args[0]}"
|
||||||
|
script_name=$(basename "${script_args[0]}")
|
||||||
|
extra_args=("${script_args[@]:1}")
|
||||||
|
label="$script_name"
|
||||||
|
[[ -n "${extra_args[*]}" ]] && label="$script_name ${extra_args[*]}"
|
||||||
|
|
||||||
|
if [[ ! -f "$script_path" ]]; then
|
||||||
|
error "$script_name — not found at $script_path"
|
||||||
|
JOB_FAIL+=("$label")
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
[[ ! -x "$script_path" ]] && chmod +x "$script_path"
|
||||||
|
|
||||||
|
run_args=("${extra_args[@]}")
|
||||||
|
[[ "$DRY_RUN" == true ]] && run_args+=("--dry-run")
|
||||||
|
[[ "$ENABLE_LOGGING" == true ]] && run_args+=("--log")
|
||||||
|
|
||||||
|
local _start _ec
|
||||||
|
_start=$(date +%s)
|
||||||
|
log "Running: $label"
|
||||||
|
if bash "$script_path" "${run_args[@]}"; then
|
||||||
|
log "$script_name — done in $(format_duration $(( $(date +%s) - _start )))"
|
||||||
|
JOB_PASS+=("$label")
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
_ec=$?
|
||||||
|
error "$label — failed (exit $_ec, $(format_duration $(( $(date +%s) - _start ))))"
|
||||||
|
JOB_FAIL+=("$label")
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
# ── PATH TRANSLATION ──────────────────────────────────────────────────────────────────────────
|
# ── PATH TRANSLATION ──────────────────────────────────────────────────────────────────────────
|
||||||
# ==============================================================================================
|
# ==============================================================================================
|
||||||
|
|||||||
Reference in New Issue
Block a user