Gmer4Lfe
2a062e5140
Every orchestrator invoked its children differently — four near-duplicate run_job() copies, a differently-shaped run_watchdog(), or plain inline bash calls, each with its own take on path resolution, pass/fail naming, and dry-run threading. Extracted one shared run_orch_child() into common.sh so there's a single place to fix or extend this behavior going forward. Along the way: watchdog_orchestrator.sh and monthly_maintenance.sh were checking $VERBOSE, a variable nothing in the codebase ever assigns, so --log silently did nothing beyond basic logging on those two. Fixed to $ENABLE_LOGGING. watchdog_orchestrator.sh and array_started.sh had no trailing exit, so their exit codes reflected whatever the last command happened to return rather than actual success/failure. transcode_management.sh had no failure notification and no summary at all. Also made transcode_management.sh's two-script pipeline config-driven (TRANSCODE_MANAGEMENT_SCRIPTS in master.conf) instead of hardcoded, for room to extend it later without editing the orchestrator itself.
293 lines
12 KiB
Bash
Executable File
293 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ========================= Monthly Maintenance Orchestrator ===================================
|
|
# ==============================================================================================
|
|
#
|
|
# PURPOSE
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Uptime-triggered monthly maintenance — runs heavy tasks that need a stable,
|
|
# settled system. Schedule: 0 0 15 * * (15th of each month at midnight)
|
|
# Fires only when BOTH gates pass:
|
|
# 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days
|
|
# 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run)
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL MODEL
|
|
# ==============================================================================================
|
|
#
|
|
# Runs MONTHLY_MAINTENANCE_SCRIPTS sequentially when both gates pass.
|
|
# Silent exit 0 when either gate is not met — only outputs when maintenance fires.
|
|
# If uptime or interval gate is not met on the 15th, the run is skipped until
|
|
# next month.
|
|
#
|
|
# STATE FILE
|
|
# MONTHLY_LAST_RUN_FILE lives on /boot/config — survives reboots, available
|
|
# before the array starts. Written after each run (pass or partial fail).
|
|
# Format: Unix timestamp. A reboot does NOT reset the last-run state — the
|
|
# interval gate survives independently of the uptime gate.
|
|
#
|
|
# ==============================================================================================
|
|
# DESIGN PRINCIPLES
|
|
# ==============================================================================================
|
|
#
|
|
# Uptime Gate Ensures Stability
|
|
# A scheduled reboot resets uptime. Monthly tasks (ZFS scrub, SMART long test)
|
|
# need a stable, settled system — not one that just rebooted. Both gates must
|
|
# pass before maintenance fires, ensuring the server has been healthy for a
|
|
# full month.
|
|
#
|
|
# State Survives Reboots
|
|
# MONTHLY_LAST_RUN_FILE is on /boot/config (USB flash), not on the array.
|
|
# It is always available regardless of array state, so the interval gate is
|
|
# never lost to a reboot.
|
|
#
|
|
# ==============================================================================================
|
|
# OPERATIONAL SAFEGUARDS
|
|
# ==============================================================================================
|
|
#
|
|
# Root check — ZFS scrub, SMART tests require root
|
|
# acquire_lock — prevents concurrent monthly runs
|
|
# detect_hosts() — MY_ID in notifications and logs
|
|
# Uptime gate — MONTHLY_UPTIME_THRESHOLD_DAYS must be met
|
|
# Interval gate — MONTHLY_RUN_INTERVAL_DAYS since last run must be met
|
|
# --force flag — bypasses both gates for manual override
|
|
#
|
|
# ==============================================================================================
|
|
# CONFIGURATION
|
|
# ==============================================================================================
|
|
#
|
|
# master.conf
|
|
#
|
|
# MONTHLY_MAINTENANCE_SCRIPTS — ordered list of scripts to run
|
|
# MONTHLY_UPTIME_THRESHOLD_DAYS — minimum uptime in days before maintenance fires
|
|
# MONTHLY_RUN_INTERVAL_DAYS — minimum days since last run before running again
|
|
# MONTHLY_LAST_RUN_FILE — state file path (/boot/config — survives reboots)
|
|
#
|
|
# ==============================================================================================
|
|
# RUNTIME MODES
|
|
# ==============================================================================================
|
|
#
|
|
# monthly_maintenance.sh
|
|
# Normal run — uptime + interval gates enforced.
|
|
#
|
|
# monthly_maintenance.sh --dry-run
|
|
# Preview gate state and scripts without running.
|
|
#
|
|
# monthly_maintenance.sh --status
|
|
# Show gate state, last run, and configured scripts.
|
|
#
|
|
# monthly_maintenance.sh --force
|
|
# Bypass uptime + interval gates (manual override).
|
|
#
|
|
# monthly_maintenance.sh --log
|
|
# Verbose output.
|
|
#
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
|
|
source "$ECOSYSTEM_ROOT/load_config.sh"
|
|
|
|
# ── Parse --force before common parse_args ────────────────────────────────────────────────────
|
|
FORCE_RUN=false
|
|
FILTERED_ARGS=()
|
|
for _arg in "$@"; do
|
|
if [[ "$_arg" == "--force" ]]; then
|
|
FORCE_RUN=true
|
|
else
|
|
FILTERED_ARGS+=("$_arg")
|
|
fi
|
|
done
|
|
parse_args "${FILTERED_ARGS[@]}"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
acquire_lock
|
|
|
|
detect_hosts
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scripts will be executed"
|
|
[[ "$FORCE_RUN" == true ]] && warn "FORCE — uptime and interval gates bypassed"
|
|
|
|
# Defaults — overridden by master.conf values
|
|
MONTHLY_UPTIME_THRESHOLD_DAYS="${MONTHLY_UPTIME_THRESHOLD_DAYS:-30}"
|
|
MONTHLY_RUN_INTERVAL_DAYS="${MONTHLY_RUN_INTERVAL_DAYS:-30}"
|
|
MONTHLY_LAST_RUN_FILE="${MONTHLY_LAST_RUN_FILE:-${STATE_DIR:-/tmp}/monthly_maintenance_last_run.db}"
|
|
|
|
UPTIME_THRESHOLD_SECS=$(( MONTHLY_UPTIME_THRESHOLD_DAYS * 86400 ))
|
|
INTERVAL_SECS=$(( MONTHLY_RUN_INTERVAL_DAYS * 86400 ))
|
|
UPTIME_SECS=$(awk '{print int($1)}' /proc/uptime)
|
|
NOW=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Gate Evaluation ━━━
|
|
# ==============================================================================================
|
|
_uptime_days=$(( UPTIME_SECS / 86400 ))
|
|
_uptime_hrs=$(( (UPTIME_SECS % 86400) / 3600 ))
|
|
|
|
UPTIME_GATE_PASS=false
|
|
if [[ "$UPTIME_SECS" -ge "$UPTIME_THRESHOLD_SECS" ]]; then
|
|
UPTIME_GATE_PASS=true
|
|
fi
|
|
|
|
INTERVAL_GATE_PASS=false
|
|
LAST_RUN=0
|
|
DAYS_SINCE_LAST_RUN="never"
|
|
if [[ -f "$MONTHLY_LAST_RUN_FILE" ]]; then
|
|
LAST_RUN=$(cat "$MONTHLY_LAST_RUN_FILE" 2>/dev/null || echo 0)
|
|
ELAPSED=$(( NOW - LAST_RUN ))
|
|
DAYS_SINCE=$(( ELAPSED / 86400 ))
|
|
DAYS_SINCE_LAST_RUN="${DAYS_SINCE}d"
|
|
if [[ "$ELAPSED" -ge "$INTERVAL_SECS" ]]; then
|
|
INTERVAL_GATE_PASS=true
|
|
fi
|
|
else
|
|
INTERVAL_GATE_PASS=true # never run
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo ""
|
|
echo "── Gates ──"
|
|
|
|
if [[ "$UPTIME_GATE_PASS" == true ]]; then
|
|
echo " $ICON_DONE Uptime: ${_uptime_days}d ${_uptime_hrs}h (threshold: ${MONTHLY_UPTIME_THRESHOLD_DAYS}d) ✅"
|
|
else
|
|
echo " $ICON_WARN Uptime: ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d needed — NOT met"
|
|
fi
|
|
|
|
if [[ "$INTERVAL_GATE_PASS" == true ]]; then
|
|
echo " $ICON_DONE Interval: last run ${DAYS_SINCE_LAST_RUN} ago (threshold: ${MONTHLY_RUN_INTERVAL_DAYS}d) ✅"
|
|
else
|
|
echo " $ICON_WARN Interval: last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d needed — NOT met"
|
|
fi
|
|
|
|
if [[ "$LAST_RUN" -gt 0 ]]; then
|
|
echo " Last run: $(date -d "@$LAST_RUN" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$LAST_RUN" '+%Y-%m-%d %H:%M:%S')"
|
|
else
|
|
echo " Last run: never"
|
|
fi
|
|
|
|
echo ""
|
|
echo "── Scripts ──"
|
|
if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then
|
|
echo " (none configured — add to MONTHLY_MAINTENANCE_SCRIPTS in master.conf)"
|
|
else
|
|
for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$entry" ]] && continue
|
|
read -r -a parts <<< "$entry"
|
|
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
|
|
script_name=$(basename "${parts[0]}")
|
|
if [[ -f "$script_path" ]]; then
|
|
[[ -x "$script_path" ]] && echo " $ICON_DONE $script_name" || echo " $ICON_WARN $script_name (not executable)"
|
|
else
|
|
echo " $ICON_ERROR $script_name — NOT FOUND: $script_path"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
echo ""
|
|
echo " Schedule: 0 0 15 * * (15th of each month at midnight)"
|
|
echo " Force flag: --force bypasses both gates"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Gate Check — Silent Exit When Not Due ━━━
|
|
# ==============================================================================================
|
|
if [[ "$FORCE_RUN" != true ]]; then
|
|
if [[ "$UPTIME_GATE_PASS" != true ]]; then
|
|
echo "Uptime gate not met — ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d — no-op"
|
|
exit 0
|
|
fi
|
|
if [[ "$INTERVAL_GATE_PASS" != true ]]; then
|
|
echo "Interval gate not met — last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d — no-op"
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Pre-flight ━━━
|
|
# ==============================================================================================
|
|
if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then
|
|
warn "No scripts in MONTHLY_MAINTENANCE_SCRIPTS — nothing to run"
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_START Monthly Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_HOST Uptime: ${_uptime_days}d ${_uptime_hrs}h | Last run: ${DAYS_SINCE_LAST_RUN} ago${FORCE_RUN:+ | FORCED}"
|
|
echo "$ICON_GEAR Scripts: ${#MONTHLY_MAINTENANCE_SCRIPTS[@]}"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
JOB_PASS=()
|
|
JOB_FAIL=()
|
|
STEP=0
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Run Sequence ━━━
|
|
# ==============================================================================================
|
|
for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$entry" ]] && continue
|
|
(( STEP++ ))
|
|
|
|
read -r -a parts <<< "$entry"
|
|
script_name=$(basename "${parts[0]}")
|
|
extra_args=("${parts[@]:1}")
|
|
|
|
echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
|
|
run_orch_child "$entry"
|
|
echo ""
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Record Last Run ━━━
|
|
# ==============================================================================================
|
|
# Write timestamp whether we passed or partially failed — prevents hammering broken scripts.
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
echo "$NOW" > "$MONTHLY_LAST_RUN_FILE"
|
|
log "Last run recorded: $(date -d "@$NOW" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$NOW" '+%Y-%m-%d %H:%M:%S')"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
[[ ${#JOB_PASS[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${JOB_PASS[*]}"
|
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${JOB_FAIL[*]}"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no changes made"
|
|
elif [[ ${#JOB_FAIL[@]} -eq 0 ]]; then
|
|
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
|
notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
|
"Monthly Maintenance" "normal"
|
|
else
|
|
warn "Status: ${#JOB_FAIL[@]} step(s) failed — ${JOB_FAIL[*]}"
|
|
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#JOB_FAIL[@]} step(s) failed: ${JOB_FAIL[*]}" \
|
|
"Monthly Maintenance" "warning"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
|
|
exit 0
|