add monthly_maintenance.sh — uptime-triggered orchestrator
Two-gate design: server must have ≥30 days uptime AND last run must be ≥30 days ago. Both gates must pass before any scripts fire. Called daily at 3am via cron — script self-gates, calling more often is safe. State file on /boot/config (survives reboots): the interval gate is independent of uptime. A reboot resets uptime but does not reset when maintenance last ran — both gates must independently pass. MONTHLY_MAINTENANCE_SCRIPTS added to master.conf in ORCHESTRATORS section. zfs_pool_scrub.sh and smart_long_test.sh listed but commented (neither script exists yet). Also commits mesh_monitor.sh move to Monitors/ that was staged from prior session. Supports --force to bypass both gates for manual runs.
This commit is contained in:
Gmer4Lfe
@@ -0,0 +1,282 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ========================= Monthly Maintenance Orchestrator ===================================
|
||||
# ==============================================================================================
|
||||
# Uptime-triggered monthly maintenance — runs heavy tasks that need a stable, settled system.
|
||||
# Fires only when BOTH gates pass:
|
||||
# 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days
|
||||
# 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run)
|
||||
#
|
||||
# ── WHY UPTIME-TRIGGERED, NOT CRON ───────────────────────────────────────────────────────────
|
||||
# A scheduled reboot resets uptime. Monthly tasks (ZFS scrub, SMART long test) need a
|
||||
# stable, settled system — not one that just rebooted. Uptime-gating ensures maintenance
|
||||
# only runs after the server has been healthy for a full month, never immediately post-boot.
|
||||
# The daily cron is just the trigger mechanism. The uptime and interval checks inside
|
||||
# the script are what enforce the monthly cadence.
|
||||
#
|
||||
# ── HOW TO CALL ──────────────────────────────────────────────────────────────────────────────
|
||||
# Cron: 0 3 * * * — daily 3am check. Script self-gates — calling it daily is safe.
|
||||
# Silent exit 0 when either gate is not met. Only outputs when maintenance actually fires.
|
||||
#
|
||||
# ── STATE FILE ────────────────────────────────────────────────────────────────────────────────
|
||||
# MONTHLY_LAST_RUN_FILE — /boot/config — survives reboots, available before array starts.
|
||||
# Written after each run (pass or partial fail). Format: Unix timestamp.
|
||||
# A reboot does NOT reset the last-run state — the interval gate survives independently
|
||||
# of the uptime gate. Both must pass before maintenance fires again.
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# MONTHLY_MAINTENANCE_SCRIPTS — ordered list of scripts to run
|
||||
# MONTHLY_UPTIME_THRESHOLD_DAYS — minimum uptime in days before maintenance fires
|
||||
# MONTHLY_RUN_INTERVAL_DAYS — minimum days since last run before running again
|
||||
# MONTHLY_LAST_RUN_FILE — state file path — /boot/config, survives reboots
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# monthly_maintenance.sh — normal run (uptime + interval gates enforced)
|
||||
# monthly_maintenance.sh --dry-run — preview gate state and scripts without running
|
||||
# monthly_maintenance.sh --status — show gate state, last run, and configured scripts
|
||||
# monthly_maintenance.sh --force — bypass uptime + interval gates (manual override)
|
||||
# monthly_maintenance.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
|
||||
source "$ECOSYSTEM_ROOT/load_config.sh"
|
||||
|
||||
# ── Parse --force before common parse_args ────────────────────────────────────────────────────
|
||||
FORCE_RUN=false
|
||||
FILTERED_ARGS=()
|
||||
for _arg in "$@"; do
|
||||
if [[ "$_arg" == "--force" ]]; then
|
||||
FORCE_RUN=true
|
||||
else
|
||||
FILTERED_ARGS+=("$_arg")
|
||||
fi
|
||||
done
|
||||
parse_args "${FILTERED_ARGS[@]}"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
detect_hosts
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scripts will be executed"
|
||||
[[ "$FORCE_RUN" == true ]] && warn "FORCE — uptime and interval gates bypassed"
|
||||
|
||||
# Defaults — overridden by master.conf values
|
||||
MONTHLY_UPTIME_THRESHOLD_DAYS="${MONTHLY_UPTIME_THRESHOLD_DAYS:-30}"
|
||||
MONTHLY_RUN_INTERVAL_DAYS="${MONTHLY_RUN_INTERVAL_DAYS:-30}"
|
||||
MONTHLY_LAST_RUN_FILE="${MONTHLY_LAST_RUN_FILE:-/boot/config/monthly_maintenance_last_run.db}"
|
||||
|
||||
UPTIME_THRESHOLD_SECS=$(( MONTHLY_UPTIME_THRESHOLD_DAYS * 86400 ))
|
||||
INTERVAL_SECS=$(( MONTHLY_RUN_INTERVAL_DAYS * 86400 ))
|
||||
UPTIME_SECS=$(awk '{print int($1)}' /proc/uptime)
|
||||
NOW=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Gate Evaluation ━━━
|
||||
# ==============================================================================================
|
||||
_uptime_days=$(( UPTIME_SECS / 86400 ))
|
||||
_uptime_hrs=$(( (UPTIME_SECS % 86400) / 3600 ))
|
||||
|
||||
UPTIME_GATE_PASS=false
|
||||
if [[ "$UPTIME_SECS" -ge "$UPTIME_THRESHOLD_SECS" ]]; then
|
||||
UPTIME_GATE_PASS=true
|
||||
fi
|
||||
|
||||
INTERVAL_GATE_PASS=false
|
||||
LAST_RUN=0
|
||||
DAYS_SINCE_LAST_RUN="never"
|
||||
if [[ -f "$MONTHLY_LAST_RUN_FILE" ]]; then
|
||||
LAST_RUN=$(cat "$MONTHLY_LAST_RUN_FILE" 2>/dev/null || echo 0)
|
||||
ELAPSED=$(( NOW - LAST_RUN ))
|
||||
DAYS_SINCE=$(( ELAPSED / 86400 ))
|
||||
DAYS_SINCE_LAST_RUN="${DAYS_SINCE}d"
|
||||
if [[ "$ELAPSED" -ge "$INTERVAL_SECS" ]]; then
|
||||
INTERVAL_GATE_PASS=true
|
||||
fi
|
||||
else
|
||||
INTERVAL_GATE_PASS=true # never run
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo ""
|
||||
echo "── Gates ──"
|
||||
|
||||
if [[ "$UPTIME_GATE_PASS" == true ]]; then
|
||||
echo " $ICON_DONE Uptime: ${_uptime_days}d ${_uptime_hrs}h (threshold: ${MONTHLY_UPTIME_THRESHOLD_DAYS}d) ✅"
|
||||
else
|
||||
echo " $ICON_WARN Uptime: ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d needed — NOT met"
|
||||
fi
|
||||
|
||||
if [[ "$INTERVAL_GATE_PASS" == true ]]; then
|
||||
echo " $ICON_DONE Interval: last run ${DAYS_SINCE_LAST_RUN} ago (threshold: ${MONTHLY_RUN_INTERVAL_DAYS}d) ✅"
|
||||
else
|
||||
echo " $ICON_WARN Interval: last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d needed — NOT met"
|
||||
fi
|
||||
|
||||
if [[ "$LAST_RUN" -gt 0 ]]; then
|
||||
echo " Last run: $(date -d "@$LAST_RUN" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$LAST_RUN" '+%Y-%m-%d %H:%M:%S')"
|
||||
else
|
||||
echo " Last run: never"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "── Scripts ──"
|
||||
if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then
|
||||
echo " (none configured — add to MONTHLY_MAINTENANCE_SCRIPTS in master.conf)"
|
||||
else
|
||||
for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
read -r -a parts <<< "$entry"
|
||||
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
|
||||
script_name=$(basename "${parts[0]}")
|
||||
if [[ -f "$script_path" ]]; then
|
||||
[[ -x "$script_path" ]] && echo " $ICON_DONE $script_name" || echo " $ICON_WARN $script_name (not executable)"
|
||||
else
|
||||
echo " $ICON_ERROR $script_name — NOT FOUND: $script_path"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo " Schedule: 0 3 * * * (daily check — script self-gates)"
|
||||
echo " Force flag: --force bypasses both gates"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Gate Check — Silent Exit When Not Due ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$FORCE_RUN" != true ]]; then
|
||||
if [[ "$UPTIME_GATE_PASS" != true ]]; then
|
||||
log "Uptime gate — ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d — not yet due"
|
||||
exit 0
|
||||
fi
|
||||
if [[ "$INTERVAL_GATE_PASS" != true ]]; then
|
||||
log "Interval gate — last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d — not yet due"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Pre-flight ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then
|
||||
warn "No scripts in MONTHLY_MAINTENANCE_SCRIPTS — nothing to run"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━ $ICON_START Monthly Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_HOST Uptime: ${_uptime_days}d ${_uptime_hrs}h | Last run: ${DAYS_SINCE_LAST_RUN} ago${FORCE_RUN:+ | FORCED}"
|
||||
echo "$ICON_GEAR Scripts: ${#MONTHLY_MAINTENANCE_SCRIPTS[@]}"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
PASSED=()
|
||||
FAILED=()
|
||||
STEP=0
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Run Sequence ━━━
|
||||
# ==============================================================================================
|
||||
for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
|
||||
[[ -z "$entry" ]] && continue
|
||||
(( STEP++ ))
|
||||
|
||||
read -r -a parts <<< "$entry"
|
||||
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
|
||||
script_name=$(basename "${parts[0]}")
|
||||
extra_args=("${parts[@]:1}")
|
||||
|
||||
echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
|
||||
|
||||
if [[ ! -f "$script_path" ]]; then
|
||||
error "$script_name — not found at $script_path"
|
||||
FAILED+=("$script_name")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ ! -x "$script_path" ]]; then
|
||||
warn "$script_name — not executable, fixing..."
|
||||
chmod +x "$script_path" || {
|
||||
error "$script_name — chmod +x failed"
|
||||
FAILED+=("$script_name")
|
||||
echo ""
|
||||
continue
|
||||
}
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would run: $script_name${extra_args:+ ${extra_args[*]}}"
|
||||
PASSED+=("$script_name")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
local_args=()
|
||||
[[ "$VERBOSE" == true ]] && local_args+=("--log")
|
||||
|
||||
if bash "$script_path" "${extra_args[@]}" "${local_args[@]}"; then
|
||||
log "$script_name — done ✅"
|
||||
PASSED+=("$script_name")
|
||||
else
|
||||
warn "$script_name — failed (exit $?) — continuing to next step"
|
||||
FAILED+=("$script_name")
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Record Last Run ━━━
|
||||
# ==============================================================================================
|
||||
# Write timestamp whether we passed or partially failed — prevents hammering broken scripts.
|
||||
if [[ "$DRY_RUN" != true ]]; then
|
||||
echo "$NOW" > "$MONTHLY_LAST_RUN_FILE"
|
||||
log "Last run recorded: $(date -d "@$NOW" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$NOW" '+%Y-%m-%d %H:%M:%S')"
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
||||
[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
||||
notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
||||
"Monthly Maintenance" "normal"
|
||||
else
|
||||
warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}"
|
||||
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \
|
||||
"Monthly Maintenance" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user