282 lines
13 KiB
Bash
Executable File
282 lines
13 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================================
|
|
# ========================= Monthly Maintenance Orchestrator ===================================
|
|
# ==============================================================================================
|
|
# Uptime-triggered monthly maintenance — runs heavy tasks that need a stable, settled system.
|
|
# Fires only when BOTH gates pass:
|
|
# 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days
|
|
# 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run)
|
|
#
|
|
# ── WHY UPTIME-GATED ─────────────────────────────────────────────────────────────────────────
|
|
# A scheduled reboot resets uptime. Monthly tasks (ZFS scrub, SMART long test) need a
|
|
# stable, settled system — not one that just rebooted. Uptime-gating ensures maintenance
|
|
# only runs after the server has been healthy for a full month, never immediately post-boot.
|
|
# If uptime or interval gate is not met on the 15th, the run is skipped until next month.
|
|
#
|
|
# ── HOW TO CALL ──────────────────────────────────────────────────────────────────────────────
|
|
# Schedule: 0 0 15 * * (15th of each month at midnight)
|
|
# Silent exit 0 when either gate is not met. Only outputs when maintenance actually fires.
|
|
#
|
|
# ── STATE FILE ────────────────────────────────────────────────────────────────────────────────
|
|
# MONTHLY_LAST_RUN_FILE — /boot/config — survives reboots, available before array starts.
|
|
# Written after each run (pass or partial fail). Format: Unix timestamp.
|
|
# A reboot does NOT reset the last-run state — the interval gate survives independently
|
|
# of the uptime gate. Both must pass before maintenance fires again.
|
|
#
|
|
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
|
# MONTHLY_MAINTENANCE_SCRIPTS — ordered list of scripts to run
|
|
# MONTHLY_UPTIME_THRESHOLD_DAYS — minimum uptime in days before maintenance fires
|
|
# MONTHLY_RUN_INTERVAL_DAYS — minimum days since last run before running again
|
|
# MONTHLY_LAST_RUN_FILE — state file path — /boot/config, survives reboots
|
|
#
|
|
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
|
# monthly_maintenance.sh — normal run (uptime + interval gates enforced)
|
|
# monthly_maintenance.sh --dry-run — preview gate state and scripts without running
|
|
# monthly_maintenance.sh --status — show gate state, last run, and configured scripts
|
|
# monthly_maintenance.sh --force — bypass uptime + interval gates (manual override)
|
|
# monthly_maintenance.sh --log — verbose output
|
|
# ==============================================================================================
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
|
|
source "$ECOSYSTEM_ROOT/load_config.sh"
|
|
|
|
# ── Parse --force before common parse_args ────────────────────────────────────────────────────
|
|
FORCE_RUN=false
|
|
FILTERED_ARGS=()
|
|
for _arg in "$@"; do
|
|
if [[ "$_arg" == "--force" ]]; then
|
|
FORCE_RUN=true
|
|
else
|
|
FILTERED_ARGS+=("$_arg")
|
|
fi
|
|
done
|
|
parse_args "${FILTERED_ARGS[@]}"
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Setup ━━━
|
|
# ==============================================================================================
|
|
if [[ "$EUID" -ne 0 ]]; then
|
|
error "Must be run as root"
|
|
exit 1
|
|
fi
|
|
|
|
acquire_lock
|
|
|
|
detect_hosts
|
|
|
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scripts will be executed"
|
|
[[ "$FORCE_RUN" == true ]] && warn "FORCE — uptime and interval gates bypassed"
|
|
|
|
# Defaults — overridden by master.conf values
|
|
MONTHLY_UPTIME_THRESHOLD_DAYS="${MONTHLY_UPTIME_THRESHOLD_DAYS:-30}"
|
|
MONTHLY_RUN_INTERVAL_DAYS="${MONTHLY_RUN_INTERVAL_DAYS:-30}"
|
|
MONTHLY_LAST_RUN_FILE="${MONTHLY_LAST_RUN_FILE:-${STATE_DIR:-/tmp}/monthly_maintenance_last_run.db}"
|
|
|
|
UPTIME_THRESHOLD_SECS=$(( MONTHLY_UPTIME_THRESHOLD_DAYS * 86400 ))
|
|
INTERVAL_SECS=$(( MONTHLY_RUN_INTERVAL_DAYS * 86400 ))
|
|
UPTIME_SECS=$(awk '{print int($1)}' /proc/uptime)
|
|
NOW=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Gate Evaluation ━━━
|
|
# ==============================================================================================
|
|
_uptime_days=$(( UPTIME_SECS / 86400 ))
|
|
_uptime_hrs=$(( (UPTIME_SECS % 86400) / 3600 ))
|
|
|
|
UPTIME_GATE_PASS=false
|
|
if [[ "$UPTIME_SECS" -ge "$UPTIME_THRESHOLD_SECS" ]]; then
|
|
UPTIME_GATE_PASS=true
|
|
fi
|
|
|
|
INTERVAL_GATE_PASS=false
|
|
LAST_RUN=0
|
|
DAYS_SINCE_LAST_RUN="never"
|
|
if [[ -f "$MONTHLY_LAST_RUN_FILE" ]]; then
|
|
LAST_RUN=$(cat "$MONTHLY_LAST_RUN_FILE" 2>/dev/null || echo 0)
|
|
ELAPSED=$(( NOW - LAST_RUN ))
|
|
DAYS_SINCE=$(( ELAPSED / 86400 ))
|
|
DAYS_SINCE_LAST_RUN="${DAYS_SINCE}d"
|
|
if [[ "$ELAPSED" -ge "$INTERVAL_SECS" ]]; then
|
|
INTERVAL_GATE_PASS=true
|
|
fi
|
|
else
|
|
INTERVAL_GATE_PASS=true # never run
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Status ━━━
|
|
# ==============================================================================================
|
|
if [[ "$SHOW_STATUS" == true ]]; then
|
|
echo ""
|
|
echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE STATUS ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo ""
|
|
echo "── Gates ──"
|
|
|
|
if [[ "$UPTIME_GATE_PASS" == true ]]; then
|
|
echo " $ICON_DONE Uptime: ${_uptime_days}d ${_uptime_hrs}h (threshold: ${MONTHLY_UPTIME_THRESHOLD_DAYS}d) ✅"
|
|
else
|
|
echo " $ICON_WARN Uptime: ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d needed — NOT met"
|
|
fi
|
|
|
|
if [[ "$INTERVAL_GATE_PASS" == true ]]; then
|
|
echo " $ICON_DONE Interval: last run ${DAYS_SINCE_LAST_RUN} ago (threshold: ${MONTHLY_RUN_INTERVAL_DAYS}d) ✅"
|
|
else
|
|
echo " $ICON_WARN Interval: last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d needed — NOT met"
|
|
fi
|
|
|
|
if [[ "$LAST_RUN" -gt 0 ]]; then
|
|
echo " Last run: $(date -d "@$LAST_RUN" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$LAST_RUN" '+%Y-%m-%d %H:%M:%S')"
|
|
else
|
|
echo " Last run: never"
|
|
fi
|
|
|
|
echo ""
|
|
echo "── Scripts ──"
|
|
if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then
|
|
echo " (none configured — add to MONTHLY_MAINTENANCE_SCRIPTS in master.conf)"
|
|
else
|
|
for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$entry" ]] && continue
|
|
read -r -a parts <<< "$entry"
|
|
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
|
|
script_name=$(basename "${parts[0]}")
|
|
if [[ -f "$script_path" ]]; then
|
|
[[ -x "$script_path" ]] && echo " $ICON_DONE $script_name" || echo " $ICON_WARN $script_name (not executable)"
|
|
else
|
|
echo " $ICON_ERROR $script_name — NOT FOUND: $script_path"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
echo ""
|
|
echo " Schedule: 0 0 15 * * (15th of each month at midnight)"
|
|
echo " Force flag: --force bypasses both gates"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
exit 0
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Gate Check — Silent Exit When Not Due ━━━
|
|
# ==============================================================================================
|
|
if [[ "$FORCE_RUN" != true ]]; then
|
|
if [[ "$UPTIME_GATE_PASS" != true ]]; then
|
|
echo "Uptime gate not met — ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d — no-op"
|
|
exit 0
|
|
fi
|
|
if [[ "$INTERVAL_GATE_PASS" != true ]]; then
|
|
echo "Interval gate not met — last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d — no-op"
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Pre-flight ━━━
|
|
# ==============================================================================================
|
|
if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then
|
|
warn "No scripts in MONTHLY_MAINTENANCE_SCRIPTS — nothing to run"
|
|
exit 0
|
|
fi
|
|
|
|
echo ""
|
|
echo "━━━ $ICON_START Monthly Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
|
echo "$ICON_HOST Uptime: ${_uptime_days}d ${_uptime_hrs}h | Last run: ${DAYS_SINCE_LAST_RUN} ago${FORCE_RUN:+ | FORCED}"
|
|
echo "$ICON_GEAR Scripts: ${#MONTHLY_MAINTENANCE_SCRIPTS[@]}"
|
|
echo ""
|
|
|
|
START=$(date +%s)
|
|
PASSED=()
|
|
FAILED=()
|
|
STEP=0
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Run Sequence ━━━
|
|
# ==============================================================================================
|
|
for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do
|
|
[[ -z "$entry" ]] && continue
|
|
(( STEP++ ))
|
|
|
|
read -r -a parts <<< "$entry"
|
|
script_path="$ECOSYSTEM_ROOT/${parts[0]}"
|
|
script_name=$(basename "${parts[0]}")
|
|
extra_args=("${parts[@]:1}")
|
|
|
|
echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━"
|
|
|
|
if [[ ! -f "$script_path" ]]; then
|
|
error "$script_name — not found at $script_path"
|
|
FAILED+=("$script_name")
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
if [[ ! -x "$script_path" ]]; then
|
|
warn "$script_name — not executable, fixing..."
|
|
chmod +x "$script_path" || {
|
|
error "$script_name — chmod +x failed"
|
|
FAILED+=("$script_name")
|
|
echo ""
|
|
continue
|
|
}
|
|
fi
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — would run: $script_name${extra_args:+ ${extra_args[*]}}"
|
|
PASSED+=("$script_name")
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
local_args=()
|
|
[[ "$VERBOSE" == true ]] && local_args+=("--log")
|
|
|
|
if bash "$script_path" "${extra_args[@]}" "${local_args[@]}"; then
|
|
echo "$script_name — done ✅"
|
|
PASSED+=("$script_name")
|
|
else
|
|
warn "$script_name — failed (exit $?) — continuing to next step"
|
|
FAILED+=("$script_name")
|
|
fi
|
|
|
|
echo ""
|
|
done
|
|
|
|
END=$(date +%s)
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Record Last Run ━━━
|
|
# ==============================================================================================
|
|
# Write timestamp whether we passed or partially failed — prevents hammering broken scripts.
|
|
if [[ "$DRY_RUN" != true ]]; then
|
|
echo "$NOW" > "$MONTHLY_LAST_RUN_FILE"
|
|
log "Last run recorded: $(date -d "@$NOW" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$NOW" '+%Y-%m-%d %H:%M:%S')"
|
|
fi
|
|
|
|
# ==============================================================================================
|
|
# ━━━ Summary ━━━
|
|
# ==============================================================================================
|
|
echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━"
|
|
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
|
echo "$ICON_TIME Duration: $(format_duration $(( END - START )))"
|
|
[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}"
|
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
|
echo ""
|
|
|
|
if [[ "$DRY_RUN" == true ]]; then
|
|
warn "DRY RUN — no changes made"
|
|
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
|
echo "$ICON_DONE Status: all $STEP step(s) complete ✅"
|
|
notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \
|
|
"Monthly Maintenance" "normal"
|
|
else
|
|
warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}"
|
|
notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \
|
|
"Monthly Maintenance" "warning"
|
|
fi
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
|
exit 0
|