#!/bin/bash # ============================================================================================== # ========================= Monthly Maintenance Orchestrator =================================== # ============================================================================================== # Uptime-triggered monthly maintenance — runs heavy tasks that need a stable, settled system. # Fires only when BOTH gates pass: # 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days # 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run) # # ── WHY UPTIME-GATED ───────────────────────────────────────────────────────────────────────── # A scheduled reboot resets uptime. Monthly tasks (ZFS scrub, SMART long test) need a # stable, settled system — not one that just rebooted. Uptime-gating ensures maintenance # only runs after the server has been healthy for a full month, never immediately post-boot. # If uptime or interval gate is not met on the 15th, the run is skipped until next month. # # ── HOW TO CALL ────────────────────────────────────────────────────────────────────────────── # Schedule: 0 0 15 * * (15th of each month at midnight) # Silent exit 0 when either gate is not met. Only outputs when maintenance actually fires. # # ── STATE FILE ──────────────────────────────────────────────────────────────────────────────── # MONTHLY_LAST_RUN_FILE — /boot/config — survives reboots, available before array starts. # Written after each run (pass or partial fail). Format: Unix timestamp. # A reboot does NOT reset the last-run state — the interval gate survives independently # of the uptime gate. Both must pass before maintenance fires again. # # ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── # MONTHLY_MAINTENANCE_SCRIPTS — ordered list of scripts to run # MONTHLY_UPTIME_THRESHOLD_DAYS — minimum uptime in days before maintenance fires # MONTHLY_RUN_INTERVAL_DAYS — minimum days since last run before running again # MONTHLY_LAST_RUN_FILE — state file path — /boot/config, survives reboots # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── # monthly_maintenance.sh — normal run (uptime + interval gates enforced) # monthly_maintenance.sh --dry-run — preview gate state and scripts without running # monthly_maintenance.sh --status — show gate state, last run, and configured scripts # monthly_maintenance.sh --force — bypass uptime + interval gates (manual override) # monthly_maintenance.sh --log — verbose output # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "$ECOSYSTEM_ROOT/load_config.sh" # ── Parse --force before common parse_args ──────────────────────────────────────────────────── FORCE_RUN=false FILTERED_ARGS=() for _arg in "$@"; do if [[ "$_arg" == "--force" ]]; then FORCE_RUN=true else FILTERED_ARGS+=("$_arg") fi done parse_args "${FILTERED_ARGS[@]}" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi acquire_lock detect_hosts [[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scripts will be executed" [[ "$FORCE_RUN" == true ]] && warn "FORCE — uptime and interval gates bypassed" # Defaults — overridden by master.conf values MONTHLY_UPTIME_THRESHOLD_DAYS="${MONTHLY_UPTIME_THRESHOLD_DAYS:-30}" MONTHLY_RUN_INTERVAL_DAYS="${MONTHLY_RUN_INTERVAL_DAYS:-30}" MONTHLY_LAST_RUN_FILE="${MONTHLY_LAST_RUN_FILE:-${STATE_DIR:-/tmp}/monthly_maintenance_last_run.db}" UPTIME_THRESHOLD_SECS=$(( MONTHLY_UPTIME_THRESHOLD_DAYS * 86400 )) INTERVAL_SECS=$(( MONTHLY_RUN_INTERVAL_DAYS * 86400 )) UPTIME_SECS=$(awk '{print int($1)}' /proc/uptime) NOW=$(date +%s) # ============================================================================================== # ━━━ Gate Evaluation ━━━ # ============================================================================================== _uptime_days=$(( UPTIME_SECS / 86400 )) _uptime_hrs=$(( (UPTIME_SECS % 86400) / 3600 )) UPTIME_GATE_PASS=false if [[ "$UPTIME_SECS" -ge "$UPTIME_THRESHOLD_SECS" ]]; then UPTIME_GATE_PASS=true fi INTERVAL_GATE_PASS=false LAST_RUN=0 DAYS_SINCE_LAST_RUN="never" if [[ -f "$MONTHLY_LAST_RUN_FILE" ]]; then LAST_RUN=$(cat "$MONTHLY_LAST_RUN_FILE" 2>/dev/null || echo 0) ELAPSED=$(( NOW - LAST_RUN )) DAYS_SINCE=$(( ELAPSED / 86400 )) DAYS_SINCE_LAST_RUN="${DAYS_SINCE}d" if [[ "$ELAPSED" -ge "$INTERVAL_SECS" ]]; then INTERVAL_GATE_PASS=true fi else INTERVAL_GATE_PASS=true # never run fi # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE STATUS ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "" echo "── Gates ──" if [[ "$UPTIME_GATE_PASS" == true ]]; then echo " $ICON_DONE Uptime: ${_uptime_days}d ${_uptime_hrs}h (threshold: ${MONTHLY_UPTIME_THRESHOLD_DAYS}d) ✅" else echo " $ICON_WARN Uptime: ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d needed — NOT met" fi if [[ "$INTERVAL_GATE_PASS" == true ]]; then echo " $ICON_DONE Interval: last run ${DAYS_SINCE_LAST_RUN} ago (threshold: ${MONTHLY_RUN_INTERVAL_DAYS}d) ✅" else echo " $ICON_WARN Interval: last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d needed — NOT met" fi if [[ "$LAST_RUN" -gt 0 ]]; then echo " Last run: $(date -d "@$LAST_RUN" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$LAST_RUN" '+%Y-%m-%d %H:%M:%S')" else echo " Last run: never" fi echo "" echo "── Scripts ──" if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then echo " (none configured — add to MONTHLY_MAINTENANCE_SCRIPTS in master.conf)" else for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do [[ -z "$entry" ]] && continue read -r -a parts <<< "$entry" script_path="$ECOSYSTEM_ROOT/${parts[0]}" script_name=$(basename "${parts[0]}") if [[ -f "$script_path" ]]; then [[ -x "$script_path" ]] && echo " $ICON_DONE $script_name" || echo " $ICON_WARN $script_name (not executable)" else echo " $ICON_ERROR $script_name — NOT FOUND: $script_path" fi done fi echo "" echo " Schedule: 0 0 15 * * (15th of each month at midnight)" echo " Force flag: --force bypasses both gates" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ━━━ Gate Check — Silent Exit When Not Due ━━━ # ============================================================================================== if [[ "$FORCE_RUN" != true ]]; then if [[ "$UPTIME_GATE_PASS" != true ]]; then log "Uptime gate — ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d — not yet due" exit 0 fi if [[ "$INTERVAL_GATE_PASS" != true ]]; then log "Interval gate — last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d — not yet due" exit 0 fi fi # ============================================================================================== # ━━━ Pre-flight ━━━ # ============================================================================================== if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then warn "No scripts in MONTHLY_MAINTENANCE_SCRIPTS — nothing to run" exit 0 fi echo "" echo "━━━ $ICON_START Monthly Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "$ICON_HOST Uptime: ${_uptime_days}d ${_uptime_hrs}h | Last run: ${DAYS_SINCE_LAST_RUN} ago${FORCE_RUN:+ | FORCED}" echo "$ICON_GEAR Scripts: ${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" echo "" START=$(date +%s) PASSED=() FAILED=() STEP=0 # ============================================================================================== # ━━━ Run Sequence ━━━ # ============================================================================================== for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do [[ -z "$entry" ]] && continue (( STEP++ )) read -r -a parts <<< "$entry" script_path="$ECOSYSTEM_ROOT/${parts[0]}" script_name=$(basename "${parts[0]}") extra_args=("${parts[@]:1}") echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━" if [[ ! -f "$script_path" ]]; then error "$script_name — not found at $script_path" FAILED+=("$script_name") echo "" continue fi if [[ ! -x "$script_path" ]]; then warn "$script_name — not executable, fixing..." chmod +x "$script_path" || { error "$script_name — chmod +x failed" FAILED+=("$script_name") echo "" continue } fi if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would run: $script_name${extra_args:+ ${extra_args[*]}}" PASSED+=("$script_name") echo "" continue fi local_args=() [[ "$VERBOSE" == true ]] && local_args+=("--log") if bash "$script_path" "${extra_args[@]}" "${local_args[@]}"; then log "$script_name — done ✅" PASSED+=("$script_name") else warn "$script_name — failed (exit $?) — continuing to next step" FAILED+=("$script_name") fi echo "" done END=$(date +%s) # ============================================================================================== # ━━━ Record Last Run ━━━ # ============================================================================================== # Write timestamp whether we passed or partially failed — prevents hammering broken scripts. if [[ "$DRY_RUN" != true ]]; then echo "$NOW" > "$MONTHLY_LAST_RUN_FILE" log "Last run recorded: $(date -d "@$NOW" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$NOW" '+%Y-%m-%d %H:%M:%S')" fi # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━" echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" [[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}" [[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" echo "" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — no changes made" elif [[ ${#FAILED[@]} -eq 0 ]]; then echo "$ICON_DONE Status: all $STEP step(s) complete ✅" notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \ "Monthly Maintenance" "normal" else warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}" notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \ "Monthly Maintenance" "warning" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" [[ ${#FAILED[@]} -gt 0 ]] && exit 1 exit 0