From d708d041886084e5ff79c5f5e182ead81f077e4d Mon Sep 17 00:00:00 2001 From: Gmer4Lfe Date: Fri, 22 May 2026 21:23:10 -0400 Subject: [PATCH] =?UTF-8?q?add=20monthly=5Fmaintenance.sh=20=E2=80=94=20up?= =?UTF-8?q?time-triggered=20orchestrator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-gate design: server must have ≥30 days uptime AND last run must be ≥30 days ago. Both gates must pass before any scripts fire. Called daily at 3am via cron — script self-gates, calling more often is safe. State file on /boot/config (survives reboots): the interval gate is independent of uptime. A reboot resets uptime but does not reset when maintenance last ran — both gates must independently pass. MONTHLY_MAINTENANCE_SCRIPTS added to master.conf in ORCHESTRATORS section. zfs_pool_scrub.sh and smart_long_test.sh listed but commented (neither script exists yet). Also commits mesh_monitor.sh move to Monitors/ that was staged from prior session. Supports --force to bypass both gates for manual runs. --- Configurations/master.conf | 18 ++ {Tools => Monitors}/mesh_monitor.sh | 0 Orchestrators/monthly_maintenance.sh | 282 +++++++++++++++++++++++++++ 3 files changed, 300 insertions(+) rename {Tools => Monitors}/mesh_monitor.sh (100%) create mode 100644 Orchestrators/monthly_maintenance.sh diff --git a/Configurations/master.conf b/Configurations/master.conf index a74f81c..9bef021 100644 --- a/Configurations/master.conf +++ b/Configurations/master.conf @@ -37,6 +37,7 @@ # # ── ORCHESTRATORS ────────────────────────────────────────────────────────────────────────── # ARRAY START Scripts launched at array start (array_started.sh) +# MONTHLY MAINTENANCE Uptime-triggered heavy tasks — ZFS scrub, SMART tests (monthly_maintenance.sh) # DAILY SYNC MAINTENANCE Job list + media shares (daily_sync_maintenance.sh) # WEEKLY SYNC MAINTENANCE Job list + sync shares + update toggles (weekly_sync_maintenance.sh) # CRITICAL SYNC MAINTENANCE 15-minute jobs + sync shares + partnership check (critical_sync_maintenance.sh) @@ -311,6 +312,23 @@ "Watchdogs/System/network_watchdog.sh" # internet, DDNS, Tailscale, NPM proxy ) +# ━━━ Monthly Maintenance ━━━ +# monthly_maintenance.sh fires only when BOTH gates pass: +# 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days +# 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run) +# Cron: 0 3 * * * (daily 3am check — script self-gates, calling daily is safe) +# NOT in WATCHDOG_ORCHESTRATOR_SCRIPTS — has its own cron entry. +# Add scripts that require a long-stable settled system — scrubs, extended drive tests. +# State file on /boot/config — survives reboots (interval gate independent of uptime gate). + MONTHLY_MAINTENANCE_SCRIPTS=( + #"Tools/zfs_pool_scrub.sh" # ZFS pool integrity scrub — not yet built + #"Tools/smart_long_test.sh" # SMART extended drive health test — not yet built + ) + + MONTHLY_UPTIME_THRESHOLD_DAYS=30 # minimum uptime in days before maintenance fires + MONTHLY_RUN_INTERVAL_DAYS=30 # minimum days since last run before running again + MONTHLY_LAST_RUN_FILE="/boot/config/monthly_maintenance_last_run.db" + # ━━━ Critical Sync Maintenance ━━━ # critical_sync_maintenance.sh runs every 15 minutes. # Order: CRITICAL_MAINTENANCE_SCRIPTS (jobs) → CRITICAL_SYNC_SHARES (rsync) → partnership --check diff --git a/Tools/mesh_monitor.sh b/Monitors/mesh_monitor.sh similarity index 100% rename from Tools/mesh_monitor.sh rename to Monitors/mesh_monitor.sh diff --git a/Orchestrators/monthly_maintenance.sh b/Orchestrators/monthly_maintenance.sh new file mode 100644 index 0000000..9dd46bc --- /dev/null +++ b/Orchestrators/monthly_maintenance.sh @@ -0,0 +1,282 @@ +#!/bin/bash +# ============================================================================================== +# ========================= Monthly Maintenance Orchestrator =================================== +# ============================================================================================== +# Uptime-triggered monthly maintenance — runs heavy tasks that need a stable, settled system. +# Fires only when BOTH gates pass: +# 1. Server uptime >= MONTHLY_UPTIME_THRESHOLD_DAYS days +# 2. Last run was >= MONTHLY_RUN_INTERVAL_DAYS days ago (or never run) +# +# ── WHY UPTIME-TRIGGERED, NOT CRON ─────────────────────────────────────────────────────────── +# A scheduled reboot resets uptime. Monthly tasks (ZFS scrub, SMART long test) need a +# stable, settled system — not one that just rebooted. Uptime-gating ensures maintenance +# only runs after the server has been healthy for a full month, never immediately post-boot. +# The daily cron is just the trigger mechanism. The uptime and interval checks inside +# the script are what enforce the monthly cadence. +# +# ── HOW TO CALL ────────────────────────────────────────────────────────────────────────────── +# Cron: 0 3 * * * — daily 3am check. Script self-gates — calling it daily is safe. +# Silent exit 0 when either gate is not met. Only outputs when maintenance actually fires. +# +# ── STATE FILE ──────────────────────────────────────────────────────────────────────────────── +# MONTHLY_LAST_RUN_FILE — /boot/config — survives reboots, available before array starts. +# Written after each run (pass or partial fail). Format: Unix timestamp. +# A reboot does NOT reset the last-run state — the interval gate survives independently +# of the uptime gate. Both must pass before maintenance fires again. +# +# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── +# MONTHLY_MAINTENANCE_SCRIPTS — ordered list of scripts to run +# MONTHLY_UPTIME_THRESHOLD_DAYS — minimum uptime in days before maintenance fires +# MONTHLY_RUN_INTERVAL_DAYS — minimum days since last run before running again +# MONTHLY_LAST_RUN_FILE — state file path — /boot/config, survives reboots +# +# ── USAGE ───────────────────────────────────────────────────────────────────────────────────── +# monthly_maintenance.sh — normal run (uptime + interval gates enforced) +# monthly_maintenance.sh --dry-run — preview gate state and scripts without running +# monthly_maintenance.sh --status — show gate state, last run, and configured scripts +# monthly_maintenance.sh --force — bypass uptime + interval gates (manual override) +# monthly_maintenance.sh --log — verbose output +# ============================================================================================== + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +source "$ECOSYSTEM_ROOT/load_config.sh" + +# ── Parse --force before common parse_args ──────────────────────────────────────────────────── +FORCE_RUN=false +FILTERED_ARGS=() +for _arg in "$@"; do + if [[ "$_arg" == "--force" ]]; then + FORCE_RUN=true + else + FILTERED_ARGS+=("$_arg") + fi +done +parse_args "${FILTERED_ARGS[@]}" + +# ============================================================================================== +# ━━━ Setup ━━━ +# ============================================================================================== +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +acquire_lock + +detect_hosts + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no scripts will be executed" +[[ "$FORCE_RUN" == true ]] && warn "FORCE — uptime and interval gates bypassed" + +# Defaults — overridden by master.conf values +MONTHLY_UPTIME_THRESHOLD_DAYS="${MONTHLY_UPTIME_THRESHOLD_DAYS:-30}" +MONTHLY_RUN_INTERVAL_DAYS="${MONTHLY_RUN_INTERVAL_DAYS:-30}" +MONTHLY_LAST_RUN_FILE="${MONTHLY_LAST_RUN_FILE:-/boot/config/monthly_maintenance_last_run.db}" + +UPTIME_THRESHOLD_SECS=$(( MONTHLY_UPTIME_THRESHOLD_DAYS * 86400 )) +INTERVAL_SECS=$(( MONTHLY_RUN_INTERVAL_DAYS * 86400 )) +UPTIME_SECS=$(awk '{print int($1)}' /proc/uptime) +NOW=$(date +%s) + +# ============================================================================================== +# ━━━ Gate Evaluation ━━━ +# ============================================================================================== +_uptime_days=$(( UPTIME_SECS / 86400 )) +_uptime_hrs=$(( (UPTIME_SECS % 86400) / 3600 )) + +UPTIME_GATE_PASS=false +if [[ "$UPTIME_SECS" -ge "$UPTIME_THRESHOLD_SECS" ]]; then + UPTIME_GATE_PASS=true +fi + +INTERVAL_GATE_PASS=false +LAST_RUN=0 +DAYS_SINCE_LAST_RUN="never" +if [[ -f "$MONTHLY_LAST_RUN_FILE" ]]; then + LAST_RUN=$(cat "$MONTHLY_LAST_RUN_FILE" 2>/dev/null || echo 0) + ELAPSED=$(( NOW - LAST_RUN )) + DAYS_SINCE=$(( ELAPSED / 86400 )) + DAYS_SINCE_LAST_RUN="${DAYS_SINCE}d" + if [[ "$ELAPSED" -ge "$INTERVAL_SECS" ]]; then + INTERVAL_GATE_PASS=true + fi +else + INTERVAL_GATE_PASS=true # never run +fi + +# ============================================================================================== +# ━━━ Status ━━━ +# ============================================================================================== +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE STATUS ━━━━━" + echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" + echo "" + echo "── Gates ──" + + if [[ "$UPTIME_GATE_PASS" == true ]]; then + echo " $ICON_DONE Uptime: ${_uptime_days}d ${_uptime_hrs}h (threshold: ${MONTHLY_UPTIME_THRESHOLD_DAYS}d) ✅" + else + echo " $ICON_WARN Uptime: ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d needed — NOT met" + fi + + if [[ "$INTERVAL_GATE_PASS" == true ]]; then + echo " $ICON_DONE Interval: last run ${DAYS_SINCE_LAST_RUN} ago (threshold: ${MONTHLY_RUN_INTERVAL_DAYS}d) ✅" + else + echo " $ICON_WARN Interval: last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d needed — NOT met" + fi + + if [[ "$LAST_RUN" -gt 0 ]]; then + echo " Last run: $(date -d "@$LAST_RUN" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$LAST_RUN" '+%Y-%m-%d %H:%M:%S')" + else + echo " Last run: never" + fi + + echo "" + echo "── Scripts ──" + if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then + echo " (none configured — add to MONTHLY_MAINTENANCE_SCRIPTS in master.conf)" + else + for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do + [[ -z "$entry" ]] && continue + read -r -a parts <<< "$entry" + script_path="$ECOSYSTEM_ROOT/${parts[0]}" + script_name=$(basename "${parts[0]}") + if [[ -f "$script_path" ]]; then + [[ -x "$script_path" ]] && echo " $ICON_DONE $script_name" || echo " $ICON_WARN $script_name (not executable)" + else + echo " $ICON_ERROR $script_name — NOT FOUND: $script_path" + fi + done + fi + + echo "" + echo " Schedule: 0 3 * * * (daily check — script self-gates)" + echo " Force flag: --force bypasses both gates" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ============================================================================================== +# ━━━ Gate Check — Silent Exit When Not Due ━━━ +# ============================================================================================== +if [[ "$FORCE_RUN" != true ]]; then + if [[ "$UPTIME_GATE_PASS" != true ]]; then + log "Uptime gate — ${_uptime_days}d ${_uptime_hrs}h / ${MONTHLY_UPTIME_THRESHOLD_DAYS}d — not yet due" + exit 0 + fi + if [[ "$INTERVAL_GATE_PASS" != true ]]; then + log "Interval gate — last run ${DAYS_SINCE_LAST_RUN} ago / ${MONTHLY_RUN_INTERVAL_DAYS}d — not yet due" + exit 0 + fi +fi + +# ============================================================================================== +# ━━━ Pre-flight ━━━ +# ============================================================================================== +if [[ "${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" -eq 0 ]]; then + warn "No scripts in MONTHLY_MAINTENANCE_SCRIPTS — nothing to run" + exit 0 +fi + +echo "" +echo "━━━ $ICON_START Monthly Maintenance — $MY_ID — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_HOST Uptime: ${_uptime_days}d ${_uptime_hrs}h | Last run: ${DAYS_SINCE_LAST_RUN} ago${FORCE_RUN:+ | FORCED}" +echo "$ICON_GEAR Scripts: ${#MONTHLY_MAINTENANCE_SCRIPTS[@]}" +echo "" + +START=$(date +%s) +PASSED=() +FAILED=() +STEP=0 + +# ============================================================================================== +# ━━━ Run Sequence ━━━ +# ============================================================================================== +for entry in "${MONTHLY_MAINTENANCE_SCRIPTS[@]}"; do + [[ -z "$entry" ]] && continue + (( STEP++ )) + + read -r -a parts <<< "$entry" + script_path="$ECOSYSTEM_ROOT/${parts[0]}" + script_name=$(basename "${parts[0]}") + extra_args=("${parts[@]:1}") + + echo "━━━ $ICON_GEAR Step $STEP: $script_name${extra_args:+ ${extra_args[*]}} ━━━" + + if [[ ! -f "$script_path" ]]; then + error "$script_name — not found at $script_path" + FAILED+=("$script_name") + echo "" + continue + fi + + if [[ ! -x "$script_path" ]]; then + warn "$script_name — not executable, fixing..." + chmod +x "$script_path" || { + error "$script_name — chmod +x failed" + FAILED+=("$script_name") + echo "" + continue + } + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would run: $script_name${extra_args:+ ${extra_args[*]}}" + PASSED+=("$script_name") + echo "" + continue + fi + + local_args=() + [[ "$VERBOSE" == true ]] && local_args+=("--log") + + if bash "$script_path" "${extra_args[@]}" "${local_args[@]}"; then + log "$script_name — done ✅" + PASSED+=("$script_name") + else + warn "$script_name — failed (exit $?) — continuing to next step" + FAILED+=("$script_name") + fi + + echo "" +done + +END=$(date +%s) + +# ============================================================================================== +# ━━━ Record Last Run ━━━ +# ============================================================================================== +# Write timestamp whether we passed or partially failed — prevents hammering broken scripts. +if [[ "$DRY_RUN" != true ]]; then + echo "$NOW" > "$MONTHLY_LAST_RUN_FILE" + log "Last run recorded: $(date -d "@$NOW" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || date -r "$NOW" '+%Y-%m-%d %H:%M:%S')" +fi + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +echo "━━━━━ $ICON_SUMMARY MONTHLY MAINTENANCE SUMMARY ━━━━━" +echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" +echo "$ICON_TIME Duration: $(format_duration $(( END - START )))" +[[ ${#PASSED[@]} -gt 0 ]] && echo "$ICON_DONE Passed: ${PASSED[*]}" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" +echo "" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — no changes made" +elif [[ ${#FAILED[@]} -eq 0 ]]; then + echo "$ICON_DONE Status: all $STEP step(s) complete ✅" + notify "Monthly maintenance complete on $(hostname) ($MY_ID) — $STEP step(s) done" \ + "Monthly Maintenance" "normal" +else + warn "Status: ${#FAILED[@]} step(s) failed — ${FAILED[*]}" + notify "Monthly maintenance on $(hostname) ($MY_ID) — ${#FAILED[@]} step(s) failed: ${FAILED[*]}" \ + "Monthly Maintenance" "warning" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +exit 0