add monthly_maintenance.sh — uptime-triggered orchestrator
Two-gate design: server must have ≥30 days uptime AND last run must be ≥30 days ago. Both gates must pass before any scripts fire. Called daily at 3am via cron — script self-gates, calling more often is safe. State file on /boot/config (survives reboots): the interval gate is independent of uptime. A reboot resets uptime but does not reset when maintenance last ran — both gates must independently pass. MONTHLY_MAINTENANCE_SCRIPTS added to master.conf in ORCHESTRATORS section. zfs_pool_scrub.sh and smart_long_test.sh listed but commented (neither script exists yet). Also commits mesh_monitor.sh move to Monitors/ that was staged from prior session. Supports --force to bypass both gates for manual runs.
This commit is contained in:
Gmer4Lfe
@@ -1,217 +0,0 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================================
|
||||
# ================================= Mesh Monitor ===============================================
|
||||
# ==============================================================================================
|
||||
#
|
||||
# PURPOSE
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# At-a-glance view of everyone in the mesh — who they are, how to reach them,
|
||||
# and what services each node has put up for mutual fallback protection.
|
||||
# Read-only — sources conf data only, no SSH or API calls, no sensitive data.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# OPERATIONAL MODEL
|
||||
# ==============================================================================================
|
||||
#
|
||||
# Reads directly from all three conf files without detect_hosts().
|
||||
# Iterates all defined HOST* vars — scales automatically as new nodes join.
|
||||
#
|
||||
# Sections:
|
||||
# MEMBERS — server name, owner, contact email for each HOST*
|
||||
# COVERAGE — what each host runs for others when they go down, by tier
|
||||
# PARTNERSHIP — enabled/disabled, current owner, sync interval
|
||||
#
|
||||
# Safe to share with mesh members — contains no API keys, passwords, or SSH keys.
|
||||
#
|
||||
# ==============================================================================================
|
||||
# CONFIGURATION
|
||||
# ==============================================================================================
|
||||
#
|
||||
# host*.conf — per-host identity vars read by this script:
|
||||
#
|
||||
# HOST*_OWNER human-readable owner name
|
||||
# HOST*_OWNER_EMAIL contact email — also used by NPM for Let's Encrypt
|
||||
#
|
||||
# ==============================================================================================
|
||||
# RUNTIME MODES
|
||||
# ==============================================================================================
|
||||
#
|
||||
# mesh_monitor.sh — show full mesh overview
|
||||
# mesh_monitor.sh --log — verbose output
|
||||
# mesh_monitor.sh --status — show config and exit
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
# Populate ALL_HOST_IDS with every defined HOST* var
|
||||
collect_hosts() {
|
||||
ALL_HOST_IDS=()
|
||||
for h in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do
|
||||
[[ -n "${!h:-}" ]] && ALL_HOST_IDS+=("$h")
|
||||
done
|
||||
}
|
||||
|
||||
# Read a bash array by computed name — returns space-separated elements
|
||||
get_array() { eval "echo \"\${${1}[*]}\""; }
|
||||
|
||||
# Convert minutes to human-readable delay string
|
||||
format_delay() {
|
||||
local m="${1:-0}"
|
||||
(( m == 0 )) && { echo "immediate"; return; }
|
||||
(( m >= 1440 )) && { echo "$((m / 1440))d"; return; }
|
||||
(( m >= 60 )) && { echo "$((m / 60))hr"; return; }
|
||||
echo "${m}min"
|
||||
}
|
||||
|
||||
collect_hosts
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo " Hosts: ${ALL_HOST_IDS[*]}"
|
||||
for h in "${ALL_HOST_IDS[@]}"; do
|
||||
owner_var="${h}_OWNER"; owner="${!owner_var:-unknown}"
|
||||
email_var="${h}_OWNER_EMAIL"; email="${!email_var:-(not set)}"
|
||||
echo " $h: ${!h} / $owner / $email"
|
||||
done
|
||||
echo " Mode: read-only — conf data only, no SSH or API calls"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Mesh Overview ━━━
|
||||
# ==============================================================================================
|
||||
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " MESH OVERVIEW"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# ── Members ───────────────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_HOST Members ━━━"
|
||||
echo ""
|
||||
|
||||
# Dynamic column widths
|
||||
W_HOST=4; W_SERVER=6; W_OWNER=5
|
||||
for h in "${ALL_HOST_IDS[@]}"; do
|
||||
server="${!h}"
|
||||
owner_var="${h}_OWNER"; owner="${!owner_var:-unknown}"
|
||||
(( ${#h} > W_HOST )) && W_HOST=${#h}
|
||||
(( ${#server} > W_SERVER )) && W_SERVER=${#server}
|
||||
(( ${#owner} > W_OWNER )) && W_OWNER=${#owner}
|
||||
done
|
||||
|
||||
printf " %-${W_HOST}s %-${W_SERVER}s %-${W_OWNER}s %s\n" \
|
||||
"HOST" "SERVER" "OWNER" "EMAIL"
|
||||
printf " %-${W_HOST}s %-${W_SERVER}s %-${W_OWNER}s %s\n" \
|
||||
"$(printf '─%.0s' $(seq 1 $W_HOST))" \
|
||||
"$(printf '─%.0s' $(seq 1 $W_SERVER))" \
|
||||
"$(printf '─%.0s' $(seq 1 $W_OWNER))" \
|
||||
"─────────────────────────"
|
||||
|
||||
for h in "${ALL_HOST_IDS[@]}"; do
|
||||
server="${!h}"
|
||||
owner_var="${h}_OWNER"; owner="${!owner_var:-unknown}"
|
||||
email_var="${h}_OWNER_EMAIL"; email="${!email_var:-(not set)}"
|
||||
printf " %-${W_HOST}s %-${W_SERVER}s %-${W_OWNER}s %s\n" \
|
||||
"$h" "$server" "$owner" "$email"
|
||||
done
|
||||
|
||||
# ── Protected Services ────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Protected Services ━━━"
|
||||
|
||||
COVERAGE_FOUND=false
|
||||
|
||||
# Iterate from the perspective of each protected host — what do they want covered
|
||||
for covered in "${ALL_HOST_IDS[@]}"; do
|
||||
covered_owner_var="${covered}_OWNER"; covered_owner="${!covered_owner_var:-$covered}"
|
||||
covered_email_var="${covered}_OWNER_EMAIL"; covered_email="${!covered_email_var:-}"
|
||||
|
||||
# Find who covers this host and collect their tiers
|
||||
any_tiers=false
|
||||
declare -A tier_containers=()
|
||||
declare -A tier_delays=()
|
||||
covered_by=""
|
||||
|
||||
for covering in "${ALL_HOST_IDS[@]}"; do
|
||||
[[ "$covering" == "$covered" ]] && continue
|
||||
|
||||
for tier in 1 2 3 4; do
|
||||
containers=$(get_array "FALLBACK_${covering}_COVERS_${covered}_TIER${tier}")
|
||||
[[ -z "$containers" ]] && continue
|
||||
any_tiers=true
|
||||
tier_containers[$tier]="$containers"
|
||||
delay_var="${covered}_TIER${tier}_DELAY"
|
||||
tier_delays[$tier]="${!delay_var:-0}"
|
||||
done
|
||||
|
||||
[[ "$any_tiers" == true ]] && {
|
||||
covering_owner_var="${covering}_OWNER"
|
||||
covering_owner="${!covering_owner_var:-$covering}"
|
||||
covered_by="$covering ($covering_owner)"
|
||||
}
|
||||
done
|
||||
|
||||
[[ "$any_tiers" == false ]] && continue
|
||||
|
||||
COVERAGE_FOUND=true
|
||||
echo ""
|
||||
|
||||
header="$covered $covered_owner"
|
||||
[[ -n "$covered_email" ]] && header+=" — $covered_email"
|
||||
echo " $header"
|
||||
|
||||
for tier in 1 2 3 4; do
|
||||
[[ -z "${tier_containers[$tier]:-}" ]] && continue
|
||||
if [[ "$tier" -eq 1 ]]; then
|
||||
printf " Tier 1 %-12s %s\n" "(immediate)" "${tier_containers[$tier]// /, }"
|
||||
else
|
||||
delay=$(format_delay "${tier_delays[$tier]:-0}")
|
||||
printf " Tier %s %-12s %s\n" "$tier" "($delay)" "${tier_containers[$tier]// /, }"
|
||||
fi
|
||||
done
|
||||
|
||||
[[ -n "$covered_by" ]] && echo " Covered by: $covered_by"
|
||||
|
||||
unset tier_containers tier_delays
|
||||
declare -A tier_containers=()
|
||||
declare -A tier_delays=()
|
||||
done
|
||||
|
||||
if [[ "$COVERAGE_FOUND" == false ]]; then
|
||||
echo ""
|
||||
echo " No fallback coverage configured"
|
||||
fi
|
||||
|
||||
# ── Partnership ───────────────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo "━━━ $ICON_NET Partnership ━━━"
|
||||
echo ""
|
||||
|
||||
if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then
|
||||
owner_host="${PARTNERSHIP_OWNER_HOST:-HOST1}"
|
||||
owner_server="${!owner_host:-unknown}"
|
||||
owner_name_var="${owner_host}_OWNER"; owner_name="${!owner_name_var:-unknown}"
|
||||
echo " Status enabled"
|
||||
echo " Owner $owner_host ($owner_server / $owner_name)"
|
||||
echo " Sync every ${PARTNERSHIP_SYNC_INTERVAL:-15}min"
|
||||
else
|
||||
echo " Status disabled"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
Reference in New Issue
Block a user