#!/bin/bash # ============================================================================================== # ============================= Sunday Morning Coffee Report =================================== # ============================================================================================== # # PURPOSE # ───────────────────────────────────────────────────────────────────────────── # Weekly monitoring orchestrator — runs all Sunday monitor scripts in sequence. # Designed to be read over coffee while the system is fully caught up from the # 2:30am maintenance window. # Schedule: 0 7 * * 0 (7am Sunday — after weekly_sync_maintenance.sh finishes) # # ============================================================================================== # OPERATIONAL MODEL # ============================================================================================== # # Runs COFFEE_REPORT_SCRIPTS from master.conf in order. Each script runs # independently, produces its own output, and notifies on findings. # All scripts receive --dry-run and --log flags from this orchestrator when set. # # HOST1 primary: runs all scripts. # HOST2: limited to host-aware scripts only — each script handles its own host logic. # # ============================================================================================== # DESIGN PRINCIPLES # ============================================================================================== # # Independent, Isolated Scripts # Each monitor script is fully self-contained — a failure in one does not # prevent the others from running. The orchestrator logs the failure and # continues to the next script. # # Runs After the Full Weekly Window # Scheduled 4+ hours after weekly_sync_maintenance.sh — the system is fully # synced and containers are back up before any monitoring reads run. Reports # reflect the settled post-maintenance state. # # ============================================================================================== # OPERATIONAL SAFEGUARDS # ============================================================================================== # # Root Enforcement # Required for the docker and system reads the report is built from. # # Lock Acquisition # acquire_lock prevents overlapping weekly runs. # # Host Detection # detect_hosts() sets MY_ID for the banner and summary. # # Empty Job List Guard # Exits with an error and a notification if COFFEE_REPORT_SCRIPTS is empty. A report # that silently contains nothing still arrives looking like a report. # # Read-Only by Composition # Every child here is a reporting script. This orchestrator changes nothing itself — # it only sequences reads and assembles their output. # # Non-Fatal Steps # A failing script is logged and the remaining ones still run, so one unavailable # subsystem costs a section of the report rather than the whole thing. # # Flag Pass-Through # --dry-run and --log are forwarded to every child script. # # Notification Contract # notify() fires on failure only outside --dry-run, matching the runtime-mode contract # below — a dry run never sends anything outward. # # ============================================================================================== # CONFIGURATION # ============================================================================================== # # master.conf # # COFFEE_REPORT_SCRIPTS — ordered list of monitor scripts to run # # ============================================================================================== # RUNTIME MODES # ============================================================================================== # # sunday_morning_coffee_report.sh # Normal run. # # sunday_morning_coffee_report.sh --dry-run # Preview without any writes or notifications. # # sunday_morning_coffee_report.sh --log # Verbose per-script output. # # sunday_morning_coffee_report.sh --status # Show configured scripts and exit. # # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "$ECOSYSTEM_ROOT/load_config.sh" # Timed from here so the standard summary can report a real duration; this report had none. REPORT_START=$(date +%s) parse_args "$@" # ============================================================================================== # ━━━ Setup ━━━ # ============================================================================================== if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 fi acquire_lock detect_hosts # An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable # from a healthy run. Fail loudly instead of silently doing no work. if [[ ${#COFFEE_REPORT_SCRIPTS[@]} -eq 0 ]]; then error "COFFEE_REPORT_SCRIPTS is empty — no coffee report scripts will run" error "Check COFFEE_REPORT_SCRIPTS in master.conf" notify "coffee report scripts skipped on $(hostname) ($MY_ID) — COFFEE_REPORT_SCRIPTS is empty" \ "$(basename "$0" .sh)" "warning" exit 1 fi # ============================================================================================== # ━━━ Helpers ━━━ # ============================================================================================== JOB_PASS=() JOB_FAIL=() # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then echo "━━━ Sunday Morning Coffee Report — Status ━━━" echo " Host: $MY_ID" echo " Scripts:" if [[ ${#COFFEE_REPORT_SCRIPTS[@]} -eq 0 ]]; then echo " None configured (COFFEE_REPORT_SCRIPTS in master.conf)" else for entry in "${COFFEE_REPORT_SCRIPTS[@]}"; do [[ -n "$entry" ]] && echo " ☕ ${entry##*/}" done fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi # ============================================================================================== # ━━━ Main ━━━ # ============================================================================================== _unraid_ver=$(platform_get_os_version 2>/dev/null || echo "unknown") _uptime_s=$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0) _container_count=$(docker ps -q 2>/dev/null | wc -l || echo 0) _rootfs_pct=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "☕ Sunday Morning Coffee Report — $MY_ID" [[ "$DRY_RUN" == true ]] && echo " [DRY RUN]" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log "$ICON_HOST Server: $MY_ID ($LOCAL_SERVER_NAME) — unRAID $_unraid_ver" log "$ICON_TIME Uptime: $(format_duration $_uptime_s)" log "$ICON_CONTAINERS Docker: $_container_count container(s) running" log "$ICON_HEALTH Rootfs: ${_rootfs_pct:-?}% used" log "$ICON_GEAR Scripts: ${#COFFEE_REPORT_SCRIPTS[@]} configured" if [[ ${#COFFEE_REPORT_SCRIPTS[@]} -eq 0 ]]; then warn "No scripts configured — add entries to COFFEE_REPORT_SCRIPTS in master.conf" exit 0 fi for script_entry in "${COFFEE_REPORT_SCRIPTS[@]}"; do [[ -z "$script_entry" ]] && continue echo "" run_orch_child "$script_entry" done # ============================================================================================== # ━━━ Summary ━━━ # ============================================================================================== echo "" echo "━━━━━ ☕ COFFEE REPORT SUMMARY ━━━━━" echo "🖥️ Host: $MY_ID" echo "⏱️ Done: $(date '+%Y-%m-%d %H:%M:%S')" # Anything the Scheduler troubleshooter concluded was a Varaverk defect rather than a setting. # Counted here because a finding filed on a Tuesday and read on a Tuesday is a finding nobody # acts on — this report is the weekly moment the operator is actually looking. Count only: the # detail lives on the AI tab, and a report that reprints every open bug stops being skimmable. # Silent when there are none, and silent when AI is off, so a host without it reads the same as # it always has. if [[ "${AI_ENABLED:-false}" == true && -d "$DATA_DIR/ai_bugs" ]]; then _ai_bugs_open=$(grep -l '"open": true' "$DATA_DIR"/ai_bugs/*.json 2>/dev/null | wc -l) if [[ "${_ai_bugs_open:-0}" -gt 0 ]]; then echo "🐞 AI bugs: $_ai_bugs_open open — see the AI tab" fi fi if [[ ${#JOB_PASS[@]} -gt 0 ]]; then echo "✅ Passed: ${JOB_PASS[*]}" fi if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then echo "❌ Failed: ${JOB_FAIL[*]}" fi # Standard ending. The configured section list is the denominator, so a report that quietly # stopped producing one of its sections reads as skipped rather than simply not appearing. JOB_COUNT="${#SUNDAY_REPORT_SCRIPTS[@]:-0}" [[ "$JOB_COUNT" -eq 0 ]] && JOB_COUNT=$(( ${#JOB_PASS[@]} + ${#JOB_FAIL[@]} )) orchestrator_summary "SUNDAY MORNING COFFEE REPORT" "$REPORT_START" "Sunday Morning Coffee Report" exit $?