Varaverk/Orchestrators/sunday_morning_coffee_report.sh

#!/bin/bash
# ==============================================================================================
# ============================= Sunday Morning Coffee Report ===================================
# ==============================================================================================
#
# PURPOSE
# ─────────────────────────────────────────────────────────────────────────────
# Weekly monitoring orchestrator — runs all Sunday monitor scripts in sequence.
# Designed to be read over coffee while the system is fully caught up from the
# 2:30am maintenance window.
# Schedule: 0 7 * * 0 (7am Sunday — after weekly_sync_maintenance.sh finishes)
#
# ==============================================================================================
# OPERATIONAL MODEL
# ==============================================================================================
#
# Runs COFFEE_REPORT_SCRIPTS from master.conf in order. Each script runs
# independently, produces its own output, and notifies on findings.
# All scripts receive --dry-run and --log flags from this orchestrator when set.
#
# HOST1 primary: runs all scripts.
# HOST2: limited to host-aware scripts only — each script handles its own host logic.
#
# ==============================================================================================
# DESIGN PRINCIPLES
# ==============================================================================================
#
# Independent, Isolated Scripts
#   Each monitor script is fully self-contained — a failure in one does not
#   prevent the others from running. The orchestrator logs the failure and
#   continues to the next script.
#
# Runs After the Full Weekly Window
#   Scheduled 4+ hours after weekly_sync_maintenance.sh — the system is fully
#   synced and containers are back up before any monitoring reads run. Reports
#   reflect the settled post-maintenance state.
#
# ==============================================================================================
# OPERATIONAL SAFEGUARDS
# ==============================================================================================
#
# Root Enforcement
#   Required for the docker and system reads the report is built from.
#
# Lock Acquisition
#   acquire_lock prevents overlapping weekly runs.
#
# Host Detection
#   detect_hosts() sets MY_ID for the banner and summary.
#
# Empty Job List Guard
#   Exits with an error and a notification if COFFEE_REPORT_SCRIPTS is empty. A report
#   that silently contains nothing still arrives looking like a report.
#
# Read-Only by Composition
#   Every child here is a reporting script. This orchestrator changes nothing itself —
#   it only sequences reads and assembles their output.
#
# Non-Fatal Steps
#   A failing script is logged and the remaining ones still run, so one unavailable
#   subsystem costs a section of the report rather than the whole thing.
#
# Flag Pass-Through
#   --dry-run and --log are forwarded to every child script.
#
# Notification Contract
#   notify() fires on failure only outside --dry-run, matching the runtime-mode contract
#   below — a dry run never sends anything outward.
#
# ==============================================================================================
# CONFIGURATION
# ==============================================================================================
#
# master.conf
#
#   COFFEE_REPORT_SCRIPTS — ordered list of monitor scripts to run
#
# ==============================================================================================
# RUNTIME MODES
# ==============================================================================================
#
# sunday_morning_coffee_report.sh
#     Normal run.
#
# sunday_morning_coffee_report.sh --dry-run
#     Preview without any writes or notifications.
#
# sunday_morning_coffee_report.sh --log
#     Verbose per-script output.
#
# sunday_morning_coffee_report.sh --status
#     Show configured scripts and exit.
#
# ==============================================================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ECOSYSTEM_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

source "$ECOSYSTEM_ROOT/load_config.sh"

parse_args "$@"

# ==============================================================================================
# ━━━ Setup ━━━
# ==============================================================================================
if [[ "$EUID" -ne 0 ]]; then
    error "Must be run as root"
    exit 1
fi

acquire_lock

detect_hosts

# An unconfigured job list would run nothing and still report "0/0 passed" — indistinguishable
# from a healthy run. Fail loudly instead of silently doing no work.
if [[ ${#COFFEE_REPORT_SCRIPTS[@]} -eq 0 ]]; then
    error "COFFEE_REPORT_SCRIPTS is empty — no coffee report scripts will run"
    error "Check COFFEE_REPORT_SCRIPTS in master.conf"
    notify "coffee report scripts skipped on $(hostname) ($MY_ID) — COFFEE_REPORT_SCRIPTS is empty" \
        "$(basename "$0" .sh)" "warning"
    exit 1
fi

# ==============================================================================================
# ━━━ Helpers ━━━
# ==============================================================================================
JOB_PASS=()
JOB_FAIL=()

# ==============================================================================================
# ━━━ Status ━━━
# ==============================================================================================
if [[ "$SHOW_STATUS" == true ]]; then
    echo "━━━ Sunday Morning Coffee Report — Status ━━━"
    echo "  Host:     $MY_ID"
    echo "  Scripts:"
    if [[ ${#COFFEE_REPORT_SCRIPTS[@]} -eq 0 ]]; then
        echo "    None configured (COFFEE_REPORT_SCRIPTS in master.conf)"
    else
        for entry in "${COFFEE_REPORT_SCRIPTS[@]}"; do
            [[ -n "$entry" ]] && echo "    ☕ ${entry##*/}"
        done
    fi
    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    exit 0
fi

# ==============================================================================================
# ━━━ Main ━━━
# ==============================================================================================
_unraid_ver=$(platform_get_os_version 2>/dev/null || echo "unknown")
_uptime_s=$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)
_container_count=$(docker ps -q 2>/dev/null | wc -l || echo 0)
_rootfs_pct=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "☕  Sunday Morning Coffee Report — $MY_ID"
[[ "$DRY_RUN" == true ]] && echo "    [DRY RUN]"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log "$ICON_HOST Server:     $MY_ID ($LOCAL_SERVER_NAME) — unRAID $_unraid_ver"
log "$ICON_TIME Uptime:     $(format_duration $_uptime_s)"
log "$ICON_CONTAINERS Docker: $_container_count container(s) running"
log "$ICON_HEALTH Rootfs:   ${_rootfs_pct:-?}% used"
log "$ICON_GEAR Scripts:    ${#COFFEE_REPORT_SCRIPTS[@]} configured"

if [[ ${#COFFEE_REPORT_SCRIPTS[@]} -eq 0 ]]; then
    warn "No scripts configured — add entries to COFFEE_REPORT_SCRIPTS in master.conf"
    exit 0
fi

for script_entry in "${COFFEE_REPORT_SCRIPTS[@]}"; do
    [[ -z "$script_entry" ]] && continue
    echo ""
    run_orch_child "$script_entry"
done

# ==============================================================================================
# ━━━ Summary ━━━
# ==============================================================================================
echo ""
echo "━━━━━ ☕ COFFEE REPORT SUMMARY ━━━━━"
echo "🖥️  Host:    $MY_ID"
echo "⏱️  Done:    $(date '+%Y-%m-%d %H:%M:%S')"

if [[ ${#JOB_PASS[@]} -gt 0 ]]; then
    echo "✅  Passed:  ${JOB_PASS[*]}"
fi
if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then
    echo "❌  Failed:  ${JOB_FAIL[*]}"
fi

if [[ ${#JOB_FAIL[@]} -gt 0 && "$DRY_RUN" != true ]]; then
    notify "Sunday coffee report had failures on $(hostname) ($MY_ID) — ${JOB_FAIL[*]}" \
        "Sunday Morning Coffee Report" "warning"
fi

[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1
exit 0