diff --git a/Configurations/master.conf b/Configurations/master.conf index 968fe40..f8d5bce 100644 --- a/Configurations/master.conf +++ b/Configurations/master.conf @@ -422,6 +422,20 @@ MONTHLY_RUN_INTERVAL_DAYS=30 # minimum days since last run before running again MONTHLY_LAST_RUN_FILE="/boot/config/monthly_maintenance_last_run.db" +# ━━━ Sunday Morning Coffee Report ━━━ +# Orchestrator that runs all Sunday monitor scripts in sequence. +# Schedule: 0 7 * * 0 (Sunday 7am — after weekly_sync_maintenance.sh finishes at ~3am) +# Each script runs independently and notifies on its own findings. + COFFEE_REPORT_SCRIPTS=( + "Monitors/zfs_memory_snapshot.sh" # ZFS pool health + ARC + Docker memory snapshot + "Monitors/smart_health.sh" # drive SMART attributes — reallocated, pending, temp + "Monitors/cert_monitor.sh" # SSL certificate expiry for all configured domains + "Monitors/backup_verify.sh" # rsync mirror integrity via independent MD5 checksums + "Monitors/bandwidth_monitor.sh" # weekly rsync transfer totals and per-share breakdown + "Monitors/emby_session_report.sh" # Emby usage — streams, users, library, transcode ratio + "Monitors/weekly_health_digest.sh" # aggregated digest — watchdogs, fallback, skip list + ) + # ============================================================================================== # ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── # ============================================================================================== diff --git a/Orchestrators/sunday_morning_coffee_report.sh b/Orchestrators/sunday_morning_coffee_report.sh old mode 100644 new mode 100755 index 760fe08..42e9469 --- a/Orchestrators/sunday_morning_coffee_report.sh +++ b/Orchestrators/sunday_morning_coffee_report.sh @@ -2,963 +2,122 @@ # ============================================================================================== # ============================= Sunday Morning Coffee Report =================================== # ============================================================================================== -# Weekly system overview — everything that happened this week in one clean read. +# Weekly monitoring orchestrator — runs all Sunday monitor scripts in sequence. # Designed to be read over coffee Sunday morning while the system is fully caught up # from the 2:30am maintenance window. # Schedule: 0 7 * * 0 (7am Sunday — after weekly_sync_maintenance.sh finishes at ~3am) -# Maintenance window completes → 4 hours of fresh data → report ready ☕ # -# ── SECTIONS ────────────────────────────────────────────────────────────────────────────────── -# 🖥️ System — uptime, memory, boot drive, cache drive, reboots this week -# 📀 Array — disk count, parity status, ZFS health, drive temps -# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split -# 🎵 Media Activity — arr cleanup stats, arr recovery stats, queue depth -# 🌐 Rsync — weekly transfer totals, per-share breakdown, failures -# 🛡️ Watchdog — resource watchdog, system watchdog, docker watchdog, fallback state -# 🔐 Security — SSL cert expiry per domain -# 📊 Emby — weekly stream count, active now, top users -# ⚙️ System Health — SMART summary, inotify, php-fpm, Docker, Gitea sync -# ⚠️ Issues — anything requiring attention collected from above sections -# -# ── DATA SOURCES (reads only) ───────────────────────────────────────────────────────────────── -# DATA_DIR stats files — arr cleanup, recovery, transcode, bandwidth history -# /boot/config — fallback state, watchdog reboot log -# /tmp — watchdog strike state files -# /proc, /sys — system memory, uptime, inotify -# /var/local/emhttp/ — unRAID array info -# Emby API — session history, active streams -# Arr APIs — current queue depth (SONARR_URL etc. from detect_hosts()) -# tailscale — remote server reachability -# openssl — live SSL cert check per domain -# smartctl — drive SMART health +# ── SCRIPTS (master.conf COFFEE_REPORT_SCRIPTS) ─────────────────────────────────────────────── +# Each script runs independently, logs to its own output, and notifies on findings. +# All scripts receive --dry-run and --log flags from this orchestrator when set. # # ── HOST AWARENESS ──────────────────────────────────────────────────────────────────────────── -# detect_hosts() sets MY_ID and aliases all HOST*_ vars. -# Report header and footer show MY_ID — clear which server's weekly report this is. -# No manual HOST1/HOST2 comparisons — all via MY_ID/REMOTE_ID. -# -# ── SAFEGUARDS ──────────────────────────────────────────────────────────────────────────────── -# Root check — smartctl, docker, openssl need root -# acquire_lock — prevents duplicate reports -# detect_hosts() — correct vars per server -# DOCKER_TIMEOUT — all docker calls protected -# validate_unraid_cmd — notify validated before use -# openssl check — security section skipped gracefully if not available -# -# ── CONFIGURATION (master.conf) ─────────────────────────────────────────────────────────────── -# NOTIFY_UNRAID — send via unRAID notification system -# DISCORD_WEBHOOK — send to Discord channel (MY_ID_ prefixed per host) -# CERT_MONITOR_DOMAINS / CERT_WARN_DAYS / CERT_CRIT_DAYS -# ZFS_REPORT_IGNORE_POOLS / SMART_IGNORE_DRIVES -# All threshold vars read from master.conf at runtime +# detect_hosts() sets MY_ID — used in banner and summary. +# HOST1 primary: runs all scripts. HOST2: limited to host-aware scripts only. # # ── USAGE ───────────────────────────────────────────────────────────────────────────────────── -# sunday_morning_coffee_report.sh — generate and send report -# sunday_morning_coffee_report.sh --dry-run — generate without sending notification -# sunday_morning_coffee_report.sh --status — show data file availability -# sunday_morning_coffee_report.sh --log — verbose section output +# sunday_morning_coffee_report.sh — normal run +# sunday_morning_coffee_report.sh --dry-run — preview without any writes or notifications +# sunday_morning_coffee_report.sh --log — verbose per-script output +# sunday_morning_coffee_report.sh --status — show configured scripts and exit # ============================================================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/../load_config.sh" +SCRIPTS_ROOT="$SCRIPT_DIR/.." + parse_args "$@" - -DOCKER_TIMEOUT=15 - -# ============================================================================================== -# ━━━ Setup ━━━ -# ============================================================================================== -if [[ "$EUID" -ne 0 ]]; then - error "Must be run as root — smartctl and docker require root" - exit 1 -fi - -validate_unraid_cmd \ - "/usr/local/emhttp/plugins/dynamix/scripts/notify" \ - "" "" \ - "unRAID notify script" || warn "unRAID notify script not found — unRAID notifications disabled" - -acquire_lock - detect_hosts -REPORT_DATE=$(date '+%A, %B %-d, %Y') -WEEK_START=$(date -d "7 days ago" '+%Y-%m-%d') -WEEK_EPOCH=$(date -d "$WEEK_START" +%s) -TODAY=$(date '+%Y-%m-%d') -NOW=$(date +%s) +# ============================================================================================== +# ━━━ Helpers ━━━ +# ============================================================================================== +JOB_PASS=() +JOB_FAIL=() -REPORT=() -ISSUES=() -FINDINGS=() +run_job() { + local script_entry="$1" + local extra_dry="" + local extra_log="" + [[ "$DRY_RUN" == true ]] && extra_dry="--dry-run" + [[ "$ENABLE_LOGGING" == true ]] && extra_log="--log" + + read -r -a script_args <<< "$script_entry" + local script_path="$SCRIPTS_ROOT/${script_args[0]}" + local script_name + script_name=$(basename "${script_args[0]}") + local extra_args=("${script_args[@]:1}") + + if [[ ! -f "$script_path" ]]; then + error "$script_name — not found at $script_path" + JOB_FAIL+=("$script_name") + return 1 + fi + + log "Running: $script_name ${extra_args[*]}" + if bash "$script_path" "${extra_args[@]}" $extra_dry $extra_log; then + log "$script_name — done ✅" + JOB_PASS+=("$script_name") + else + error "$script_name — failed (exit $?)" + JOB_FAIL+=("$script_name") + fi +} # ============================================================================================== # ━━━ Status ━━━ # ============================================================================================== if [[ "$SHOW_STATUS" == true ]]; then - echo "" - echo "━━━━━ $ICON_SUMMARY COFFEE REPORT STATUS ━━━━━" - echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)" - echo "" - echo "━━━ Data Files ━━━" - for f in \ - "$ARR_CLEANUP_STATS:arr cleanup stats" \ - "$ARR_RECOVERY_STATS:arr recovery stats" \ - "$TRANSCODE_DAILY_LOG:transcode daily log" \ - "$BANDWIDTH_LOG:bandwidth log" \ - "$TUNING_MONITOR_LOG:tuning monitor log" \ - "$SYS_WATCHDOG_STATE_FILE:system watchdog state" \ - "$SYS_WATCHDOG_REBOOT_LOG:watchdog reboot log" \ - "$WATCHDOG_STATE_FILE:docker watchdog state" \ - "$WATCHDOG_CONTAINER_RESTART_LOG:container restart log" \ - "$FALLBACK_STATE_FILE:fallback state"; do - path="${f%%:*}" - label="${f##*:}" - if [[ -f "$path" ]] && [[ -s "$path" ]]; then - COUNT=$(wc -l < "$path" 2>/dev/null || echo "?") - echo " $ICON_SUCCESS $label ($COUNT lines)" - elif [[ -f "$path" ]]; then - echo " $ICON_WARN $label (exists but empty)" - else - echo " $ICON_SKIP $label (not found)" - fi - done - echo "" - echo "━━━ Runtime Dependencies ━━━" - command -v smartctl >/dev/null 2>&1 && echo " $ICON_SUCCESS smartctl" || \ - echo " $ICON_SKIP smartctl (not installed)" - command -v openssl >/dev/null 2>&1 && echo " $ICON_SUCCESS openssl" || \ - echo " $ICON_SKIP openssl (security section disabled)" - command -v docker >/dev/null 2>&1 && echo " $ICON_SUCCESS docker" || \ - echo " $ICON_SKIP docker" - command -v zpool >/dev/null 2>&1 && echo " $ICON_SUCCESS zpool" || \ - echo " $ICON_SKIP zpool (ZFS section disabled)" - echo "━━━━━━━━━━━━━━━━━━━━━━━" + echo "━━━ Sunday Morning Coffee Report — Status ━━━" + echo " Host: $MY_ID" + echo " Scripts:" + if [[ ${#COFFEE_REPORT_SCRIPTS[@]} -eq 0 ]]; then + echo " None configured (COFFEE_REPORT_SCRIPTS in master.conf)" + else + for entry in "${COFFEE_REPORT_SCRIPTS[@]}"; do + [[ -n "$entry" ]] && echo " ☕ ${entry##*/}" + done + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi -[[ "$DRY_RUN" == true ]] && warn "DRY RUN — report generated, notification not sent" - # ============================================================================================== -# ── HELPER FUNCTIONS ────────────────────────────────────────────────────────────────────────── +# ━━━ Main ━━━ # ============================================================================================== -section() { - REPORT+=("") - REPORT+=("$1") - REPORT+=("$(printf '%.0s─' {1..50})") -} - -line() { REPORT+=(" $1"); } -issue() { ISSUES+=("$1"); REPORT+=(" ⚠️ $1"); } -finding() { FINDINGS+=("$1"); REPORT+=(" ℹ️ $1"); } - -get_array() { eval "echo \"\${${1}[*]}\""; } - -format_bytes() { - local bytes=$1 - if (( bytes > 1073741824 )); then - awk "BEGIN {printf \"%.1fGB\", $bytes / 1073741824}" - elif (( bytes > 1048576 )); then - awk "BEGIN {printf \"%.1fMB\", $bytes / 1048576}" - else - echo "${bytes}B" - fi -} - -# Lock/uptime helpers — local copies needed in report context -_get_lock_pid() { - local f="$LOCK_DIR/${1}.lock" - [[ -f "$f" ]] && { local c; c=$(cat "$f" 2>/dev/null); echo "${c%%:*}"; } -} -_get_lock_name() { - local f="$LOCK_DIR/${1}.lock" - [[ -f "$f" ]] && { local c; c=$(cat "$f" 2>/dev/null); echo "${c##*:}"; } -} -_is_running() { - local pid name - pid=$(_get_lock_pid "$1"); name=$(_get_lock_name "$1") - [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$name" == "$1" ]] -} -_lock_age() { - local f="$LOCK_DIR/${1}.lock" - [[ -f "$f" ]] && echo $(( NOW - $(stat -c %Y "$f" 2>/dev/null || echo "$NOW") )) || echo 0 -} -_fmt_uptime() { - local s=$1 d=$(($1/86400)) h=$((($1%86400)/3600)) m=$((($1%3600)/60)) - (( d > 0 )) && echo "${d}d ${h}h ${m}m" || \ - (( h > 0 )) && echo "${h}h ${m}m" || echo "${m}m" -} - -# ============================================================================================== -# ━━━ 🖥️ SYSTEM ━━━ -# ============================================================================================== -section "🖥️ SYSTEM — $MY_ID ($LOCAL_SERVER_NAME)" - -UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime) -UPTIME_DAYS=$(( UPTIME_SECONDS / 86400 )) -UPTIME_HOURS=$(( (UPTIME_SECONDS % 86400) / 3600 )) -BOOT_TIME=$(date -d "@$(( NOW - UPTIME_SECONDS ))" '+%A %-d %b at %-I:%M%p') -line "Uptime: ${UPTIME_DAYS}d ${UPTIME_HOURS}hr (up since $BOOT_TIME)" - -REBOOT_COUNT=0 -if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then - REBOOT_COUNT=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \ - "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l) -fi -[[ "${REBOOT_COUNT:-0}" -gt 0 ]] && \ - issue "Reboots this week: $REBOOT_COUNT (system watchdog triggered)" || \ - line "Reboots this week: 0 ✅" - -MEM_TOTAL_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo) -MEM_AVAIL_KB=$(awk '/MemAvailable/{print $2}' /proc/meminfo) -MEM_USED_GB=$(awk "BEGIN {printf \"%.1f\", ($MEM_TOTAL_KB - $MEM_AVAIL_KB) / 1048576}") -MEM_TOTAL_GB=$(awk "BEGIN {printf \"%.0f\", $MEM_TOTAL_KB / 1048576}") -ARC_SIZE="n/a" -if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then - ARC_B=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null || echo 0) - ARC_SIZE=$(awk "BEGIN {printf \"%.1f\", $ARC_B / 1073741824}") -fi -line "Memory: ${MEM_USED_GB}GB used / ${MEM_TOTAL_GB}GB total (ARC: ${ARC_SIZE}GB)" - -BOOT_PCT=$(df /boot --output=pcent 2>/dev/null | tail -1 | tr -d ' %') -BOOT_USED=$(df /boot -h --output=used 2>/dev/null | tail -1 | tr -d ' ') -BOOT_SIZE=$(df /boot -h --output=size 2>/dev/null | tail -1 | tr -d ' ') -[[ "${BOOT_PCT:-0}" -ge 80 ]] && \ - issue "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE}) — getting full" || \ - line "Boot drive: ${BOOT_PCT}% used (${BOOT_USED}/${BOOT_SIZE})" - -CACHE_PCT=$(df /mnt/cache --output=pcent 2>/dev/null | tail -1 | tr -d ' %') -CACHE_AVAIL=$(df /mnt/cache -h --output=avail 2>/dev/null | tail -1 | tr -d ' ') -CACHE_SIZE=$(df /mnt/cache -h --output=size 2>/dev/null | tail -1 | tr -d ' ') -[[ "${CACHE_PCT:-0}" -ge 85 ]] && \ - issue "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})" || \ - line "Cache drive: ${CACHE_PCT}% used (${CACHE_AVAIL} free of ${CACHE_SIZE})" - -# ============================================================================================== -# ━━━ 📀 ARRAY ━━━ -# ============================================================================================== -section "📀 ARRAY" - -if [[ -f /var/local/emhttp/disks.ini ]]; then - DISK_COUNT=$(grep -c '^\["disk[0-9]' /var/local/emhttp/disks.ini 2>/dev/null || echo "?") - PARITY_COUNT=$(grep -c '^\["parity' /var/local/emhttp/disks.ini 2>/dev/null || echo "?") - DISK_COUNT="${DISK_COUNT//[^0-9]/}"; DISK_COUNT="${DISK_COUNT:-?}" - PARITY_COUNT="${PARITY_COUNT//[^0-9]/}"; PARITY_COUNT="${PARITY_COUNT:-?}" - line "Array: ${DISK_COUNT} data disks + ${PARITY_COUNT} parity" -else - line "Array: disks.ini not found" -fi - -PARITY_LOG="/boot/config/parity-checks.log" -if [[ -f "$PARITY_LOG" ]]; then - LAST_CHECK=$(tail -1 "$PARITY_LOG" 2>/dev/null) - if [[ -n "$LAST_CHECK" ]]; then - PARITY_DATE=$(echo "$LAST_CHECK" | cut -d'|' -f1 | xargs) - PARITY_ERRORS=$(echo "$LAST_CHECK" | cut -d'|' -f4) - PARITY_ACTION=$(echo "$LAST_CHECK" | cut -d'|' -f6) - PARITY_DURATION=$(echo "$LAST_CHECK" | cut -d'|' -f2) - PARITY_DUR_HR=$(( PARITY_DURATION / 3600 )) - PARITY_DUR_MIN=$(( (PARITY_DURATION % 3600) / 60 )) - if [[ "${PARITY_ERRORS:-0}" -gt 0 ]]; then - issue "Parity: $PARITY_ERRORS errors on last check ($PARITY_DATE)" - elif [[ "${PARITY_ERRORS:-0}" -lt 0 ]]; then - CORRECTIONS=$(( PARITY_ERRORS * -1 )) - line "Parity: OK — last: $PARITY_DATE ($PARITY_ACTION, ${PARITY_DUR_HR}h${PARITY_DUR_MIN}m, $CORRECTIONS correction(s)) ✅" - else - line "Parity: OK — last: $PARITY_DATE ($PARITY_ACTION, ${PARITY_DUR_HR}h${PARITY_DUR_MIN}m, 0 errors) ✅" - fi - RESYNC=$(grep "^mdResync=" /var/local/emhttp/var.ini 2>/dev/null | cut -d= -f2 | tr -d '"') - [[ "$RESYNC" != "0" ]] && finding "Parity check currently in progress" - fi -else - line "Parity: no check log found" -fi - -if command -v zpool >/dev/null 2>&1; then - while IFS=$'\t' read -r pool health; do - [[ -z "$pool" ]] && continue - SKIP=false - for ignore in "${ZFS_REPORT_IGNORE_POOLS[@]}"; do - [[ "$pool" == "$ignore" ]] && SKIP=true && break - done - [[ "$SKIP" == true ]] && continue - [[ "$health" == "ONLINE" ]] && \ - line "ZFS $pool: ONLINE ✅" || \ - issue "ZFS $pool: $health — check immediately" - done < <(zpool list -H -o name,health 2>/dev/null) -fi - -if command -v smartctl >/dev/null 2>&1; then - declare -A DISK_TEMPS - ALL_NORMAL=true - get_unraid_temp_thresholds - - for disk in /dev/sd? /dev/nvme?; do - [[ ! -e "$disk" ]] && continue - DISK_NAME=$(basename "$disk") - SKIP=false - for ignore in "${SMART_IGNORE_DRIVES[@]}"; do - [[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break - done - [[ "$SKIP" == true ]] && continue - - TEMP=$(smartctl -A "$disk" 2>/dev/null | awk ' - /^190 / || /^194 / { print $10; exit } - /Temperature_Celsius/ { print $10; exit } - ') - [[ -z "$TEMP" ]] && TEMP=$(smartctl -A "$disk" 2>/dev/null | \ - awk '/^Temperature:/ { print $2; exit }') - TEMP="${TEMP//[^0-9]/}" - [[ -z "$TEMP" ]] && continue - - DISK_TEMPS["$DISK_NAME"]="$TEMP" - - if is_ssd "$disk"; then - WARN_T="${UNRAID_SSD_HOT:-50}"; CRIT_T="${UNRAID_SSD_MAX:-60}" - else - WARN_T="${UNRAID_DISK_HOT:-45}"; CRIT_T="${UNRAID_DISK_MAX:-55}" - fi - - if [[ "$TEMP" -ge "$CRIT_T" ]]; then - issue "Drive $DISK_NAME: ${TEMP}°C — CRITICAL" - ALL_NORMAL=false - elif [[ "$TEMP" -ge "$WARN_T" ]]; then - finding "Drive $DISK_NAME: ${TEMP}°C — warm" - ALL_NORMAL=false - fi - done - - [[ "$ALL_NORMAL" == true ]] && line "Drive temps: all normal ✅" - - if [[ -f /var/local/emhttp/disks.ini ]] && [[ ${#DISK_TEMPS[@]} -gt 0 ]]; then - DISK_SUMMARY="" - CURRENT_DISK="" - while IFS= read -r ini_line; do - if echo "$ini_line" | grep -qE '^\["(disk[0-9]+|parity[0-9]?|cache)"\]'; then - CURRENT_DISK=$(echo "$ini_line" | grep -o '"[^"]*"' | head -1 | tr -d '"') - elif echo "$ini_line" | grep -q '^device='; then - DEV=$(echo "$ini_line" | cut -d= -f2 | tr -d '"') - TEMP="${DISK_TEMPS[$DEV]:-}" - if [[ -n "$TEMP" ]]; then - [[ -n "$DISK_SUMMARY" ]] && DISK_SUMMARY+=", " - DISK_SUMMARY+="${CURRENT_DISK}(${DEV}):${TEMP}°C" - fi - fi - done < /var/local/emhttp/disks.ini - [[ -n "$DISK_SUMMARY" ]] && line "Temps: $DISK_SUMMARY" - fi -fi - -# ============================================================================================== -# ━━━ 🎬 TRANSCODES ━━━ -# ============================================================================================== -section "🎬 TRANSCODES" - -if mountpoint -q "$RAMDISK_PATH" 2>/dev/null; then - RAMDISK_USED_KB=$(df "$RAMDISK_PATH" --output=used 2>/dev/null | tail -1 | tr -d ' ') - RAMDISK_AVAIL_KB=$(df "$RAMDISK_PATH" --output=avail 2>/dev/null | tail -1 | tr -d ' ') - RAMDISK_USED_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_USED_KB / 1048576}") - RAMDISK_AVAIL_GB=$(awk "BEGIN {printf \"%.2f\", $RAMDISK_AVAIL_KB / 1048576}") - SYMLINK_TARGET=$(readlink "$TRANSCODE_LINK" 2>/dev/null | xargs basename 2>/dev/null || echo "unknown") - line "Ramdisk now: ${RAMDISK_USED_GB}GB used / ${RAMDISK_AVAIL_GB}GB free → $SYMLINK_TARGET" -else - issue "Ramdisk not mounted at $RAMDISK_PATH" -fi - -if [[ -f "${TRANSCODE_DAILY_LOG:-}" ]] && [[ -s "$TRANSCODE_DAILY_LOG" ]]; then - WEEK_PEAK=$(awk -F'|' -v c="$WEEK_START" '$1>=c {if($2>max)max=$2} END{printf "%.2f",max+0}' "$TRANSCODE_DAILY_LOG") - WEEK_FLIPS=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$3} END{print sum+0}' "$TRANSCODE_DAILY_LOG") - WEEK_RAM=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$4} END{print sum+0}' "$TRANSCODE_DAILY_LOG") - WEEK_SSD=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$5} END{print sum+0}' "$TRANSCODE_DAILY_LOG") - line "Week peak: ${WEEK_PEAK}GB | flips: ${WEEK_FLIPS}" - line "Sessions: ${WEEK_RAM} ramdisk / ${WEEK_SSD} SSD" - PEAK_INT=$(printf "%.0f" "$WEEK_PEAK" 2>/dev/null || echo 0) - WARN_INT=$(printf "%.0f" "${RAMDISK_WARN_GB:-6.8}" 2>/dev/null || echo 0) - [[ "$PEAK_INT" -ge "$WARN_INT" ]] && \ - finding "Transcode peak ${WEEK_PEAK}GB near threshold — consider increasing HOST*_RAMDISK_SIZE" - [[ "${WEEK_FLIPS:-0}" -ge "${TRANSCODE_FLIP_WARN:-3}" ]] && \ - finding "Transcode flips this week: $WEEK_FLIPS — monitor ramdisk headroom" -fi - -# ============================================================================================== -# ━━━ 🎵 MEDIA ACTIVITY ━━━ -# ============================================================================================== -section "🎵 MEDIA ACTIVITY" - -if [[ -f "${ARR_CLEANUP_STATS:-}" ]] && [[ -s "$ARR_CLEANUP_STATS" ]]; then - for arr in lidarr sonarr radarr; do - WEEK_ORPHANS=$(awk -F'|' -v c="$WEEK_START" -v a="$arr" \ - '$1>=c && $2==a {sum+=$3} END{print sum+0}' "$ARR_CLEANUP_STATS") - WEEK_BYTES=$(awk -F'|' -v c="$WEEK_START" -v a="$arr" \ - '$1>=c && $2==a {sum+=$4} END{print sum+0}' "$ARR_CLEANUP_STATS") - WEEK_TRACKED=$(awk -F'|' -v c="$WEEK_START" -v a="$arr" \ - 'BEGIN{max=0} $1>=c && $2==a && $8+0>max {max=$8} END{print max+0}' "$ARR_CLEANUP_STATS") - if [[ "${WEEK_ORPHANS:-0}" -gt 0 ]]; then - FREED=$(format_bytes "${WEEK_BYTES:-0}") - line "${arr^} cleanup: $WEEK_ORPHANS orphans removed ($FREED freed) | tracked: $WEEK_TRACKED files" - else - line "${arr^} cleanup: clean ✅ (tracked: $WEEK_TRACKED files)" - fi - done -else - line "Arr cleanup stats: no data yet (runs after first weekly cleanup)" -fi - -if [[ -f "${ARR_RECOVERY_STATS:-}" ]] && [[ -s "$ARR_RECOVERY_STATS" ]]; then - WEEK_ACTIONED=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$3} END{print sum+0}' "$ARR_RECOVERY_STATS") - WEEK_RUNS=$(awk -F'|' -v c="$WEEK_START" '$1>=c {count++} END{print count+0}' "$ARR_RECOVERY_STATS") - if [[ "${WEEK_ACTIONED:-0}" -gt 0 ]]; then - line "Arr recovery: $WEEK_ACTIONED items auto-recovered across $WEEK_RUNS runs" - else - line "Arr recovery: no failed imports this week ✅" - fi -fi - -# Arr queue depth — uses detect_hosts() aliased SONARR_URL/RADARR_URL/LIDARR_URL -for arr_entry in "Sonarr|${SONARR_URL:-}|${SONARR_API_KEY:-}|v3" \ - "Radarr|${RADARR_URL:-}|${RADARR_API_KEY:-}|v3" \ - "Lidarr|${LIDARR_URL:-}|${LIDARR_API_KEY:-}|v1"; do - IFS='|' read -r name url key ver <<< "$arr_entry" - [[ -z "$url" || -z "$key" ]] && continue - QUEUE=$(curl -sf --max-time 5 -H "X-Api-Key: $key" \ - "${url}/api/${ver}/queue?pageSize=1" 2>/dev/null | \ - grep -o '"totalRecords":[0-9]*' | grep -o '[0-9]*' || echo "?") - if [[ "$QUEUE" == "0" || -z "$QUEUE" ]]; then - line "$name queue: empty ✅" - elif [[ "$QUEUE" == "?" ]]; then - finding "$name queue: API unavailable" - else - line "$name queue: $QUEUE items" - fi -done - -# ============================================================================================== -# ━━━ 🌐 RSYNC ━━━ -# ============================================================================================== -section "🌐 RSYNC" - -# New bandwidth log format: date|time|profile|duration|status|bytes|warn_flag -if [[ -f "${BANDWIDTH_LOG:-}" ]] && [[ -s "$BANDWIDTH_LOG" ]]; then - WEEK_TOTAL_BYTES=$(awk -F'|' -v c="$WEEK_START" '$1>=c {sum+=$6} END{print sum+0}' "$BANDWIDTH_LOG") - WEEK_SYNCS=$(awk -F'|' -v c="$WEEK_START" '$1>=c' "$BANDWIDTH_LOG" | wc -l) - WEEK_FAILED=$(awk -F'|' -v c="$WEEK_START" '$1>=c && $5!="success"' "$BANDWIDTH_LOG" | wc -l) - WEEK_LARGE=$(awk -F'|' -v c="$WEEK_START" '$1>=c && $7=="LARGE"' "$BANDWIDTH_LOG" | wc -l) - WEEK_GB=$(awk "BEGIN {printf \"%.1f\", $WEEK_TOTAL_BYTES / 1073741824}") - line "Total: ${WEEK_GB}GB across $WEEK_SYNCS syncs" - [[ "${WEEK_FAILED:-0}" -gt 0 ]] && issue "Failed syncs this week: $WEEK_FAILED" || line "Sync failures: none ✅" - [[ "${WEEK_LARGE:-0}" -gt 0 ]] && finding "Large transfers (>${BANDWIDTH_WARN_GB}GB): $WEEK_LARGE" - - # Top 3 profiles by transfer — profile is field $3 - TOP_SHARES=$(awk -F'|' -v c="$WEEK_START" \ - '$1>=c {bytes[$3]+=$6} END {for(s in bytes) print bytes[s], s}' \ - "$BANDWIDTH_LOG" | sort -rn | head -3) - if [[ -n "$TOP_SHARES" ]]; then - while IFS=' ' read -r bytes profile; do - [[ -z "$profile" ]] && continue - line " → $profile: $(format_bytes "$bytes")" - done <<< "$TOP_SHARES" - fi -else - line "No bandwidth data yet" -fi - -# ============================================================================================== -# ━━━ 🛡️ WATCHDOG ━━━ -# ============================================================================================== -section "🛡️ WATCHDOG" - -# ── System Watchdog ─────────────────────────────────────────────────────────────────────────── -line "⚙️ System Watchdog (cron via watchdog_orchestrator)" -if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then - _sw_last=$(stat -c %Y "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || echo 0) - _sw_ago=$(( NOW - _sw_last )) - if [[ "$_sw_ago" -lt 600 ]]; then - line " ✅ Last run: $(_fmt_uptime "$_sw_ago") ago" - else - issue " Last run: $(_fmt_uptime "$_sw_ago") ago — watchdog_orchestrator may not be running" - fi -else - issue " stability_watchdog has never run (state file missing)" -fi - -if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then - SYS_ACTIVE=$(grep -v ":0$\|^watchdog_cycle=" \ - "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") - if [[ -n "$SYS_ACTIVE" ]]; then - while IFS=: read -r key count; do - [[ -z "$key" ]] && continue - issue " Strike: $key — $count/$SYS_WATCHDOG_STRIKE_LIMIT" - done <<< "$SYS_ACTIVE" - else - line " ✅ Strikes: none" - fi -fi - -if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then - WD_REBOOTS=$(awk -v c="$WEEK_EPOCH" '$1>=c' "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l) - [[ "${WD_REBOOTS:-0}" -gt 0 ]] && \ - issue " Watchdog reboots this week: $WD_REBOOTS" || \ - line " ✅ Watchdog reboots this week: 0" -fi - -if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then - SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE") - SKIP_LIST=$(tr '\n' ' ' < "$SYS_WATCHDOG_FAILED_FILE") - issue " Skip list ($SKIP_COUNT): $SKIP_LIST" -else - line " ✅ Skip list: empty" -fi - -ROOTFS_PCT=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %') -MEM_AVAIL_GB=$(awk '/MemAvailable/{printf "%.1f",$2/1048576}' /proc/meminfo) -MEM_TOTAL_GB_SYS=$(awk '/MemTotal/{printf "%.0f",$2/1048576}' /proc/meminfo) -LOAD_NOW=$(awk '{print $1}' /proc/loadavg) -ZOMBIE_NOW=$(ps aux 2>/dev/null | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0) -ZOMBIE_NOW="${ZOMBIE_NOW//[^0-9]/}"; ZOMBIE_NOW="${ZOMBIE_NOW:-0}" -ARC_NOW="n/a" -[[ -f /proc/spl/kstat/zfs/arcstats ]] && \ - ARC_NOW=$(awk '/^size /{printf "%.1f",$3/1073741824}' /proc/spl/kstat/zfs/arcstats) -line " 📊 rootfs:${ROOTFS_PCT}% │ RAM:${MEM_AVAIL_GB}GB free/${MEM_TOTAL_GB_SYS}GB │ ARC:${ARC_NOW}GB │ load:${LOAD_NOW} │ zombies:${ZOMBIE_NOW}" - -REPORT+=("") - -# ── Docker Watchdog ─────────────────────────────────────────────────────────────────────────── -line "🐳 Docker Watchdog (cron via watchdog_orchestrator)" -if [[ -f "$WATCHDOG_STATE_FILE" ]]; then - _dw_last=$(stat -c %Y "$WATCHDOG_STATE_FILE" 2>/dev/null || echo 0) - _dw_ago=$(( NOW - _dw_last )) - if [[ "$_dw_ago" -lt 600 ]]; then - line " ✅ Last run: $(_fmt_uptime "$_dw_ago") ago" - else - issue " Last run: $(_fmt_uptime "$_dw_ago") ago — watchdog_orchestrator may not be running" - fi -else - issue " docker_watchdog has never run (state file missing)" -fi - -if [[ -f "$WATCHDOG_STATE_FILE" ]]; then - DOCK_ACTIVE=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$") - if [[ -n "$DOCK_ACTIVE" ]]; then - while IFS=: read -r key count; do - [[ -z "$key" ]] && continue - issue " Strike: $key — $count" - done <<< "$DOCK_ACTIVE" - else - line " ✅ Container strikes: none" - fi -fi - -if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then - WK_RESTARTS=$(awk -F'|' -v c="$WEEK_START" '$2>=c' \ - "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l) - if [[ "${WK_RESTARTS:-0}" -gt 0 ]]; then - RESTARTED=$(awk -F'|' -v c="$WEEK_START" '$2>=c{print $1}' \ - "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \ - sort | uniq -c | sort -rn | head -5 | \ - awk '{print $2"("$1")"}' | tr '\n' ' ') - finding " Container restarts this week: $WK_RESTARTS — $RESTARTED" - else - line " ✅ Container restarts this week: none" - fi -fi - -# ── Resource Watchdog ───────────────────────────────────────────────────────────────────────── -line "🎛️ Resource Watchdog" -if [[ -f "$RW_STATE_FILE" ]]; then - _rm_last=$(stat -c %Y "$RW_STATE_FILE" 2>/dev/null || echo 0) - _rm_ago=$(( NOW - _rm_last )) - _rm_level=$(grep "^current_level:" "$RW_STATE_FILE" 2>/dev/null | cut -d: -f2) - _rm_level="${_rm_level:-0}" - if [[ "$_rm_level" -gt 0 ]]; then - issue " Pressure level ${_rm_level} active │ Last run: $(_fmt_uptime "$_rm_ago") ago" - elif [[ "$_rm_ago" -lt 600 ]]; then - line " ✅ Level 0 (normal) │ Last run: $(_fmt_uptime "$_rm_ago") ago" - else - issue " Last run: $(_fmt_uptime "$_rm_ago") ago — watchdog_orchestrator may not be running" - fi -else - line " ℹ️ State file not found (resource_watchdog may not have run yet)" -fi - -REPORT+=("") - -if command -v docker >/dev/null 2>&1; then - RUNNING_NOW=$(timeout "$DOCKER_TIMEOUT" docker ps -q 2>/dev/null | wc -l) - TOTAL_NOW=$( timeout "$DOCKER_TIMEOUT" docker ps -aq 2>/dev/null | wc -l) - UNHEALTHY_NOW=$(timeout "$DOCKER_TIMEOUT" docker ps \ - --filter health=unhealthy -q 2>/dev/null | wc -l) - STOPPED_NAMES=() - while IFS= read -r name; do - [[ -z "$name" ]] && continue - SKIP=false - for ignore in "${WATCHDOG_SCAN_IGNORE[@]}"; do - [[ "$name" == "$ignore" ]] && SKIP=true && break - done - [[ "$SKIP" == false ]] && STOPPED_NAMES+=("$name") - done < <(timeout "$DOCKER_TIMEOUT" docker ps -af "status=exited" \ - --format "{{.Names}}" 2>/dev/null) - line " 📦 $RUNNING_NOW/$TOTAL_NOW running │ unhealthy: $UNHEALTHY_NOW" - [[ ${#STOPPED_NAMES[@]} -gt 0 ]] && \ - issue " Stopped (unexpected): ${STOPPED_NAMES[*]}" || \ - line " ✅ All containers running" -fi - -REPORT+=("") - -# ── Failover ───────────────────────────────────────────────────────────────────────────────── -line "🔀 Fallback" -FO_PID=$(_get_lock_pid "fallback") -if _is_running "fallback"; then - FO_AGE=$(_lock_age "fallback") - line " ✅ Running │ PID: $FO_PID │ Uptime: $(_fmt_uptime "$FO_AGE")" -elif [[ "${FALLBACK_ENABLED:-true}" == false ]]; then - line " ⏸️ Not running — FALLBACK_ENABLED=false" -else - issue "fallback NOT RUNNING" -fi - -FO_STATE="UNKNOWN" -FO_STATE_SECS=0 -if [[ -f "$FALLBACK_STATE_FILE" ]]; then - FO_STATE=$(grep "^state=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) - FO_EPOCH=$(grep "^last_change_epoch=" "$FALLBACK_STATE_FILE" 2>/dev/null | cut -d= -f2) - [[ -n "$FO_EPOCH" ]] && FO_STATE_SECS=$(( NOW - FO_EPOCH )) -fi - -FO_DUR=$(_fmt_uptime "${FO_STATE_SECS:-0}") - -case "$FO_STATE" in - NORMAL) - line " ✅ State: NORMAL │ Duration: $FO_DUR" ;; - FALLBACK) - issue " State: FALLBACK — $REMOTE_SERVER_NAME down for $FO_DUR" - FO_MINS=$(( FO_STATE_SECS / 60 )) - # Use REMOTE_ID-based tier delay vars — no HOST1/HOST2 hardcoding - T2_VAR="${REMOTE_ID}_TIER2_DELAY"; T3_VAR="${REMOTE_ID}_TIER3_DELAY"; T4_VAR="${REMOTE_ID}_TIER4_DELAY" - T2="${!T2_VAR:-240}"; T3="${!T3_VAR:-720}"; T4="${!T4_VAR:-1440}" - (( FO_MINS >= T2 )) && line " Tier 2: ✅ active" || line " Tier 2: ⏳ in $(( T2 - FO_MINS ))min" - (( FO_MINS >= T3 )) && line " Tier 3: ✅ active" || line " Tier 3: ⏳ in $(( T3 - FO_MINS ))min" - (( FO_MINS >= T4 )) && line " Tier 4: ✅ active" || line " Tier 4: ⏳ in $(( T4 - FO_MINS ))min" - ;; - NO_INTERNET) issue " State: NO_INTERNET — DDNS stopped │ Duration: $FO_DUR" ;; - DARK) issue " State: DARK — remote down AND no internet │ Duration: $FO_DUR" ;; - *) finding " State: ${FO_STATE:-unknown}" ;; -esac - -if command -v tailscale >/dev/null 2>&1; then - REMOTE_TS_IP=$(tailscale ip -4 "${REMOTE_SERVER_NAME,,}" 2>/dev/null) - if [[ -n "$REMOTE_TS_IP" ]]; then - ping -c 1 -W 2 "$REMOTE_TS_IP" >/dev/null 2>&1 && \ - line " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_TS_IP — reachable ✅" || \ - issue " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME): $REMOTE_TS_IP — not responding" - else - issue " 🌐 $REMOTE_ID ($REMOTE_SERVER_NAME) not visible on Tailscale" - fi -fi - -# ============================================================================================== -# ━━━ 🔗 MESH ━━━ -# ============================================================================================== -section "🔗 MESH" - -_ALL_HOST_IDS=() -for _h in HOST1 HOST2 HOST3 HOST4 HOST5 HOST6 HOST7 HOST8; do - [[ -n "${!_h:-}" ]] && _ALL_HOST_IDS+=("$_h") -done - -# ── Members ── -if [[ ${#_ALL_HOST_IDS[@]} -eq 0 ]]; then - line "No hosts defined" -else - line "Members:" - for _h in "${_ALL_HOST_IDS[@]}"; do - _server="${!_h}" - _owner_var="${_h}_OWNER"; _owner="${!_owner_var:-unknown}" - _email_var="${_h}_OWNER_EMAIL"; _email="${!_email_var:-(not set)}" - line " $_h $_server / $_owner / $_email" - done -fi - -# ── Protected Services ── -_coverage_found=false -for _covered in "${_ALL_HOST_IDS[@]}"; do - _covered_owner_var="${_covered}_OWNER"; _covered_owner="${!_covered_owner_var:-$_covered}" - _covered_email_var="${_covered}_OWNER_EMAIL"; _covered_email="${!_covered_email_var:-}" - - declare -A _tier_containers=() - declare -A _tier_delays=() - _covered_by="" - _any_tiers=false - - for _covering in "${_ALL_HOST_IDS[@]}"; do - [[ "$_covering" == "$_covered" ]] && continue - for _tier in 1 2 3 4; do - _containers=$(get_array "FALLBACK_${_covering}_COVERS_${_covered}_TIER${_tier}") - [[ -z "$_containers" ]] && continue - _any_tiers=true - _tier_containers[$_tier]="$_containers" - _delay_var="${_covered}_TIER${_tier}_DELAY" - _tier_delays[$_tier]="${!_delay_var:-0}" - done - if [[ "$_any_tiers" == true ]]; then - _cov_owner_var="${_covering}_OWNER"; _cov_owner="${!_cov_owner_var:-$_covering}" - _covered_by="$_covering ($_cov_owner)" - fi - done - - [[ "$_any_tiers" == false ]] && { unset _tier_containers _tier_delays; declare -A _tier_containers=() _tier_delays=(); continue; } - - _coverage_found=true - _hdr="$_covered_owner" - [[ -n "$_covered_email" ]] && _hdr+=" — $_covered_email" - line "Protected: $_hdr" - for _tier in 1 2 3 4; do - [[ -z "${_tier_containers[$_tier]:-}" ]] && continue - _d="${_tier_delays[$_tier]:-0}" - if (( _d == 0 )); then _dlabel="immediate" - elif (( _d >= 1440 )); then _dlabel="$(( _d / 1440 ))d" - elif (( _d >= 60 )); then _dlabel="$(( _d / 60 ))hr" - else _dlabel="${_d}min" - fi - line " Tier $_tier (${_dlabel}): ${_tier_containers[$_tier]// /, }" - done - [[ -n "$_covered_by" ]] && line " Covered by: $_covered_by" - - unset _tier_containers _tier_delays - declare -A _tier_containers=() _tier_delays=() -done - -[[ "$_coverage_found" == false ]] && line "No fallback coverage configured" - -# ── Partnership ── -if [[ "${PARTNERSHIP_ENABLED:-false}" == true ]]; then - _po_host="${PARTNERSHIP_OWNER_HOST:-HOST1}" - _po_server="${!_po_host:-unknown}" - _po_name_var="${_po_host}_OWNER"; _po_name="${!_po_name_var:-unknown}" - line "Partnership: enabled — owner $_po_host ($_po_server / $_po_name), sync every ${PARTNERSHIP_SYNC_INTERVAL:-15}min" -else - line "Partnership: disabled" -fi - -# ============================================================================================== -# ━━━ 🔐 SECURITY ━━━ -# ============================================================================================== -section "🔐 SECURITY" - -if ! command -v openssl >/dev/null 2>&1; then - line "SSL certs: openssl not available — skipping" -elif [[ ${#CERT_MONITOR_DOMAINS[@]} -eq 0 ]]; then - line "SSL certs: no domains configured (CERT_MONITOR_DOMAINS empty)" -else - for domain in "${CERT_MONITOR_DOMAINS[@]}"; do - [[ -z "$domain" ]] && continue - EXPIRY=$(echo | timeout "${CERT_TIMEOUT:-10}" openssl s_client \ - -connect "${domain}:443" -servername "$domain" 2>/dev/null | \ - openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2) - if [[ -z "$EXPIRY" ]]; then - issue "$domain: could not check certificate" - continue - fi - EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s 2>/dev/null || echo 0) - DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW) / 86400 )) - if [[ "$DAYS_LEFT" -le "${CERT_CRIT_DAYS:-7}" ]]; then - issue "$domain: ${DAYS_LEFT} days remaining — CRITICAL, renew now" - elif [[ "$DAYS_LEFT" -le "${CERT_WARN_DAYS:-30}" ]]; then - finding "$domain: ${DAYS_LEFT} days remaining — renew soon" - else - line "$domain: ${DAYS_LEFT} days remaining ✅" - fi - done -fi - -# ============================================================================================== -# ━━━ 📊 EMBY ━━━ -# ============================================================================================== -section "📊 EMBY" - -# detect_hosts() already aliased EMBY_URL and EMBY_API_KEY -if [[ -z "${EMBY_API_KEY:-}" ]]; then - line "Emby: API key not configured" -elif [[ "${EMBY_API_KEY:-}" == *"your-"* ]]; then - line "Emby: placeholder API key — configure HOST*_EMBY_API_KEY" -else - EMBY_SYSTEM=$(curl -sf --max-time 5 \ - -H "X-Emby-Token: $EMBY_API_KEY" \ - "${EMBY_URL}/System/Info" 2>/dev/null) - - if [[ -z "$EMBY_SYSTEM" ]]; then - line "Emby: API unavailable — check if Emby is running" - else - EMBY_VERSION=$(echo "$EMBY_SYSTEM" | grep -o '"Version":"[^"]*"' | cut -d'"' -f4) - line "Emby: v${EMBY_VERSION:-unknown} — reachable ✅" - - SESSIONS=$(curl -sf --max-time 5 \ - -H "X-Emby-Token: $EMBY_API_KEY" \ - "${EMBY_URL}/Sessions" 2>/dev/null) - ACTIVE_COUNT=$(echo "$SESSIONS" | grep -o '"NowPlayingItem"' | wc -l) - ACTIVE_COUNT="${ACTIVE_COUNT//[^0-9]/}"; ACTIVE_COUNT="${ACTIVE_COUNT:-0}" - line "Active streams now: $ACTIVE_COUNT" - - ACTIVITY_LOG=$(curl -sf --max-time 10 \ - -H "X-Emby-Token: $EMBY_API_KEY" \ - "${EMBY_URL}/user_usage_stats/user_activity?days=7" 2>/dev/null) - - if [[ -n "$ACTIVITY_LOG" ]] && echo "$ACTIVITY_LOG" | grep -q "user_name"; then - TOTAL_PLAYS=$(echo "$ACTIVITY_LOG" | \ - grep -o '"total_plays":[0-9]*' | \ - awk -F: '{sum+=$2} END {print sum+0}') - line "Streams this week: $TOTAL_PLAYS total plays" - echo "$ACTIVITY_LOG" | \ - grep -o '"user_name":"[^"]*","total_plays":[0-9]*' | \ - awk -F'"' '{name=$4; plays=$NF; gsub(/.*:/,"",plays); print plays, name}' | \ - sort -rn | head -3 | \ - while read -r plays name; do - line " → $name: $plays plays" - done - else - line "Weekly stats: user_usage_stats plugin not available" - fi - fi -fi - -# ============================================================================================== -# ━━━ ⚙️ SYSTEM HEALTH ━━━ -# ============================================================================================== -section "⚙️ SYSTEM HEALTH" - -if [[ -f "${TUNING_MONITOR_LOG:-}" ]] && [[ -s "$TUNING_MONITOR_LOG" ]]; then - INOTIFY_PEAK=$(awk -F'|' -v c="$WEEK_START" \ - '$1>=c {if($3+0>max) max=$3+0} END{print max+0}' "$TUNING_MONITOR_LOG") - INOTIFY_AVG=$(awk -F'|' -v c="$WEEK_START" \ - '$1>=c {sum+=$3;cnt++} END{if(cnt>0)printf "%.0f",sum/cnt;else print 0}' "$TUNING_MONITOR_LOG") - INOTIFY_WARNS=$(awk -F'|' -v c="$WEEK_START" \ - '$1>=c && $6==1 {cnt++} END{print cnt+0}' "$TUNING_MONITOR_LOG") - INOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l) - INOW="${INOW//[^0-9]/}"; INOW="${INOW:-0}" - ILIM=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024) - IPCT=$(( INOW * 100 / ILIM )) - [[ "${INOTIFY_WARNS:-0}" -gt 0 ]] && \ - issue "inotify: ${INOW}/${ILIM} now (${IPCT}%) | week peak:$INOTIFY_PEAK avg:$INOTIFY_AVG | ⚠️ $INOTIFY_WARNS warning(s)" || \ - line "inotify: ${INOW}/${ILIM} now (${IPCT}%) | week peak:$INOTIFY_PEAK avg:$INOTIFY_AVG ✅" - - PHPFPM_PEAK=$(awk -F'|' -v c="$WEEK_START" \ - '$1>=c {if($7+0>max) max=$7+0} END{print max+0}' "$TUNING_MONITOR_LOG") - PHPFPM_AVG=$(awk -F'|' -v c="$WEEK_START" \ - '$1>=c {sum+=$7;cnt++} END{if(cnt>0)printf "%.0f",sum/cnt;else print 0}' "$TUNING_MONITOR_LOG") - PHPFPM_WARNS=$(awk -F'|' -v c="$WEEK_START" \ - '$1>=c && $10==1 {cnt++} END{print cnt+0}' "$TUNING_MONITOR_LOG") - PNOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0) - PNOW="${PNOW//[^0-9]/}"; PNOW="${PNOW:-0}" - PMAX="${PHP_MAX_CHILDREN:-250}" - [[ "${PHPFPM_WARNS:-0}" -gt 0 ]] && \ - issue "php-fpm: ${PNOW}/${PMAX} workers now | week peak:$PHPFPM_PEAK avg:$PHPFPM_AVG | ⚠️ $PHPFPM_WARNS warning(s)" || \ - line "php-fpm: ${PNOW}/${PMAX} workers now | week peak:$PHPFPM_PEAK avg:$PHPFPM_AVG ✅" -else - INOW=$(find /proc/*/fd -lname 'anon_inode:inotify' 2>/dev/null | wc -l) - INOW="${INOW//[^0-9]/}"; INOW="${INOW:-0}" - ILIM=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 1024) - PNOW=$(ps aux 2>/dev/null | grep -c "php-fpm: pool" || echo 0) - PNOW="${PNOW//[^0-9]/}"; PNOW="${PNOW:-0}" - line "inotify: ${INOW}/${ILIM} — no weekly data yet" - line "php-fpm: ${PNOW}/${PHP_MAX_CHILDREN:-250} workers — no weekly data yet" -fi - -if command -v smartctl >/dev/null 2>&1; then - SMART_ISSUES=0 - for disk in /dev/sd? /dev/nvme?; do - [[ ! -e "$disk" ]] && continue - DISK_NAME=$(basename "$disk") - SKIP=false - for ignore in "${SMART_IGNORE_DRIVES[@]}"; do - [[ "$DISK_NAME" == "$ignore" ]] && SKIP=true && break - done - [[ "$SKIP" == true ]] && continue - HEALTH=$(smartctl -H "$disk" 2>/dev/null | grep "SMART overall-health" | awk '{print $NF}') - if [[ "$HEALTH" != "PASSED" ]] && [[ -n "$HEALTH" ]]; then - issue "SMART $DISK_NAME: $HEALTH — check immediately" - (( SMART_ISSUES++ )) - fi - done - [[ "$SMART_ISSUES" -eq 0 ]] && line "SMART: all drives PASSED ✅" -fi - -if command -v git >/dev/null 2>&1 && [[ -d "${TARGET_DIR:-}/.git" ]]; then - CURRENT_COMMIT=$(git -C "$TARGET_DIR" rev-parse --short HEAD 2>/dev/null || echo "unknown") - LAST_PULL=$(git -C "$TARGET_DIR" log -1 --format="%ar" 2>/dev/null || echo "unknown") - line "Gitea: commit $CURRENT_COMMIT (pulled $LAST_PULL)" -fi - -# ============================================================================================== -# ━━━ ⚠️ ISSUES REQUIRING ATTENTION ━━━ -# ============================================================================================== -if [[ ${#ISSUES[@]} -gt 0 ]]; then - REPORT+=("") - REPORT+=("⚠️ ISSUES REQUIRING ATTENTION") - REPORT+=("$(printf '%.0s─' {1..50})") - for issue_line in "${ISSUES[@]}"; do - REPORT+=(" ❌ $issue_line") - done -fi - -if [[ ${#FINDINGS[@]} -gt 0 ]]; then - REPORT+=("") - REPORT+=("ℹ️ NOTABLE") - REPORT+=("$(printf '%.0s─' {1..50})") - for finding_line in "${FINDINGS[@]}"; do - REPORT+=(" → $finding_line") - done -fi - -# ============================================================================================== -# ━━━ Footer ━━━ -# ============================================================================================== -REPORT+=("") -if [[ ${#ISSUES[@]} -eq 0 ]]; then - REPORT+=("✅ All systems healthy — enjoy your Sunday ☕") -else - REPORT+=("⚠️ ${#ISSUES[@]} issue(s) need attention") -fi -REPORT+=("$MY_ID ($LOCAL_SERVER_NAME) — $(date '+%Y-%m-%d %H:%M:%S')") -REPORT+=("$(printf '%.0s━' {1..50})") - -# ============================================================================================== -# ━━━ Output and Send ━━━ -# ============================================================================================== -HEADER="☕ SUNDAY MORNING COFFEE REPORT — $REPORT_DATE" -DIVIDER="$(printf '%.0s━' {1..50})" -BODY=$(printf '%s\n' "$HEADER" "${REPORT[@]}") - -echo "" -echo "$DIVIDER" -echo "$HEADER" -echo "$DIVIDER" -for report_line in "${REPORT[@]}"; do - echo "$report_line" -done echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "☕ Sunday Morning Coffee Report — $MY_ID" +[[ "$DRY_RUN" == true ]] && echo " [DRY RUN]" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -if [[ "$DRY_RUN" == true ]]; then - warn "DRY RUN — notification not sent" -else - if [[ "${NOTIFY_UNRAID:-false}" == true ]]; then - NOTIFY_SCRIPT="/usr/local/emhttp/plugins/dynamix/scripts/notify" - if [[ -x "$NOTIFY_SCRIPT" ]]; then - "$NOTIFY_SCRIPT" -s "☕ Weekly Report — $MY_ID" -d "$BODY" -i "normal" 2>/dev/null - echo "unRAID notification sent" - fi - fi - if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then - # Escape body for JSON - ESCAPED_BODY=$(echo "$BODY" | python3 -c \ - 'import sys,json; print(json.dumps(sys.stdin.read()))' 2>/dev/null || \ - echo "\"$BODY\"") - PAYLOAD="{\"content\": ${ESCAPED_BODY}}" - curl -sf -H "Content-Type: application/json" \ - -d "$PAYLOAD" "$DISCORD_WEBHOOK" >/dev/null 2>&1 && \ - echo "Discord notification sent" || \ - warn "Discord notification failed" - fi - echo "Report generated — $MY_ID — ${#ISSUES[@]} issue(s) ${#FINDINGS[@]} finding(s)" -fi \ No newline at end of file +if [[ ${#COFFEE_REPORT_SCRIPTS[@]} -eq 0 ]]; then + warn "No scripts configured — add entries to COFFEE_REPORT_SCRIPTS in master.conf" + exit 0 +fi + +for script_entry in "${COFFEE_REPORT_SCRIPTS[@]}"; do + [[ -z "$script_entry" ]] && continue + echo "" + run_job "$script_entry" +done + +# ============================================================================================== +# ━━━ Summary ━━━ +# ============================================================================================== +echo "" +echo "━━━━━ ☕ COFFEE REPORT SUMMARY ━━━━━" +echo "🖥️ Host: $MY_ID" +echo "⏱️ Done: $(date '+%Y-%m-%d %H:%M:%S')" + +if [[ ${#JOB_PASS[@]} -gt 0 ]]; then + echo "✅ Passed: ${JOB_PASS[*]}" +fi +if [[ ${#JOB_FAIL[@]} -gt 0 ]]; then + echo "❌ Failed: ${JOB_FAIL[*]}" +fi + +[[ ${#JOB_FAIL[@]} -gt 0 ]] && exit 1 +exit 0 diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/dryrun.php b/Plugin/usr/local/emhttp/plugins/varaverk/api/dryrun.php index 5e3965d..911637f 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/api/dryrun.php +++ b/Plugin/usr/local/emhttp/plugins/varaverk/api/dryrun.php @@ -21,6 +21,7 @@ if (!is_dir($logDir)) mkdir($logDir, 0755, true); file_put_contents($logFile, date('[Y-m-d H:i:s]') . " [DRY RUN] started\n", FILE_APPEND); -exec('nohup env DRY_RUN=1 bash ' . escapeshellarg($script) . ' >> ' . escapeshellarg($logFile) . ' 2>&1 > ' . escapeshellarg($logFile) . ' 2>&1 true]); diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/run.php b/Plugin/usr/local/emhttp/plugins/varaverk/api/run.php index bcf237f..4e89579 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/api/run.php +++ b/Plugin/usr/local/emhttp/plugins/varaverk/api/run.php @@ -22,6 +22,7 @@ if (!is_dir($logDir)) mkdir($logDir, 0755, true); // Stamp the log so the panel shows when the run started file_put_contents($logFile, date('[Y-m-d H:i:s]') . " Manual run started\n", FILE_APPEND); -exec('nohup bash ' . escapeshellarg($script) . ' >> ' . escapeshellarg($logFile) . ' 2>&1 > ' . escapeshellarg($logFile) . ' 2>&1 true]); diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/api/scheduler.php b/Plugin/usr/local/emhttp/plugins/varaverk/api/scheduler.php index 30c716e..eb086b6 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/api/scheduler.php +++ b/Plugin/usr/local/emhttp/plugins/varaverk/api/scheduler.php @@ -2,9 +2,10 @@ header('Content-Type: application/json'); require_once dirname(__DIR__) . '/include/scheduler.php'; -$id = trim($_POST['id'] ?? ''); -$enabled = (bool)($_POST['enabled'] ?? false); -$cron = trim($_POST['cron'] ?? ''); +$id = trim($_POST['id'] ?? ''); +$enabled = (bool)($_POST['enabled'] ?? false); +$cron = trim($_POST['cron'] ?? ''); +$log_enabled = ($_POST['log_enabled'] ?? '0') === '1'; if (!$id) { echo json_encode(['ok' => false, 'error' => 'Missing id']); @@ -17,5 +18,5 @@ if ($cron && !preg_match('/^(\S+\s+){4}\S+$/', $cron)) { exit; } -$ok = vv_schedule_update($id, $enabled, $cron); +$ok = vv_schedule_update($id, $enabled, $cron, $log_enabled); echo json_encode(['ok' => $ok, 'error' => $ok ? null : 'Failed to write schedule']); diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/css/varaverk.css b/Plugin/usr/local/emhttp/plugins/varaverk/css/varaverk.css index ccad283..6c915fd 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/css/varaverk.css +++ b/Plugin/usr/local/emhttp/plugins/varaverk/css/varaverk.css @@ -38,7 +38,6 @@ .vv-script { background: #161616; border: 1px solid #333; border-radius: 4px; padding: 6px 10px; margin: 4px 0; margin-left: 20px; } .vv-job-row { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; } -.vv-name-cron { display: flex; align-items: center; gap: 8px; flex: 1; min-width: 0; } .vv-job-label { flex: 1; font-size: 16px; font-weight: bold; color: #6fcf97; min-width: 80px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } .vv-job-actions { display: flex; align-items: center; gap: 8px; padding-left: 44px; margin-top: 6px; } @@ -48,11 +47,14 @@ cursor: default; } .vv-cron { flex: 0 0 110px; width: 110px; background: #111; border: 1px solid #444; color: #ddd; padding: 4px 6px; border-radius: 4px; font-family: monospace; font-size: 13px; } +.vv-log-label { display: flex; align-items: center; gap: 4px; font-size: 12px; color: #888; + cursor: pointer; white-space: nowrap; flex-shrink: 0; } +.vv-log-label input { cursor: pointer; accent-color: #4caf50; } +.vv-log-label:has(input:checked) { color: #4caf50; } .vv-children { padding-top: 8px; border-top: 1px solid #333; margin-top: 8px; } .vv-advanced-toggle { background: none; border: 1px solid #555; color: #aaa; padding: 2px 8px; border-radius: 4px; cursor: pointer; font-size: 12px; } .vv-advanced-toggle:hover { border-color: #888; color: #fff; } -.vv-job-status { font-size: 12px; min-width: 20px; } .vv-btn-sm { padding: 3px 10px; background: #2a2a2a; border: 1px solid #555; color: #ccc; border-radius: 4px; cursor: pointer; font-size: 12px; white-space: nowrap; } .vv-btn-sm:hover { border-color: #888; color: #fff; } @@ -70,23 +72,26 @@ /* Two-panel layout */ #vv-sched-layout { display: flex; gap: 16px; align-items: stretch; } -#vv-sched-left { flex: 1 1 0; min-width: 0; } +#vv-sched-left { flex: 1 1 0; min-width: 0; display: flex; flex-direction: column; } +#vv-sched-cards { flex: 1; } #vv-sched-right { display: none; flex: 1 1 0; min-width: 0; flex-direction: column; } #vv-sched-right.vv-panel-visible { display: flex; } -#vv-sched-right > .vv-card { flex: 1; display: flex; flex-direction: column; min-height: 0; } -.vv-log-right-pre { flex: 1; min-height: 0; max-height: none; } +.vv-log-card { flex: 1; display: flex; flex-direction: column; padding-bottom: 0; } +.vv-log-right-pre { max-height: none; overflow-y: auto; } /* Mobile: stack vertically, right panel full-width below cards */ @media (max-width: 880px) { #vv-sched-layout { flex-direction: column; align-items: stretch; } #vv-sched-left { flex: none; width: 100%; } #vv-sched-right { flex-direction: column; width: 100%; } - .vv-log-right-pre { min-height: 260px; max-height: 340px; flex: none; } + .vv-log-right-pre { min-height: 260px; max-height: 340px; } } -/* Card footer */ -.vv-card-footer { display: flex; align-items: center; gap: 10px; - margin-top: 12px; padding-top: 10px; border-top: 1px solid #333; } +/* Shared footer (Save Schedule left, info right) — same min-height so log card ends level with script cards */ +.vv-sched-footer { display: flex; align-items: center; gap: 10px; + margin-top: 12px; padding: 10px 0; border-top: 1px solid #333; + flex-shrink: 0; min-height: 72px; box-sizing: border-box; } +.vv-sched-info { color: #666; font-size: 14px; display: flex; flex-direction: column; gap: 9px; } .vv-save-btn { padding: 5px 18px; background: #4caf50; border: none; color: #fff; border-radius: 4px; cursor: pointer; font-size: 13px; } .vv-save-btn:hover { background: #388e3c; } diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/include/scheduler.php b/Plugin/usr/local/emhttp/plugins/varaverk/include/scheduler.php index 68de6e9..1d97ce6 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/include/scheduler.php +++ b/Plugin/usr/local/emhttp/plugins/varaverk/include/scheduler.php @@ -20,18 +20,24 @@ function vv_schedule_save(array $schedule): bool { return file_put_contents(SCHEDULE_FILE, json_encode($schedule, JSON_PRETTY_PRINT)) !== false; } -function vv_schedule_update(string $id, bool $enabled, string $cron): bool { +function vv_schedule_update(string $id, bool $enabled, string $cron, bool $log_enabled = false): bool { $schedule = vv_schedule_load(); $schedule[$id] = [ - 'id' => $id, - 'enabled' => $enabled, - 'cron' => $cron, - 'updated' => date('c'), + 'id' => $id, + 'enabled' => $enabled, + 'cron' => $cron, + 'log_enabled' => $log_enabled, + 'updated' => date('c'), ]; if (!vv_schedule_save($schedule)) return false; return vv_cron_rebuild($schedule); } +function vv_job_flags(string $id): string { + $schedule = vv_schedule_load(); + return !empty($schedule[$id]['log_enabled']) ? '--log' : ''; +} + define('LOG_DIR', '/var/log/varaverk'); function vv_job_log_path(string $id): string { @@ -52,7 +58,8 @@ function vv_cron_rebuild(array $schedule): bool { $logFile = vv_job_log_path($entry['id']); $logDir = dirname($logFile); if (!is_dir($logDir)) mkdir($logDir, 0755, true); - $lines[] = "{$entry['cron']} " . CRON_USER . " bash \"$script\" >> \"$logFile\" 2>&1"; + $flags = !empty($entry['log_enabled']) ? ' --log' : ''; + $lines[] = "{$entry['cron']} " . CRON_USER . " bash \"$script\"$flags >> \"$logFile\" 2>&1"; } $lines[] = ""; @@ -103,13 +110,14 @@ function vv_job_tree(): array { $id = 'Orchestrators/' . basename($path); $entry = $schedule[$id] ?? ['enabled' => false, 'cron' => '']; $orchs[] = [ - 'id' => $id, - 'label' => basename($path, '.sh'), - 'desc' => vv_script_description($path), - 'type' => 'orchestrator', - 'enabled' => (bool)($entry['enabled'] ?? false), - 'cron' => $entry['cron'] ?? '', - 'children' => vv_script_children($path, $schedule), + 'id' => $id, + 'label' => basename($path, '.sh'), + 'desc' => vv_script_description($path), + 'type' => 'orchestrator', + 'enabled' => (bool)($entry['enabled'] ?? false), + 'cron' => $entry['cron'] ?? '', + 'log_enabled' => (bool)($entry['log_enabled'] ?? false), + 'children' => vv_script_children($path, $schedule), ]; } return $orchs; @@ -148,12 +156,13 @@ function vv_script_children(string $orchPath, array $schedule): array { $seen[$rel] = true; $entry = $schedule[$rel] ?? ['enabled' => false, 'cron' => '']; $children[] = [ - 'id' => $rel, - 'label' => basename($rel, '.sh'), - 'desc' => vv_script_description("$scriptsDir/$rel"), - 'type' => 'script', - 'enabled' => (bool)($entry['enabled'] ?? false), - 'cron' => $entry['cron'] ?? '', + 'id' => $rel, + 'label' => basename($rel, '.sh'), + 'desc' => vv_script_description("$scriptsDir/$rel"), + 'type' => 'script', + 'enabled' => (bool)($entry['enabled'] ?? false), + 'cron' => $entry['cron'] ?? '', + 'log_enabled' => (bool)($entry['log_enabled'] ?? false), ]; }; diff --git a/Plugin/usr/local/emhttp/plugins/varaverk/pages/scheduler.php b/Plugin/usr/local/emhttp/plugins/varaverk/pages/scheduler.php index 0073428..76d887d 100644 --- a/Plugin/usr/local/emhttp/plugins/varaverk/pages/scheduler.php +++ b/Plugin/usr/local/emhttp/plugins/varaverk/pages/scheduler.php @@ -12,6 +12,7 @@ $tree = vv_job_tree();
+
@@ -23,15 +24,9 @@ $tree = vv_job_tree(); onchange="vvSaveJob(this)"> -
- - -
- - - - + +
@@ -40,7 +35,16 @@ $tree = vv_job_tree(); + + + +
@@ -54,12 +58,9 @@ $tree = vv_job_tree(); onchange="vvSaveJob(this)"> -
- - -
- + +
@@ -68,6 +69,12 @@ $tree = vv_job_tree(); + @@ -78,8 +85,10 @@ $tree = vv_job_tree(); + + -
+
@@ -88,7 +97,7 @@ $tree = vv_job_tree();
-
+
No script selected @@ -101,6 +110,29 @@ $tree = vv_job_tree(); 
Click Log, Run, or Dry Run on a script to view output here.
+ /dev/null', $psLines); + foreach ($psLines as $line) { + if (preg_match('#' . preg_quote(SCRIPTS_DIR, '#') . '/([^\s]+\.sh)#', $line, $rm)) { + $runningScripts[] = basename($rm[1], '.sh'); + } + } + $runningScripts = array_unique($runningScripts); + ?> +
+ of job scheduled + +
@@ -122,6 +154,40 @@ function vvPost(url, data) { }).then(r => r.json()); } +function vvFitRight() { + const right = document.getElementById('vv-sched-right'); + if (!right.classList.contains('vv-panel-visible')) return; + const left = document.getElementById('vv-sched-left'); + + const lf = left.querySelector('.vv-sched-footer'); + const rf = right.querySelector('.vv-sched-info'); + const toolbar = right.querySelector('.vv-log-toolbar'); + const pre = document.getElementById('vv-log-pre'); + + // Read viewport rects BEFORE any mutations + const leftRect = left.getBoundingClientRect(); + const rightRect = right.getBoundingClientRect(); + + // Match footer heights + rf.style.height = lf.offsetHeight + 'px'; + + // Pin right panel bottom to left panel bottom using viewport coords + right.style.height = Math.round(leftRect.bottom - rightRect.top) + 'px'; + + // Pre fills from toolbar bottom to footer border, leaving 12px card padding below + // -12 footer margin, -12 card padding-bottom, -8 toolbar margin-bottom + const preH = Math.max(80, lf.getBoundingClientRect().top - 32 - toolbar.getBoundingClientRect().bottom); + pre.style.maxHeight = 'none'; + pre.style.height = preH + 'px'; + + console.log('[vvFitRight]', { + leftBottom: Math.round(leftRect.bottom), rightTop: Math.round(rightRect.top), + rightBottomAfter: Math.round(right.getBoundingClientRect().bottom), + rfH: lf.offsetHeight, preH, preActual: pre.offsetHeight, + }); +} +window.addEventListener('resize', vvFitRight); + // Open the right panel for a given job id function vvOpenRight(id) { // Deselect old @@ -137,6 +203,7 @@ function vvOpenRight(id) { const name = id.replace(/\.sh$/, '').split('/').pop(); document.getElementById('vv-log-title').textContent = name; document.getElementById('vv-sched-right').classList.add('vv-panel-visible'); + requestAnimationFrame(vvFitRight); vvFetchRight(); if (!vvPollTimer) vvPollTimer = setInterval(vvFetchRight, 2000); @@ -220,13 +287,12 @@ function vvDryRun(btn) { } function vvSaveJob(el) { - const job = el.closest('[data-id]'); - const id = job.dataset.id; - const enabled = job.querySelector('.vv-enabled').checked ? '1' : '0'; - const cron = job.querySelector('.vv-cron').value.trim(); - const status = job.querySelector('.vv-job-status'); - vvPost('/plugins/varaverk/api/scheduler.php', {id, enabled, cron}) - .then(d => { if (status) vvFlashStatus(status, d.ok ? '✓' : '✗ ' + (d.error ?? ''), d.ok); }); + const job = el.closest('[data-id]'); + const id = job.dataset.id; + const enabled = job.querySelector('.vv-enabled').checked ? '1' : '0'; + const cron = job.querySelector('.vv-cron').value.trim(); + const log_enabled = job.querySelector('.vv-log-enabled')?.checked ? '1' : '0'; + vvPost('/plugins/varaverk/api/scheduler.php', {id, enabled, cron, log_enabled}); } function vvSaveAll() { @@ -236,10 +302,11 @@ function vvSaveAll() { if (!pending) { vvFlashStatus(status, '✗ No jobs found', false); return; } status.textContent = 'Saving…'; jobs.forEach(job => { - const id = job.dataset.id; - const enabled = job.querySelector('.vv-enabled').checked ? '1' : '0'; - const cron = job.querySelector('.vv-cron').value.trim(); - vvPost('/plugins/varaverk/api/scheduler.php', {id, enabled, cron}) + const id = job.dataset.id; + const enabled = job.querySelector('.vv-enabled').checked ? '1' : '0'; + const cron = job.querySelector('.vv-cron').value.trim(); + const log_enabled = job.querySelector('.vv-log-enabled')?.checked ? '1' : '0'; + vvPost('/plugins/varaverk/api/scheduler.php', {id, enabled, cron, log_enabled}) .then(d => { if (!d.ok) allOk = false; if (--pending === 0) vvFlashStatus(status, allOk ? '✓ Saved' : '✗ Some failed', allOk);