FailedProxy/Varaverk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added /mnt/user/appdata/unraid_scripts/Monitors/continuous_scripts_status.sh

Browse Source
This commit is contained in:
FailedProxy
2026-04-26 02:30:12 -04:00
parent d3906a8efa
commit b3d53b792b
4 changed files with 736 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Orchestrators/sunday_morning_coffee_report.sh
+210 -62
View File
@@ -11,11 +11,11 @@
# Sections:
# 🖥️ System — uptime, memory, boot drive, cache drive, reboots
# 📀 Array — disk count, parity status, ZFS health, drive temps
# 🔀 Failover — state, last change, Tailscale connectivity
# 🎬 Transcodes — ramdisk usage, weekly peak, flips, session split
# 🎵 Media Activity — arr cleanup stats, arr recovery stats, library health
# 🌐 Rsync — weekly transfer totals, per-share breakdown
# 🛡️ Watchdog — container strikes, restarts, system strikes, skip list
# 🛡️ Watchdog — system watchdog, docker watchdog, failover (all continuous loops)
# strikes, skip list, restarts, system snapshot, container overview
# 🔐 Security — SSL cert expiry per domain
# 📊 Emby — weekly stream count, top users, top content
# ⚙️ Health — SMART summary, docker container count, Gitea sync status
@@ -225,36 +225,6 @@ if command -v smartctl >/dev/null 2>&1; then
[[ -n "$TEMP_SUMMARY" ]] && line "Temps: $TEMP_SUMMARY"
fi
# -----------------------------------------------------------------------------------------------
# ━━━ 🔀 Failover ━━━
# -----------------------------------------------------------------------------------------------
section "🔀 FAILOVER"
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
FAILOVER_STATE=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
FAILOVER_CHANGE=$(grep "^last_change=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
if [[ "$FAILOVER_STATE" == "NORMAL" ]]; then
line "State: NORMAL ✅"
else
issue "Failover state: $FAILOVER_STATE — not normal"
fi
line "Last change: ${FAILOVER_CHANGE:-unknown}"
else
issue "Failover state file not found"
fi
# Tailscale connectivity
if command -v tailscale >/dev/null 2>&1; then
TS_STATUS=$(tailscale status 2>/dev/null)
REMOTE_VISIBLE=$(echo "$TS_STATUS" | grep -c "$HOST2" 2>/dev/null || echo 0)
if [[ "$REMOTE_VISIBLE" -gt 0 ]]; then
REMOTE_IP=$(tailscale ip -4 "$HOST2" 2>/dev/null || echo "unknown")
line "Tailscale: $HOST2 visible at $REMOTE_IP ✅"
else
issue "Tailscale: $HOST2 not visible — check connectivity"
fi
fi
# -----------------------------------------------------------------------------------------------
# ━━━ 🎬 Transcodes ━━━
# -----------------------------------------------------------------------------------------------
@@ -394,35 +364,143 @@ fi
# -----------------------------------------------------------------------------------------------
section "🛡️ WATCHDOG"
# Container watchdog strikes
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
ACTIVE_STRIKES=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$" | wc -l)
if [[ "$ACTIVE_STRIKES" -gt 0 ]]; then
STRIKE_LIST=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
issue "Container watchdog: $ACTIVE_STRIKES active strikes — $STRIKE_LIST"
else
line "Container watchdog: no active strikes ✅"
get_lock_pid_cr() {
local script_name="$1"
local lockfile="$LOCK_DIR/${script_name}.lock"
if [[ -f "$lockfile" ]]; then
local content
content=$(cat "$lockfile" 2>/dev/null)
echo "${content%%:*}"
fi
}
get_lock_name_cr() {
local script_name="$1"
local lockfile="$LOCK_DIR/${script_name}.lock"
if [[ -f "$lockfile" ]]; then
local content
content=$(cat "$lockfile" 2>/dev/null)
echo "${content##*:}"
fi
}
is_loop_running() {
local script_name="$1"
local pid name
pid=$(get_lock_pid_cr "$script_name")
name=$(get_lock_name_cr "$script_name")
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null && [[ "$name" == "$script_name" ]]
}
get_lock_age_cr() {
local script_name="$1"
local lockfile="$LOCK_DIR/${script_name}.lock"
if [[ -f "$lockfile" ]]; then
local mtime
mtime=$(stat -c %Y "$lockfile" 2>/dev/null || echo 0)
echo $(( $(date +%s) - mtime ))
else
echo 0
fi
}
format_uptime_cr() {
local seconds=$1
local days=$(( seconds / 86400 ))
local hours=$(( (seconds % 86400) / 3600 ))
local mins=$(( (seconds % 3600) / 60 ))
if (( days > 0 )); then echo "${days}d ${hours}h ${mins}m"
elif (( hours > 0 )); then echo "${hours}h ${mins}m"
else echo "${mins}m"; fi
}
# ── System Watchdog ──
line "⚙️ System Watchdog"
SYS_PID_CR=$(get_lock_pid_cr "system_watchdog")
if is_loop_running "system_watchdog"; then
SYS_AGE_CR=$(get_lock_age_cr "system_watchdog")
SYS_UP_CR=$(format_uptime_cr "$SYS_AGE_CR")
SYS_CYCLE_CR=$(( SYS_AGE_CR / SYSTEM_WATCHDOG_INTERVAL ))
line " ✅ Running │ PID: $SYS_PID_CR │ Uptime: $SYS_UP_CR │ ~Cycle: $SYS_CYCLE_CR"
else
issue "system_watchdog NOT RUNNING"
fi
# System watchdog strikes
# System strikes
if [[ -f "$SYS_WATCHDOG_STATE_FILE" ]]; then
SYS_STRIKES=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$" | wc -l)
if [[ "$SYS_STRIKES" -gt 0 ]]; then
SYS_LIST=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | tr '\n' ' ')
issue "System watchdog: $SYS_STRIKES active strikes — $SYS_LIST"
SYS_ACTIVE=$(grep -v ":0$" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
if [[ -n "$SYS_ACTIVE" ]]; then
while IFS=: read -r key count; do
[[ -z "$key" ]] && continue
issue " ⚠️ Strike: $key — $count/$SYS_WATCHDOG_STRIKE_LIMIT"
done <<< "$SYS_ACTIVE"
else
line "System watchdog: no active strikes ✅"
line " ✅ Strikes: none"
fi
fi
# Container skip list
# Reboots this week
if [[ -f "$SYS_WATCHDOG_REBOOT_LOG" ]]; then
WEEK_EPOCH=$(date -d "$WEEK_START" +%s)
REBOOT_COUNT=$(awk -v cutoff="$WEEK_EPOCH" '$1 >= cutoff' \
"$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | wc -l)
if [[ "${REBOOT_COUNT:-0}" -gt 0 ]]; then
issue " 🔄 Reboots this week: $REBOOT_COUNT (watchdog triggered)"
else
line " ✅ Reboots this week: 0"
fi
fi
# Skip list
if [[ -f "$SYS_WATCHDOG_FAILED_FILE" ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
SKIP_COUNT=$(wc -l < "$SYS_WATCHDOG_FAILED_FILE")
SKIP_LIST=$(cat "$SYS_WATCHDOG_FAILED_FILE" | tr '\n' ' ')
issue "Skip list: $SKIP_COUNT containers — $SKIP_LIST — manual intervention needed"
SKIP_LIST=$(tr '\n' ' ' < "$SYS_WATCHDOG_FAILED_FILE")
issue " ⛔ Skip list ($SKIP_COUNT): $SKIP_LIST"
else
line "Skip list: empty ✅"
line " ✅ Skip list: empty"
fi
# System snapshot
ROOTFS_PCT_CR=$(df / --output=pcent 2>/dev/null | tail -1 | tr -d ' %')
MEM_AVAIL_CR=$(awk '/MemAvailable/ {printf "%.1f", $2/1048576}' /proc/meminfo)
MEM_TOTAL_CR=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
LOAD_CR=$(awk '{print $1}' /proc/loadavg)
ZOMBIE_CR=$(ps aux | awk '{print $8}' | grep -c "^Z$" 2>/dev/null || echo 0)
ZOMBIE_CR="${ZOMBIE_CR//[^0-9]/}"; ZOMBIE_CR="${ZOMBIE_CR:-0}"
ARC_GB_CR="n/a"
if [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
ARC_B=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
ARC_GB_CR=$(awk "BEGIN {printf \"%.1f\", $ARC_B / 1073741824}")
fi
line " 📊 rootfs: ${ROOTFS_PCT_CR}% │ RAM: ${MEM_AVAIL_CR}GB free/${MEM_TOTAL_CR}GB │ ARC: ${ARC_GB_CR}GB │ load: ${LOAD_CR} │ zombies: ${ZOMBIE_CR}"
REPORT+=("")
# ── Docker Watchdog ──
line "🐳 Docker Watchdog"
DOCKER_PID_CR=$(get_lock_pid_cr "docker_watchdog")
if is_loop_running "docker_watchdog"; then
DOCKER_AGE_CR=$(get_lock_age_cr "docker_watchdog")
DOCKER_UP_CR=$(format_uptime_cr "$DOCKER_AGE_CR")
DOCKER_CYCLE_CR=$(( DOCKER_AGE_CR / DOCKER_WATCHDOG_INTERVAL ))
line " ✅ Running │ PID: $DOCKER_PID_CR │ Uptime: $DOCKER_UP_CR │ ~Cycle: $DOCKER_CYCLE_CR"
else
issue "docker_watchdog NOT RUNNING"
fi
# Container strikes
if [[ -f "$WATCHDOG_STATE_FILE" ]]; then
DOCK_ACTIVE=$(grep -v ":0$" "$WATCHDOG_STATE_FILE" 2>/dev/null | grep -v "^$")
if [[ -n "$DOCK_ACTIVE" ]]; then
while IFS=: read -r key count; do
[[ -z "$key" ]] && continue
issue " ⚠️ Strike: $key — $count"
done <<< "$DOCK_ACTIVE"
else
line " ✅ Container strikes: none"
fi
fi
# Container restarts this week
@@ -433,24 +511,94 @@ if [[ -f "$WATCHDOG_CONTAINER_RESTART_LOG" ]]; then
RESTARTED=$(awk -F'|' -v cutoff="$WEEK_START" \
'$2 >= cutoff {print $1}' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | \
sort | uniq -c | sort -rn | head -5 | \
awk '{print $2 "(" $1 ")"}' | tr '\n' ' ')
finding "Watchdog restarted $WEEK_RESTARTS containers this week: $RESTARTED"
awk '{print $2"("$1")"}' | tr '\n' ' ')
finding " 🔄 Restarts this week: $WEEK_RESTARTS — $RESTARTED"
else
line "Watchdog restarts: none this week ✅"
line " ✅ Container restarts this week: none"
fi
fi
# Docker container count
# Docker overview
if command -v docker >/dev/null 2>&1; then
RUNNING=$(docker ps -q 2>/dev/null | wc -l)
TOTAL=$(docker ps -aq 2>/dev/null | wc -l)
STOPPED=$(( TOTAL - RUNNING ))
if [[ "$STOPPED" -gt 0 ]]; then
STOPPED_NAMES=$(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null | \
head -5 | tr '\n' ' ')
finding "Docker: $RUNNING/$TOTAL running — $STOPPED stopped: $STOPPED_NAMES"
RUNNING_CR=$(docker ps -q 2>/dev/null | wc -l)
TOTAL_CR=$(docker ps -aq 2>/dev/null | wc -l)
UNHEALTHY_CR=$(docker ps --filter health=unhealthy -q 2>/dev/null | wc -l)
STOPPED_CR=$(docker ps -af "status=exited" --format "{{.Names}}" 2>/dev/null | head -5 | tr '\n' ' ')
STOPPED_COUNT_CR=$(docker ps -af "status=exited" -q 2>/dev/null | wc -l)
line " 📦 $RUNNING_CR/$TOTAL_CR running │ unhealthy: $UNHEALTHY_CR"
if [[ "${STOPPED_COUNT_CR:-0}" -gt 0 ]]; then
issue " ⚠️ Stopped: $STOPPED_CR"
else
line "Docker: $RUNNING/$TOTAL containers running ✅"
line " ✅ All containers running"
fi
fi
REPORT+=("")
# ── Failover ──
line "🔀 Failover"
FAILOVER_PID_CR=$(get_lock_pid_cr "failover")
if is_loop_running "failover"; then
FAILOVER_AGE_CR=$(get_lock_age_cr "failover")
FAILOVER_UP_CR=$(format_uptime_cr "$FAILOVER_AGE_CR")
line " ✅ Running │ PID: $FAILOVER_PID_CR │ Uptime: $FAILOVER_UP_CR"
else
issue "failover NOT RUNNING"
fi
FAILOVER_STATE_CR="UNKNOWN"
FAILOVER_STATE_SECS_CR=0
if [[ -f "$FAILOVER_STATE_FILE" ]]; then
FAILOVER_STATE_CR=$(grep "^state=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
FAILOVER_LAST_EPOCH_CR=$(grep "^last_change_epoch=" "$FAILOVER_STATE_FILE" 2>/dev/null | cut -d= -f2)
[[ -n "$FAILOVER_LAST_EPOCH_CR" ]] && \
FAILOVER_STATE_SECS_CR=$(( $(date +%s) - FAILOVER_LAST_EPOCH_CR ))
fi
STATE_DUR_CR=$(format_uptime_cr "${FAILOVER_STATE_SECS_CR:-0}")
case "$FAILOVER_STATE_CR" in
NORMAL)
line " ✅ State: NORMAL │ Duration: $STATE_DUR_CR"
;;
FAILOVER)
issue " ⚠️ State: FAILOVER — remote down for $STATE_DUR_CR"
FAILOVER_MINS_CR=$(( FAILOVER_STATE_SECS_CR / 60 ))
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
T2=$HOST2_TIER2_DELAY; T3=$HOST2_TIER3_DELAY; T4=$HOST2_TIER4_DELAY
else
T2=$HOST1_TIER2_DELAY; T3=$HOST1_TIER3_DELAY; T4=$HOST1_TIER4_DELAY
fi
(( FAILOVER_MINS_CR >= T2 )) && line " 🔄 Tier 2: ✅ active" || \
line " 🔄 Tier 2: ⏳ in $(( T2 - FAILOVER_MINS_CR ))min"
(( FAILOVER_MINS_CR >= T3 )) && line " 🔄 Tier 3: ✅ active" || \
line " 🔄 Tier 3: ⏳ in $(( T3 - FAILOVER_MINS_CR ))min"
(( FAILOVER_MINS_CR >= T4 )) && line " 🔄 Tier 4: ✅ active" || \
line " 🔄 Tier 4: ⏳ in $(( T4 - FAILOVER_MINS_CR ))min"
;;
NO_INTERNET)
issue " ❌ State: NO_INTERNET — DDNS stopped │ Duration: $STATE_DUR_CR"
;;
DARK)
issue " ❌ State: DARK — remote down AND no internet │ Duration: $STATE_DUR_CR"
;;
*)
finding " ❓ State: ${FAILOVER_STATE_CR:-unknown}"
;;
esac
# Tailscale
if command -v tailscale >/dev/null 2>&1; then
REMOTE_IP_CR=$(tailscale ip -4 "$HOST2" 2>/dev/null)
if [[ -n "$REMOTE_IP_CR" ]]; then
if ping -c 1 -W 2 "$REMOTE_IP_CR" >/dev/null 2>&1; then
line " 🌐 $HOST2: $REMOTE_IP_CR — reachable ✅"
else
issue " 🌐 $HOST2: $REMOTE_IP_CR — not responding"
fi
else
issue " 🌐 $HOST2 not visible on Tailscale"
fi
fi