Added system watchdog. and way to many other changes
This commit is contained in:
@@ -22,7 +22,6 @@ parse_args "$@"
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
# ROOT CHECK
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
@@ -30,10 +29,9 @@ fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
# Verify Docker is available
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found — check PATH or Docker installation"
|
||||
notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "alert"
|
||||
notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -60,9 +58,6 @@ fi
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Attempts a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
|
||||
# Returns 0 on success, 1 if all attempts fail.
|
||||
# Usage: retry_docker docker restart Emby
|
||||
retry_docker() {
|
||||
local attempt=1
|
||||
|
||||
@@ -84,7 +79,7 @@ retry_docker() {
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CONTAINERS $ICON_STOP $ICON_START Daily Restart ━━━
|
||||
# ━━━ $ICON_CONTAINERS Daily Restart ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Daily Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
@@ -100,7 +95,6 @@ STARTED=()
|
||||
for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
# Verify container exists
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
error "$container does not exist — skipping"
|
||||
FAILED+=("$container")
|
||||
@@ -121,7 +115,8 @@ for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
echo "$ICON_STARTED $container restarted"
|
||||
RESTARTED+=("$container")
|
||||
else
|
||||
error "Failed to restart $container"
|
||||
error "Failed to restart $container after $RETRY_COUNT attempts"
|
||||
notify "$container failed to restart on $(hostname)" "Docker Daily Restart" "warning"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
fi
|
||||
@@ -136,7 +131,8 @@ for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
echo "$ICON_STARTED $container started"
|
||||
STARTED+=("$container")
|
||||
else
|
||||
error "Failed to start $container"
|
||||
error "Failed to start $container after $RETRY_COUNT attempts"
|
||||
notify "$container failed to start on $(hostname)" "Docker Daily Restart" "warning"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
fi
|
||||
@@ -157,27 +153,19 @@ END=$(date +%s)
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY RESTART SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
|
||||
if [[ ${#RESTARTED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||
fi
|
||||
if [[ ${#STARTED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_STARTED Started: ${STARTED[*]}"
|
||||
fi
|
||||
if [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
fi
|
||||
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||
[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE"
|
||||
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started" "Docker Daily Restart" "normal"
|
||||
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Daily Restart" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed"
|
||||
notify "Daily restart completed with errors — failed: ${FAILED[*]}" "Docker Daily Restart" "alert"
|
||||
notify "Daily restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Daily Restart" "warning"
|
||||
fi
|
||||
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
|
||||
@@ -2,24 +2,28 @@
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Docker Watchdog --------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Self-healing watchdog for Docker containers — monitors memory, CPU and HTTP responsiveness.
|
||||
# Restarts containers that exceed configured thresholds using a strike system for CPU and
|
||||
# responsiveness checks to avoid restarting on brief spikes.
|
||||
# First line of defense — monitors Docker containers for memory, CPU, HTTP responsiveness,
|
||||
# and unexpected stops. Restarts containers that exceed thresholds or go offline.
|
||||
#
|
||||
# Works alongside system_watchdog.sh:
|
||||
# docker_watchdog.sh — container level, minimal disruption, tries to self-heal
|
||||
# system_watchdog.sh — system level, last resort, reboots when healing fails
|
||||
#
|
||||
# Behaviour:
|
||||
# Memory — immediate restart if hard limit is exceeded
|
||||
# CPU — strike system, restarts after CPU_FAIL_LIMIT consecutive over-threshold checks
|
||||
# HTTP — strike system, restarts after RESP_FAIL_LIMIT consecutive failed curl checks
|
||||
# Memory — immediate restart if hard limit exceeded
|
||||
# CPU — strike system, restarts after CPU_FAIL_LIMIT consecutive hits
|
||||
# HTTP — strike system, restarts after RESP_FAIL_LIMIT consecutive failures
|
||||
# Required — strike system, restarts stopped containers, persistent skip list
|
||||
# prevents reboot loops, auto-clears when container recovers
|
||||
# Daemon — immediate notify if Docker daemon is unresponsive
|
||||
#
|
||||
# Strike system:
|
||||
# Strikes persist between runs via WATCHDOG_STATE_FILE (/tmp — resets on reboot)
|
||||
# Strike cadence depends on cron schedule:
|
||||
# Every 15min + 2 strikes = 30min sustained abuse before restart
|
||||
# Every 10min + 2 strikes = 20min sustained abuse before restart
|
||||
# Every 5min + 2 strikes = 10min sustained abuse before restart
|
||||
# Strike cadence depends on cron schedule:
|
||||
# Every 15min + 2 strikes = 30min sustained before restart
|
||||
# Every 10min + 2 strikes = 20min sustained before restart
|
||||
# Every 5min + 2 strikes = 10min sustained before restart
|
||||
#
|
||||
# All configuration lives in Master.conf under the Docker Watchdog section.
|
||||
# Supports --dry-run to show what would be restarted without taking any action.
|
||||
# All configuration in Master.conf under Docker Watchdog section.
|
||||
# Supports --dry-run to show what would happen without acting.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -29,16 +33,18 @@ source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# Auto-detect total CPU cores for normalisation
|
||||
TOTAL_CORES=$(nproc)
|
||||
|
||||
# Persistent skip list — shared with system_watchdog.sh
|
||||
# Containers in this list are skipped until they recover
|
||||
SKIP_LIST_FILE="$SYS_WATCHDOG_FAILED_FILE"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
# ROOT CHECK
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
@@ -47,27 +53,32 @@ fi
|
||||
success "Running as root"
|
||||
info "$ICON_WATCHDOG Watchdog initialising — $TOTAL_CORES cores detected"
|
||||
|
||||
# Ensure state file exists
|
||||
touch "$WATCHDOG_STATE_FILE" 2>/dev/null || {
|
||||
error "Cannot create state file: $WATCHDOG_STATE_FILE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
touch "$SKIP_LIST_FILE" 2>/dev/null || {
|
||||
error "Cannot create skip list: $SKIP_LIST_FILE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_WATCHDOG Containers monitored: ${!WATCHDOG_CONTAINERS[*]}"
|
||||
echo "$ICON_MEM Soft mem threshold: ${SOFT_MEM_THRESHOLD}% of per-container limit"
|
||||
echo "$ICON_ZFS CPU soft threshold: ${SOFT_CPU_THRESHOLD}%"
|
||||
echo "$ICON_ZFS CPU hard threshold: ${HARD_CPU_THRESHOLD}%"
|
||||
echo "$ICON_RETRY CPU fail limit: ${CPU_FAIL_LIMIT} strikes"
|
||||
echo "$ICON_PING Resp fail limit: ${RESP_FAIL_LIMIT} strikes"
|
||||
echo "$ICON_TIME Curl timeout: ${CURL_TIMEOUT}s"
|
||||
echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_WATCHDOG Monitored: ${!WATCHDOG_CONTAINERS[*]}"
|
||||
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}"
|
||||
echo "$ICON_MEM Soft mem: ${SOFT_MEM_THRESHOLD}% of limit"
|
||||
echo "$ICON_ZFS CPU soft: ${SOFT_CPU_THRESHOLD}%"
|
||||
echo "$ICON_ZFS CPU hard: ${HARD_CPU_THRESHOLD}%"
|
||||
echo "$ICON_RETRY CPU strikes: ${CPU_FAIL_LIMIT}"
|
||||
echo "$ICON_RETRY Resp strikes: ${RESP_FAIL_LIMIT}"
|
||||
echo "$ICON_TIME Curl timeout: ${CURL_TIMEOUT}s"
|
||||
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
@@ -76,19 +87,13 @@ fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# STATE HELPERS
|
||||
# Reads and writes per-container strike counts to the state file.
|
||||
# State file format: container:metric:count
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Returns current strike count for a container/metric pair.
|
||||
# Usage: get_strikes "Emby" "CPU"
|
||||
get_strikes() {
|
||||
local container="$1" metric="$2"
|
||||
grep -E "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f3
|
||||
}
|
||||
|
||||
# Sets strike count for a container/metric pair.
|
||||
# Usage: set_strikes "Emby" "CPU" 2
|
||||
set_strikes() {
|
||||
local container="$1" metric="$2" count="$3"
|
||||
grep -vE "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null > "${WATCHDOG_STATE_FILE}.tmp"
|
||||
@@ -96,19 +101,83 @@ set_strikes() {
|
||||
mv "${WATCHDOG_STATE_FILE}.tmp" "$WATCHDOG_STATE_FILE"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# SKIP LIST HELPERS
|
||||
# Container skip list — persistent across reboots via /boot/
|
||||
# Auto-clears entries when container is found running again.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
is_in_skip_list() {
|
||||
local container="$1"
|
||||
grep -qE "^${container}$" "$SKIP_LIST_FILE" 2>/dev/null
|
||||
}
|
||||
|
||||
add_to_skip_list() {
|
||||
local container="$1"
|
||||
if ! is_in_skip_list "$container"; then
|
||||
echo "$container" >> "$SKIP_LIST_FILE"
|
||||
warn "$ICON_WATCHDOG $container added to persistent skip list"
|
||||
notify "$container added to watchdog skip list on $(hostname) — manual check recommended" "Docker Watchdog" "warning"
|
||||
fi
|
||||
}
|
||||
|
||||
remove_from_skip_list() {
|
||||
local container="$1"
|
||||
grep -vE "^${container}$" "$SKIP_LIST_FILE" 2>/dev/null > "${SKIP_LIST_FILE}.tmp"
|
||||
mv "${SKIP_LIST_FILE}.tmp" "$SKIP_LIST_FILE"
|
||||
success "$ICON_WATCHDOG $container recovered — removed from skip list"
|
||||
notify "$container recovered and removed from watchdog skip list on $(hostname)" "Docker Watchdog" "normal"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# SKIP LIST AUTO-HEAL CHECK
|
||||
# On every run check if any skipped containers are now running.
|
||||
# If running remove from skip list — could have recovered after reboot or manual fix.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_skip_list_recovery() {
|
||||
[[ ! -s "$SKIP_LIST_FILE" ]] && return
|
||||
|
||||
info "$ICON_WATCHDOG Checking skip list for recovered containers..."
|
||||
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
remove_from_skip_list "$container"
|
||||
else
|
||||
log "$container still not running — remains on skip list"
|
||||
fi
|
||||
done < "$SKIP_LIST_FILE"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# DOCKER DAEMON HEALTH CHECK
|
||||
# Verifies Docker daemon is responding before attempting any container operations.
|
||||
# A hung daemon means all checks will fail — notify immediately and exit.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_docker_daemon() {
|
||||
info "$ICON_CONTAINERS Checking Docker daemon..."
|
||||
|
||||
if ! timeout 10 docker ps >/dev/null 2>&1; then
|
||||
error "Docker daemon is not responding"
|
||||
notify "Docker daemon unresponsive on $(hostname) — immediate attention required" "Docker Watchdog" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Docker daemon is healthy"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# HELPERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Fetches memory and CPU stats for a container in a single docker stats call.
|
||||
# Returns: MEM_USAGE|CPU_PERCENT
|
||||
parse_stats() {
|
||||
local container="$1"
|
||||
docker stats --no-stream --format "{{.MemUsage}}|{{.CPUPerc}}" "$container"
|
||||
}
|
||||
|
||||
# Converts a memory value and unit to MB.
|
||||
# Supports KiB, MiB, GiB — returns UNKNOWN for unrecognised units.
|
||||
convert_to_mb() {
|
||||
local value="$1" unit="$2"
|
||||
case "$unit" in
|
||||
@@ -119,32 +188,32 @@ convert_to_mb() {
|
||||
esac
|
||||
}
|
||||
|
||||
# Restarts a container locally and sends a notification.
|
||||
# In dry run mode reports what would happen without acting.
|
||||
# Restarts a container and sends notification.
|
||||
# Failed restarts also notify — system_watchdog.sh is the next line of defense.
|
||||
restart_container() {
|
||||
local container="$1" reason="$2"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container ($reason)"
|
||||
return
|
||||
return 0
|
||||
fi
|
||||
|
||||
info "Restarting $container ($reason)..."
|
||||
if docker restart "$container" >/dev/null 2>&1; then
|
||||
echo "$ICON_STARTED $container restarted"
|
||||
notify "$container restarted — $reason" "Docker Watchdog" "warning"
|
||||
log "Restarted $container — reason: $reason"
|
||||
notify "$container restarted on $(hostname) — $reason" "Docker Watchdog" "warning"
|
||||
return 0
|
||||
else
|
||||
error "Failed to restart $container"
|
||||
notify "Failed to restart $container — $reason" "Docker Watchdog" "alert"
|
||||
notify "Failed to restart $container on $(hostname) — $reason" "Docker Watchdog" "warning"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# MEMORY CHECK
|
||||
# Compares current container memory usage against its configured hard limit.
|
||||
# Restarts immediately if at or above 100% of limit.
|
||||
# Warns if at or above SOFT_MEM_THRESHOLD % of limit.
|
||||
# Restarts immediately if container exceeds hard memory limit.
|
||||
# Warns if approaching soft threshold.
|
||||
# Usage: check_memory "Emby" 16384
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_memory() {
|
||||
@@ -174,9 +243,8 @@ check_memory() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CPU CHECK
|
||||
# Normalises CPU usage against total core count and applies the strike system.
|
||||
# Warns at SOFT_CPU_THRESHOLD, strikes at HARD_CPU_THRESHOLD.
|
||||
# Restarts after CPU_FAIL_LIMIT consecutive strikes — resets strikes on restart or recovery.
|
||||
# Strike system — restarts after CPU_FAIL_LIMIT consecutive over-threshold checks.
|
||||
# Resets strikes on recovery or restart.
|
||||
# Usage: check_cpu "Emby"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_cpu() {
|
||||
@@ -193,24 +261,21 @@ check_cpu() {
|
||||
|
||||
if (( cpu_int >= HARD_CPU_THRESHOLD )); then
|
||||
((violations++))
|
||||
error "$ICON_ZFS $container CPU ${cpu_int}% — hard threshold hit ($violations/$CPU_FAIL_LIMIT strikes)"
|
||||
error "$ICON_ZFS $container CPU ${cpu_int}% — hard threshold ($violations/$CPU_FAIL_LIMIT strikes)"
|
||||
set_strikes "$container" "CPU" "$violations"
|
||||
elif (( cpu_int >= SOFT_CPU_THRESHOLD )); then
|
||||
((violations++))
|
||||
warn "$ICON_ZFS $container CPU ${cpu_int}% — soft threshold hit ($violations/$CPU_FAIL_LIMIT strikes)"
|
||||
warn "$ICON_ZFS $container CPU ${cpu_int}% — soft threshold ($violations/$CPU_FAIL_LIMIT strikes)"
|
||||
set_strikes "$container" "CPU" "$violations"
|
||||
else
|
||||
if (( violations > 0 )); then
|
||||
info "$ICON_ZFS $container CPU ${cpu_int}% — recovered, resetting strikes"
|
||||
else
|
||||
success "$ICON_ZFS $container CPU ${cpu_int}%"
|
||||
fi
|
||||
[[ $violations -gt 0 ]] && info "$ICON_ZFS $container CPU ${cpu_int}% — recovered, resetting strikes"
|
||||
[[ $violations -eq 0 ]] && success "$ICON_ZFS $container CPU ${cpu_int}%"
|
||||
set_strikes "$container" "CPU" 0
|
||||
violations=0
|
||||
fi
|
||||
|
||||
if (( violations >= CPU_FAIL_LIMIT )); then
|
||||
error "$ICON_ZFS $container hit CPU limit for $CPU_FAIL_LIMIT consecutive checks"
|
||||
error "$ICON_ZFS $container CPU limit hit for $CPU_FAIL_LIMIT consecutive checks"
|
||||
restart_container "$container" "sustained CPU abuse"
|
||||
set_strikes "$container" "CPU" 0
|
||||
fi
|
||||
@@ -218,9 +283,8 @@ check_cpu() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# RESPONSIVENESS CHECK
|
||||
# Sends an HTTP request to the container's configured URL.
|
||||
# Strike system — restarts after RESP_FAIL_LIMIT consecutive failed HTTP checks.
|
||||
# Skips containers with no URL defined in WATCHDOG_CONTAINER_URLS.
|
||||
# Applies the same strike system as CPU — restarts after RESP_FAIL_LIMIT consecutive failures.
|
||||
# Usage: check_responsiveness "Emby"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_responsiveness() {
|
||||
@@ -238,11 +302,8 @@ check_responsiveness() {
|
||||
warn "$ICON_PING $container unresponsive at $url ($fails/$RESP_FAIL_LIMIT strikes)"
|
||||
set_strikes "$container" "RESP" "$fails"
|
||||
else
|
||||
if (( fails > 0 )); then
|
||||
info "$ICON_PING $container responsive again — resetting strikes"
|
||||
else
|
||||
success "$ICON_PING $container responsive at $url"
|
||||
fi
|
||||
[[ $fails -gt 0 ]] && info "$ICON_PING $container responsive again — resetting strikes"
|
||||
[[ $fails -eq 0 ]] && success "$ICON_PING $container responsive at $url"
|
||||
set_strikes "$container" "RESP" 0
|
||||
fails=0
|
||||
fi
|
||||
@@ -255,54 +316,131 @@ check_responsiveness() {
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_WATCHDOG Watchdog Check ━━━
|
||||
# REQUIRED CONTAINER CHECK
|
||||
# Monitors WATCHDOG_REQUIRED_CONTAINERS for unexpected stops.
|
||||
# Strike system — attempts restart on each strike.
|
||||
# After strike limit hit — adds to persistent skip list and notifies system_watchdog handoff.
|
||||
# Skip list auto-clears at start of each run if container has recovered.
|
||||
# Usage: check_required_containers
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_required_containers() {
|
||||
[[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -eq 0 ]] && return
|
||||
|
||||
info "$ICON_CONTAINERS Checking required containers..."
|
||||
|
||||
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
|
||||
# Skip if on persistent skip list
|
||||
if is_in_skip_list "$container"; then
|
||||
warn "$ICON_NOT_RUNNING $container is on skip list — skipping until recovered"
|
||||
continue
|
||||
fi
|
||||
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
success "$ICON_RUNNING $container is running"
|
||||
set_strikes "$container" "STOP" 0
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$STATUS" == "unknown" ]]; then
|
||||
warn "$container not found on this host — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Container is stopped — apply strike
|
||||
local strikes
|
||||
strikes=$(get_strikes "$container" "STOP")
|
||||
[[ -z "$strikes" ]] && strikes=0
|
||||
((strikes++))
|
||||
|
||||
warn "$ICON_NOT_RUNNING $container is stopped ($strikes/$SYS_WATCHDOG_STRIKE_LIMIT strikes)"
|
||||
set_strikes "$container" "STOP" "$strikes"
|
||||
|
||||
# Attempt restart on each strike
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would attempt restart of $container"
|
||||
else
|
||||
if restart_container "$container" "unexpected stop"; then
|
||||
set_strikes "$container" "STOP" 0
|
||||
else
|
||||
# Restart failed
|
||||
if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then
|
||||
error "$container failed to restart after $SYS_WATCHDOG_STRIKE_LIMIT attempts"
|
||||
add_to_skip_list "$container"
|
||||
set_strikes "$container" "STOP" 0
|
||||
notify "$container handed off to system_watchdog on $(hostname) — added to skip list" "Docker Watchdog" "warning"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_WATCHDOG Watchdog Run ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Watchdog Check — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "━━━ $ICON_WATCHDOG Watchdog Run — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
|
||||
SKIPPED=()
|
||||
RESTARTED=0
|
||||
|
||||
START=$(date +%s)
|
||||
SKIPPED=()
|
||||
|
||||
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
|
||||
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
# Daemon check first — if daemon is down nothing else works
|
||||
check_docker_daemon
|
||||
|
||||
# Verify container exists
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
warn "$container not found on this host — skipping"
|
||||
SKIPPED+=("$container")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
# Verify container is running
|
||||
if ! docker ps --filter "name=^/${container}$" --format "{{.Names}}" | grep -qw "$container"; then
|
||||
warn "$ICON_NOT_RUNNING $container is not running — skipping"
|
||||
SKIPPED+=("$container")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
check_memory "$container" "${WATCHDOG_CONTAINERS[$container]}"
|
||||
check_cpu "$container"
|
||||
check_responsiveness "$container"
|
||||
# Auto-heal skip list before processing
|
||||
check_skip_list_recovery
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Resource monitoring — WATCHDOG_CONTAINERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
done
|
||||
echo "━━━ $ICON_MEM Resource Monitoring ━━━"
|
||||
|
||||
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
|
||||
echo ""
|
||||
info "$ICON_CONTAINERS $container"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
warn "$container not found — skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! docker ps --filter "name=^/${container}$" --format "{{.Names}}" | grep -qw "$container"; then
|
||||
warn "$ICON_NOT_RUNNING $container is not running — skipping resource checks"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
check_memory "$container" "${WATCHDOG_CONTAINERS[$container]}"
|
||||
check_cpu "$container"
|
||||
check_responsiveness "$container"
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Required container monitoring — WATCHDOG_REQUIRED_CONTAINERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Required Container Check ━━━"
|
||||
check_required_containers
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_WATCHDOG Monitored: ${#WATCHDOG_CONTAINERS[@]} containers"
|
||||
echo "$ICON_NOT_RUNNING Skipped: ${#SKIPPED[@]} containers"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Dry Run: no restarts executed"
|
||||
fi
|
||||
echo "$ICON_TIME $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_WATCHDOG Monitored: ${#WATCHDOG_CONTAINERS[@]} containers"
|
||||
echo "$ICON_CONTAINERS Required: ${#WATCHDOG_REQUIRED_CONTAINERS[@]} containers"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]}"
|
||||
[[ "$DRY_RUN" == true ]] && echo "$ICON_WARN Dry Run: no actions taken"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -0,0 +1,174 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Docker Weekly Restart --------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Restarts or starts specified Docker containers with retry logic.
|
||||
# Containers are configured in Master.conf under WEEKLY_RESTART_CONTAINERS.
|
||||
# Uses global RETRY_COUNT and SLEEP from Master.conf for retry behaviour.
|
||||
# Sends notifications on completion or failure via common.sh notify().
|
||||
# Supports --dry-run to preview what would be restarted without taking action.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found — check PATH or Docker installation"
|
||||
notify "Docker weekly restart failed — Docker not found on $(hostname)" "Docker Weekly Restart" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Docker found"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
|
||||
echo "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||
echo "$ICON_TIME Sleep: ${SLEEP}s between retries"
|
||||
echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Attempts a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
|
||||
# Returns 0 on success, 1 if all attempts fail.
|
||||
retry_docker() {
|
||||
local attempt=1
|
||||
|
||||
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
|
||||
info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
||||
|
||||
if "$@"; then
|
||||
success "Succeeded on attempt $attempt"
|
||||
return 0
|
||||
else
|
||||
warn "Attempt $attempt failed"
|
||||
(( attempt++ ))
|
||||
[[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP"
|
||||
fi
|
||||
done
|
||||
|
||||
error "Command failed after $RETRY_COUNT attempts: $*"
|
||||
return 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CONTAINERS Weekly Restart ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Weekly Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
|
||||
echo "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
FAILED=()
|
||||
RESTARTED=()
|
||||
STARTED=()
|
||||
|
||||
for container in "${WEEKLY_RESTART_CONTAINERS[@]}"; do
|
||||
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
error "$container does not exist — skipping"
|
||||
FAILED+=("$container")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
|
||||
case "$STATUS" in
|
||||
true)
|
||||
echo "$ICON_RUNNING $container is running — restarting..."
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container"
|
||||
else
|
||||
if retry_docker docker restart "$container"; then
|
||||
echo "$ICON_STARTED $container restarted"
|
||||
RESTARTED+=("$container")
|
||||
else
|
||||
error "Failed to restart $container after $RETRY_COUNT attempts"
|
||||
notify "$container failed to restart on $(hostname)" "Docker Weekly Restart" "warning"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
false)
|
||||
echo "$ICON_NOT_RUNNING $container is stopped — starting..."
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would start $container"
|
||||
else
|
||||
if retry_docker docker start "$container"; then
|
||||
echo "$ICON_STARTED $container started"
|
||||
STARTED+=("$container")
|
||||
else
|
||||
error "Failed to start $container after $RETRY_COUNT attempts"
|
||||
notify "$container failed to start on $(hostname)" "Docker Weekly Restart" "warning"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
error "Unknown status for $container: $STATUS"
|
||||
FAILED+=("$container")
|
||||
;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY RESTART SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||
[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE"
|
||||
notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Weekly Restart" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed"
|
||||
notify "Weekly restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Weekly Restart" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Media Permissions Script -----------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Applies permissions and ownership to all configured media shares.
|
||||
# Shares, permissions mode and owner are configured in Master.conf.
|
||||
# Supports --dry-run to preview what would be changed without making changes.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_PERMS Mode: $PERMISSIONS_MODE"
|
||||
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||
echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_PERMS Media Permissions ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_PERMS Media Permissions ━━━"
|
||||
echo "$ICON_PERMS Mode: $PERMISSIONS_MODE"
|
||||
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||
echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
FAILED=()
|
||||
UPDATED=()
|
||||
SKIPPED=()
|
||||
|
||||
for SHARE in "${MEDIA_PERMISSION_SHARES[@]}"; do
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
|
||||
if [[ ! -d "$SHARE" ]]; then
|
||||
warn "$ICON_PERMS $SHARE_NAME not found — skipping"
|
||||
SKIPPED+=("$SHARE_NAME")
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would apply $PERMISSIONS_MODE $PERMISSIONS_OWNER to $SHARE"
|
||||
continue
|
||||
fi
|
||||
|
||||
info "$ICON_PERMS Updating $SHARE_NAME..."
|
||||
|
||||
CHMOD_OK=true
|
||||
CHOWN_OK=true
|
||||
|
||||
chmod -R "$PERMISSIONS_MODE" "$SHARE" 2>/dev/null || CHMOD_OK=false
|
||||
chown -R "$PERMISSIONS_OWNER" "$SHARE" 2>/dev/null || CHOWN_OK=false
|
||||
|
||||
if [[ "$CHMOD_OK" == true && "$CHOWN_OK" == true ]]; then
|
||||
echo "$ICON_UNLOCKED $SHARE_NAME — permissions applied"
|
||||
UPDATED+=("$SHARE_NAME")
|
||||
else
|
||||
error "$SHARE_NAME — permissions failed (chmod=$CHMOD_OK chown=$CHOWN_OK)"
|
||||
FAILED+=("$SHARE_NAME")
|
||||
fi
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY MEDIA PERMISSIONS SUMMARY ━━━━━"
|
||||
echo "$ICON_PERMS Mode: $PERMISSIONS_MODE"
|
||||
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo ""
|
||||
[[ ${#UPDATED[@]} -gt 0 ]] && echo " $ICON_UNLOCKED Updated: ${#UPDATED[@]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && echo " $ICON_WARN Skipped: ${#SKIPPED[@]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo " $ICON_ERROR Failed: ${#FAILED[@]} — ${FAILED[*]}"
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR SOME SHARES FAILED"
|
||||
notify "Media permissions failed on $(hostname) — ${FAILED[*]}" "Media Permissions" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
notify "Media permissions applied on $(hostname) — ${#UPDATED[@]} shares updated" "Media Permissions" "normal"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -0,0 +1,183 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Media Cleaner Script ---------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Removes unwanted files from media share folders using configurable file patterns.
|
||||
# Supports two profiles: anime and media — each with their own folder list and file patterns.
|
||||
# Profiles and patterns are configured in Master.conf.
|
||||
# Supports --dry-run to preview what would be deleted without making changes.
|
||||
#
|
||||
# Usage:
|
||||
# media_cleaner.sh anime — clean anime shares
|
||||
# media_cleaner.sh media — clean media shares
|
||||
# media_cleaner.sh anime --dry-run — preview anime clean
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Separate profile argument from flags
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
PROFILE=""
|
||||
RAW_ARGS=()
|
||||
|
||||
for ARG in "$@"; do
|
||||
case "$ARG" in
|
||||
--*|*=*) RAW_ARGS+=("$ARG") ;;
|
||||
anime|media) PROFILE="$ARG" ;;
|
||||
*) RAW_ARGS+=("$ARG") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${RAW_ARGS[@]}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
if ! command -v find >/dev/null 2>&1; then
|
||||
error "find command not found — check findutils installation"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$PROFILE" ]]; then
|
||||
error "No profile specified. Usage: media_cleaner.sh <anime|media> [--dry-run]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Resolve profile folders and patterns
|
||||
case "$PROFILE" in
|
||||
anime)
|
||||
CLEAN_FOLDERS=("${ANIME_CLEAN_FOLDERS[@]}")
|
||||
FILE_PATTERNS=("${ANIME_FILE_PATTERNS[@]}")
|
||||
;;
|
||||
media)
|
||||
CLEAN_FOLDERS=("${MEDIA_CLEAN_FOLDERS[@]}")
|
||||
FILE_PATTERNS=("${MEDIA_FILE_PATTERNS[@]}")
|
||||
;;
|
||||
*)
|
||||
error "Unknown profile: $PROFILE — must be anime or media"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
info "$ICON_GEAR Profile: $PROFILE"
|
||||
info "$ICON_CLEAN Folders: ${#CLEAN_FOLDERS[@]}"
|
||||
info "$ICON_TRASH Patterns: ${#FILE_PATTERNS[@]}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_GEAR Profile: $PROFILE"
|
||||
echo "$ICON_CLEAN Folders: ${CLEAN_FOLDERS[*]}"
|
||||
echo "$ICON_TRASH Patterns: ${FILE_PATTERNS[*]}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CLEAN Media Cleaner ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Media Cleaner — $PROFILE ━━━"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
TOTAL_REMOVED=0
|
||||
FAILED=()
|
||||
SKIPPED=()
|
||||
|
||||
for FOLDER in "${CLEAN_FOLDERS[@]}"; do
|
||||
FOLDER_NAME=$(basename "$FOLDER")
|
||||
echo "━━━ $ICON_CLEAN $FOLDER_NAME ━━━"
|
||||
|
||||
if [[ ! -d "$FOLDER" ]]; then
|
||||
warn "$FOLDER_NAME not found — skipping"
|
||||
SKIPPED+=("$FOLDER_NAME")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
# Build find command dynamically from FILE_PATTERNS array
|
||||
CMD=(find "$FOLDER" -type f \()
|
||||
for ((i = 0; i < ${#FILE_PATTERNS[@]}; i++)); do
|
||||
CMD+=(-iname "${FILE_PATTERNS[i]}")
|
||||
if [[ $i -lt $(( ${#FILE_PATTERNS[@]} - 1 )) ]]; then
|
||||
CMD+=(-o)
|
||||
fi
|
||||
done
|
||||
CMD+=(\))
|
||||
|
||||
# Count matching files before acting
|
||||
FILE_COUNT=$("${CMD[@]}" 2>/dev/null | wc -l)
|
||||
|
||||
if [[ "$FILE_COUNT" -eq 0 ]]; then
|
||||
success "$FOLDER_NAME — no matching files found"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
info "$ICON_TRASH $FILE_COUNT file(s) found in $FOLDER_NAME"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — files that would be deleted:"
|
||||
"${CMD[@]}" 2>/dev/null | while IFS= read -r f; do
|
||||
echo " $ICON_TRASH $f"
|
||||
done
|
||||
else
|
||||
CLEAN_CMD=("${CMD[@]}" -exec rm -f {} +)
|
||||
if "${CLEAN_CMD[@]}" 2>/dev/null; then
|
||||
success "$FOLDER_NAME — $FILE_COUNT file(s) removed"
|
||||
TOTAL_REMOVED=$((TOTAL_REMOVED + FILE_COUNT))
|
||||
else
|
||||
error "$FOLDER_NAME — cleanup failed"
|
||||
FAILED+=("$FOLDER_NAME")
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo "━━━━━ $ICON_SUMMARY MEDIA CLEANER SUMMARY ━━━━━"
|
||||
echo "$ICON_GEAR Profile: $PROFILE"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_WARN Skipped: ${SKIPPED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no files deleted"
|
||||
elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR SOME FOLDERS FAILED"
|
||||
notify "Media cleaner ($PROFILE) failed on $(hostname) — ${FAILED[*]}" "Media Cleaner" "warning"
|
||||
else
|
||||
echo "$ICON_TRASH Removed: $TOTAL_REMOVED file(s)"
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
notify "Media cleaner ($PROFILE) complete on $(hostname) — $TOTAL_REMOVED file(s) removed" "Media Cleaner" "normal"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
+302
-104
@@ -1,63 +1,96 @@
|
||||
#!/bin/bash
|
||||
# ----------------------------------------------------------------------------------------------
|
||||
# ---------------------------- Master Variables for unRAID scripts -----------------------------
|
||||
# ----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= MASTER CONFIGURATION =======================================
|
||||
# ==============================================================================================
|
||||
# All user-facing variables for the unRAID script ecosystem.
|
||||
# Scripts source this file — edit here, changes apply everywhere on next git pull.
|
||||
#
|
||||
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||
#
|
||||
# Section Description
|
||||
# ───────────────────────────────────────────────────────────────────────────────────────────
|
||||
# HOST CONFIGURATION Server hostnames and SSH key paths
|
||||
# LOGGING Enable or disable verbose logging
|
||||
# NOTIFICATIONS unRAID native and Discord webhook settings
|
||||
# GIT / REPO Gitea repository and SSH settings
|
||||
#
|
||||
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||
# RSYNC DEFAULTS Global fallback rsync settings
|
||||
# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort
|
||||
# DAILY SYNC SHARES Media shares synced by daily_sync.sh
|
||||
# RSYNC PROFILE SYSTEM Per-profile overrides (appdata profiles)
|
||||
#
|
||||
# ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
|
||||
# DOCKER DAILY RESTART Containers restarted daily
|
||||
# DOCKER WEEKLY RESTART Containers restarted weekly
|
||||
# DOCKER WATCHDOG Container health monitoring — memory, CPU, HTTP
|
||||
#
|
||||
# ── UNRAID ESSENTIALS ──────────────────────────────────────────────────────────────────────
|
||||
# REBOOT User warning delay before scheduled reboot
|
||||
# MOVER Mover stop timeout
|
||||
# SYSLOG FILTER Docker veth noise filter file path
|
||||
# PHP-FPM PHP-FPM max children config
|
||||
# CLEAR LOGS System log file paths
|
||||
#
|
||||
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||
# MEDIA PERMISSIONS Share list, mode and owner for permissions script
|
||||
# MEDIA CLEANER Anime and media folder lists and file patterns
|
||||
#
|
||||
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||
# SYSTEM WATCHDOG System health monitoring — last line of defense
|
||||
#
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Host Configuration ━━━
|
||||
# List the Hostnames of both servers
|
||||
# Hostnames must match Tailscale machine names exactly — case sensitive
|
||||
HOST1="unRAID-Gmer4Lfe"
|
||||
HOST2="unRAID-Jayred365"
|
||||
# Assign SSH keys for each host pair (adjust paths as needed) HOST1 from above
|
||||
# Must have its key added to HOST1_SSH_Key same applies for HOST2
|
||||
|
||||
# SSH keys for server-to-server rsync — each server needs the other's key authorised
|
||||
HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
|
||||
HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
|
||||
|
||||
# ━━━ Logging ━━━
|
||||
ENABLE_LOGGING=true # false = only echo user-facing messages
|
||||
# true = verbose [LOG] output in scripts / false = user-facing output only
|
||||
ENABLE_LOGGING=true
|
||||
|
||||
# ━━━ Notifications ━━━
|
||||
# unRAID native notification system — set to true to enable
|
||||
# Configure unRAID to send errors only: Settings → Notification Settings
|
||||
# unRAID native — configure Settings → Notification Settings for errors/warnings only
|
||||
NOTIFY_UNRAID=true
|
||||
# Discord webhook URL — leave blank to disable
|
||||
DISCORD_WEBHOOK=""
|
||||
|
||||
# ━━━ Git, Pull & Execute Script ━━━
|
||||
REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git"
|
||||
TARGET_DIR="/mnt/user/appdata/unraid_scripts"
|
||||
GITEA_SSH_KEY="/root/.ssh/id_gitea_rsync"
|
||||
SSH_PORT=221
|
||||
# ━━━ Git / Repo ━━━
|
||||
REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git"
|
||||
TARGET_DIR="/mnt/user/appdata/unraid_scripts"
|
||||
GITEA_SSH_KEY="/root/.ssh/id_gitea_rsync"
|
||||
SSH_PORT=221
|
||||
|
||||
# ━━━ Rsync Script Defaults ━━━
|
||||
# These are the fallback values used when no matching profile is found.
|
||||
# Any share whose directory basename does not match a profile key below
|
||||
# will use these globals for all rsync behaviour.
|
||||
BW_LIMIT=12500
|
||||
# Retry logic
|
||||
RETRY_COUNT=3
|
||||
# Sleep between retries (seconds)
|
||||
SLEEP=300
|
||||
# Container start/stop/restart before and after rsync
|
||||
CRITICAL_CONTAINER_NAMES=()
|
||||
# Containers that need delayed before starting
|
||||
DELAYED_CONTAINERS=()
|
||||
# Delay in seconds between starting containers, useful for things like Authelia
|
||||
CONTAINER_DELAY=5
|
||||
# Directories to exclude during transfer
|
||||
EXCLUDE_DIRS=()
|
||||
# Default global rsync options
|
||||
# ==============================================================================================
|
||||
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Rsync Defaults ━━━
|
||||
# Global fallback values — used when no profile match is found for a share.
|
||||
# Shares in DAILY_SYNC_SHARES always use these globals (no profile defined).
|
||||
BW_LIMIT=12500 # network speed limit KB/s
|
||||
RETRY_COUNT=3 # number of retry attempts on failure
|
||||
SLEEP=300 # seconds between retries
|
||||
CRITICAL_CONTAINER_NAMES=() # containers to stop before rsync
|
||||
DELAYED_CONTAINERS=() # containers needing delay before start
|
||||
CONTAINER_DELAY=5 # seconds delay before starting delayed containers
|
||||
EXCLUDE_DIRS=() # directories to exclude from transfer
|
||||
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)
|
||||
|
||||
# ━━━ Remote Health Checks ━━━
|
||||
# Abort if remote rootfs usage is at or above this percentage.
|
||||
# Protects against rsync writing to rootfs when the remote array is down or drives are missing.
|
||||
# Recommended: 75 — gives headroom before the server becomes unstable
|
||||
# Abort rsync if remote rootfs exceeds this percentage.
|
||||
# Protects against rsync filling rootfs when remote array is down or drives are missing.
|
||||
ROOTFS_WARN=75
|
||||
|
||||
# ━━━ Daily Sync Shares ━━━
|
||||
# Shares synced once daily by Orchestrators/daily_sync.sh
|
||||
# No profile needed — all fall through to DEFAULT_RSYNC_OPTS above.
|
||||
# Add or remove paths here to manage what gets synced.
|
||||
# These shares have no profile entry and fall through to DEFAULT_RSYNC_OPTS above.
|
||||
DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows-Old
|
||||
@@ -73,81 +106,22 @@ DAILY_SYNC_SHARES=(
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# ━━━ unRAID Essential Scripts ━━━
|
||||
# unRAID reboot script user warning time (seconds)
|
||||
REBOOT_SLEEP=300
|
||||
# unRAID mover stop script timeout (seconds)
|
||||
MOVER_STOP_TIMEOUT=300
|
||||
# docker_syslog_filter.sh filter file path
|
||||
FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
|
||||
# php_fpm_max_children.sh php-fpm config file path
|
||||
PHP_CONF="/etc/php-fpm.d/www.conf"
|
||||
# php_fpm_max_children.sh max children value
|
||||
PHP_MAX_CHILDREN=250
|
||||
# clear_logs.sh system log file paths
|
||||
LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers to restart daily — space-separated, case-sensitive
|
||||
# These are the same containers as critical-data and other rsync profiles
|
||||
DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Authelia"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr"
|
||||
"Dispatcharr-Basic"
|
||||
"Code-Server"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# Containers to monitor with their memory hard limits in MB
|
||||
# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240 8GB=8192 6GB=6144 4GB=4096 1GB=1024
|
||||
declare -A WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=16384
|
||||
["jellyfin_with_request"]=12288
|
||||
["LidaTube"]=6144
|
||||
["Tdarr"]=6144
|
||||
["Code-Server"]=1024
|
||||
)
|
||||
|
||||
# Containers to check for HTTP responsiveness — omit a container to skip its check
|
||||
declare -A WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
["Jellyfin-Gmer4Lfe"]="http://localhost:8095"
|
||||
)
|
||||
|
||||
# State file for tracking CPU and responsiveness strikes between runs
|
||||
# Lives in /tmp — resets on reboot which is correct behaviour for strike tracking
|
||||
WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
|
||||
|
||||
# CPU thresholds — normalised against total core count automatically at runtime
|
||||
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
|
||||
HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU
|
||||
CPU_FAIL_LIMIT=2 # consecutive strikes before restart
|
||||
|
||||
# Memory threshold
|
||||
SOFT_MEM_THRESHOLD=80 # warn at this % of per-container hard limit
|
||||
|
||||
# Responsiveness check settings
|
||||
RESP_FAIL_LIMIT=2 # consecutive failures before restart
|
||||
CURL_TIMEOUT=5 # seconds before curl gives up
|
||||
|
||||
# ━━━ Profile System ━━━
|
||||
# ━━━ Rsync Profile System ━━━
|
||||
# Profiles are matched by directory basename (lowercased).
|
||||
# Example: /mnt/user/appdata-Failover/Arrs_Stack → profile key = arrs_stack
|
||||
#
|
||||
# How fallthrough works:
|
||||
# - If a key exists in a profile array, that value is used
|
||||
# - If a key is missing, the global default above is used instead
|
||||
# - If a key exists in a profile array that value is used
|
||||
# - If a key is missing the global default above is used instead
|
||||
# - Shares in DAILY_SYNC_SHARES have no profile and always use globals
|
||||
#
|
||||
# To add a new profile:
|
||||
# 1. Add a key to each array below with your chosen profile name
|
||||
# 2. Call rsync.sh with a directory whose basename matches that key
|
||||
# 3. Any array you omit will fall back to its global default
|
||||
# 3. Any array you omit falls back to its global default
|
||||
#
|
||||
# Note: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS —
|
||||
# if you define it for a profile you must list all desired options explicitly
|
||||
# list all desired options explicitly if you define a profile entry
|
||||
|
||||
# SPACE-SEPARATED STRINGS
|
||||
declare -A PROFILE_RSYNC_OPTS=(
|
||||
@@ -191,7 +165,7 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
|
||||
[emby]=""
|
||||
)
|
||||
|
||||
# SPACE-SEPARATED STRINGS — containers that need a delay before starting
|
||||
# SPACE-SEPARATED STRINGS — containers needing delay before starting
|
||||
declare -A PROFILE_DELAYED_CONTAINERS=(
|
||||
[arrs_stack]=""
|
||||
[critical-data]="Authelia"
|
||||
@@ -216,6 +190,230 @@ declare -A PROFILE_EXCLUDE_DIRS=(
|
||||
[important-data]="logs *.tmp"
|
||||
[emby]="logs *.tmp"
|
||||
)
|
||||
# ----------------------------------------------------------------------------------------------
|
||||
# ---------------------- End Of User Variables, Please adjust above as needed ------------------
|
||||
# ----------------------------------------------------------------------------------------------
|
||||
|
||||
# ==============================================================================================
|
||||
# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Docker Daily Restart ━━━
|
||||
# Containers restarted every day — case-sensitive names
|
||||
DAILY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Authelia"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr"
|
||||
"Dispatcharr-Basic"
|
||||
"Code-Server"
|
||||
)
|
||||
|
||||
# ━━━ Docker Weekly Restart ━━━
|
||||
# Containers restarted once per week — case-sensitive names
|
||||
WEEKLY_RESTART_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Authelia"
|
||||
"Dispatcharr-Iptv-Users"
|
||||
"Dispatcharr"
|
||||
"Dispatcharr-Basic"
|
||||
"Code-Server"
|
||||
)
|
||||
|
||||
# ━━━ Docker Watchdog ━━━
|
||||
# First line of defense — monitors and restarts unhealthy containers.
|
||||
# Runs on a cron schedule (recommended every 15 minutes).
|
||||
# Strike system prevents restarts on brief spikes.
|
||||
|
||||
# Containers to monitor with memory hard limits in MB
|
||||
# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240
|
||||
# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024
|
||||
declare -A WATCHDOG_CONTAINERS=(
|
||||
["Emby"]=16384
|
||||
["LidaTube"]=6144
|
||||
["Tdarr"]=6144
|
||||
["Code-Server"]=1024
|
||||
)
|
||||
|
||||
# Containers to check HTTP responsiveness — omit to skip
|
||||
declare -A WATCHDOG_CONTAINER_URLS=(
|
||||
["Emby"]="http://localhost:8096"
|
||||
)
|
||||
|
||||
# Containers that should always be running — monitored for unexpected stops
|
||||
# Strike system used — persistent skip list prevents reboot loops
|
||||
WATCHDOG_REQUIRED_CONTAINERS=(
|
||||
"NginxProxyManager"
|
||||
"Authelia"
|
||||
"Emby"
|
||||
)
|
||||
|
||||
# Strike state file — /tmp resets on reboot, correct for strike tracking
|
||||
WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
|
||||
|
||||
# CPU thresholds — normalised against total core count at runtime
|
||||
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
|
||||
HARD_CPU_THRESHOLD=90 # strike at this % of total system CPU
|
||||
CPU_FAIL_LIMIT=2 # consecutive strikes before restart
|
||||
|
||||
# Memory threshold
|
||||
SOFT_MEM_THRESHOLD=80 # warn at this % of per-container hard limit
|
||||
|
||||
# Responsiveness check
|
||||
RESP_FAIL_LIMIT=2 # consecutive failures before restart
|
||||
CURL_TIMEOUT=5 # seconds before curl gives up
|
||||
|
||||
# ==============================================================================================
|
||||
# ── UNRAID ESSENTIALS ─────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Reboot ━━━
|
||||
# User warning delay before scheduled reboot (seconds)
|
||||
REBOOT_SLEEP=300
|
||||
|
||||
# ━━━ Mover ━━━
|
||||
# Timeout before stopping the mover (seconds)
|
||||
MOVER_STOP_TIMEOUT=300
|
||||
|
||||
# ━━━ Syslog Filter ━━━
|
||||
# Path for the rsyslog Docker noise filter file
|
||||
FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
|
||||
|
||||
# ━━━ PHP-FPM ━━━
|
||||
# PHP-FPM config file path and max children value
|
||||
PHP_CONF="/etc/php-fpm.d/www.conf"
|
||||
PHP_MAX_CHILDREN=250
|
||||
|
||||
# ━━━ Clear Logs ━━━
|
||||
# System log files to clear on each run
|
||||
LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# ━━━ Media Permissions ━━━
|
||||
# Mode and owner applied recursively to all listed shares
|
||||
PERMISSIONS_MODE="777"
|
||||
PERMISSIONS_OWNER="nobody:users"
|
||||
|
||||
MEDIA_PERMISSION_SHARES=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
/mnt/user/appcache
|
||||
/mnt/user/Books
|
||||
/mnt/user/Downloads
|
||||
/mnt/user/Games
|
||||
/mnt/user/Intros
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movie_Recordings
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Music_Videos
|
||||
/mnt/user/Photo
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Temp_Storage
|
||||
/mnt/user/Tv_Recordings
|
||||
/mnt/user/Tv_Shows
|
||||
/mnt/user/YouTube
|
||||
)
|
||||
|
||||
# ━━━ Media Cleaner ━━━
|
||||
# Two profiles: anime and media — passed as argument to media_cleaner.sh
|
||||
# Usage: media_cleaner.sh anime or media_cleaner.sh media
|
||||
|
||||
ANIME_CLEAN_FOLDERS=(
|
||||
/mnt/user/Anime_Movies
|
||||
/mnt/user/Anime_Movies-Old
|
||||
/mnt/user/Anime_Shows
|
||||
/mnt/user/Anime_Shows-Old
|
||||
)
|
||||
|
||||
MEDIA_CLEAN_FOLDERS=(
|
||||
/mnt/user/Kids_Movies
|
||||
/mnt/user/Kids_Tv_Shows
|
||||
/mnt/user/Movies
|
||||
/mnt/user/Music
|
||||
/mnt/user/Sports
|
||||
/mnt/user/stand-up_comedy
|
||||
/mnt/user/Tv_Shows
|
||||
)
|
||||
|
||||
# Anime file patterns — junk files common in anime downloads
|
||||
ANIME_FILE_PATTERNS=(
|
||||
'*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
|
||||
'*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
|
||||
'*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp'
|
||||
'*.log' '*.json'
|
||||
)
|
||||
|
||||
# Media file patterns — includes *.iso and *.lrc not needed in anime
|
||||
MEDIA_FILE_PATTERNS=(
|
||||
'*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
|
||||
'*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
|
||||
'*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp'
|
||||
'*.log' '*.json' '*.iso' '*.lrc'
|
||||
)
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Last line of defense — reboots the system cleanly if it is about to become unstable.
|
||||
# Designed to run on a cron schedule (recommended every 15-30 minutes).
|
||||
# Works alongside docker_watchdog.sh — containers first, system second.
|
||||
#
|
||||
# Strike system — sustained threshold hits trigger reboot, not single spikes.
|
||||
# Reboot loop protection — shuts down instead of rebooting if limit hit in window.
|
||||
# Container skip list — persistent, auto-clears when container recovers.
|
||||
|
||||
# ━━━ System Watchdog State Files ━━━
|
||||
# Strike counts — /tmp resets on reboot, correct for strike tracking
|
||||
SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db"
|
||||
# Persistent container skip list — survives reboots, auto-clears on recovery
|
||||
SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db"
|
||||
# Reboot timestamp log — survives reboots for loop detection
|
||||
SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db"
|
||||
|
||||
# ━━━ Strike and Reboot Loop Settings ━━━
|
||||
# Consecutive threshold hits before triggering reboot
|
||||
SYS_WATCHDOG_STRIKE_LIMIT=2
|
||||
# Maximum reboots allowed within the window before shutdown instead
|
||||
SYS_WATCHDOG_REBOOT_LIMIT=3
|
||||
# Window in hours — controls both reboot count window AND rolling log purge
|
||||
# 12 = entries older than 12hrs purge automatically / 24 = entries older than 24hrs purge
|
||||
SYS_WATCHDOG_REBOOT_WINDOW_HRS=12
|
||||
|
||||
# ━━━ Thresholds ━━━
|
||||
# Set at "about to fall over" levels — not just high usage
|
||||
SYS_WATCHDOG_ROOTFS_PCT=95 # rootfs usage % before strike
|
||||
SYS_WATCHDOG_LOG_PCT=95 # /var/log usage % before strike
|
||||
SYS_WATCHDOG_MEM_GB=4 # free RAM in GB below which strikes
|
||||
SYS_WATCHDOG_ARC_PINNED_PCT=98 # ZFS ARC % of max before reclaim attempt
|
||||
SYS_WATCHDOG_ARC_RELEASE_PCT=95 # ZFS ARC % after reclaim that still triggers
|
||||
SYS_WATCHDOG_LOAD_MULTIPLIER=32 # strike if load avg > cores x this value
|
||||
SYS_WATCHDOG_ZOMBIE_LIMIT=50 # strike if zombie process count exceeds this
|
||||
SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — tjmax varies by CPU
|
||||
|
||||
# ━━━ Check Toggles ━━━
|
||||
# true = run this check / false = skip entirely
|
||||
SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||
SYS_WATCHDOG_CHECK_LOG=true
|
||||
SYS_WATCHDOG_CHECK_RAM=true
|
||||
SYS_WATCHDOG_CHECK_ARC=true
|
||||
SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||
SYS_WATCHDOG_CHECK_LOAD=true
|
||||
SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||
SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||
SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||
|
||||
# ━━━ Abort Toggles ━━━
|
||||
# true = abort reboot if condition is active / false = reboot anyway
|
||||
# Default true = conservative / set false to reboot regardless
|
||||
SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true
|
||||
SYS_WATCHDOG_ABORT_ON_PARITY=false
|
||||
SYS_WATCHDOG_ABORT_ON_MOVER=false
|
||||
|
||||
# ==============================================================================================
|
||||
# ──────────────────────── End Of User Variables ───────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
@@ -2,7 +2,7 @@
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ----------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Version: 2.3
|
||||
# Version: 2.4
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Changelog:
|
||||
# v1.0 — Initial stable framework
|
||||
@@ -37,6 +37,9 @@
|
||||
# v2.3 — ICON_NOTIFY added for notification operations
|
||||
# notify() added — shared notification function supporting unRAID native and Discord
|
||||
# NOTIFY_UNRAID and DISCORD_WEBHOOK configured in Master.conf
|
||||
# v2.4 — ICON_CLEAN, ICON_TRASH added for media cleaner operations
|
||||
# ICON_PERMS, ICON_UNLOCKED added for media permissions operations
|
||||
# ICON_REBOOT_SMART added for smart conditional reboot
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
@@ -76,7 +79,8 @@ ICON_TIME="⏱️" # duration line
|
||||
|
||||
# System Operations
|
||||
ICON_MOVER="🔃" # mover operations
|
||||
ICON_REBOOT="⚡" # server reboot operations
|
||||
ICON_REBOOT="⚡" # scheduled server reboot
|
||||
ICON_REBOOT_SMART="🚨" # smart conditional reboot triggered
|
||||
ICON_PLUGIN="🧩" # user scripts plugin operations
|
||||
ICON_PHP="👥" # PHP-FPM operations
|
||||
|
||||
@@ -85,6 +89,12 @@ ICON_ZFS="📊" # ZFS ARC statistics
|
||||
ICON_MEM="🧠" # memory status
|
||||
ICON_WATCHDOG="🐾" # docker watchdog monitoring operations
|
||||
|
||||
# Media Operations
|
||||
ICON_CLEAN="🧹" # media cleaner operations
|
||||
ICON_TRASH="🗑️" # files being deleted
|
||||
ICON_PERMS="🔐" # permissions operation / section header
|
||||
ICON_UNLOCKED="🔓" # permissions successfully applied to a share
|
||||
|
||||
# Notifications
|
||||
ICON_NOTIFY="🔔" # notification operations
|
||||
|
||||
@@ -116,7 +126,7 @@ log() {
|
||||
# Discord: requires DISCORD_WEBHOOK to be set to a valid webhook URL.
|
||||
# Severity levels: normal, warning, alert — maps to unRAID notification severity.
|
||||
# Usage: notify "message" "subject" "severity"
|
||||
# notify "Rsync failed: Movies" "Rsync Alert" "alert"
|
||||
# notify "Rsync failed: Movies" "Rsync Alert" "warning"
|
||||
# notify "Daily sync complete" "Daily Sync" "normal"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
notify() {
|
||||
@@ -139,8 +149,6 @@ notify() {
|
||||
|
||||
# Discord webhook notification
|
||||
if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then
|
||||
local hostname
|
||||
hostname=$(hostname)
|
||||
local payload
|
||||
payload=$(printf '{"content": "%s — **%s**\\n%s"}' \
|
||||
"$ICON_NOTIFY" "$subject" "$message")
|
||||
|
||||
@@ -0,0 +1,495 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- System Watchdog --------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Last line of defense — reboots the system cleanly if it is about to become unstable.
|
||||
# Works alongside docker_watchdog.sh which handles container-level healing first.
|
||||
#
|
||||
# Checks (all toggleable in Master.conf):
|
||||
# rootfs usage — high rootfs fills rapidly when array is down, crash imminent
|
||||
# /var/log usage — log spam can fill rootfs, indicates something is broken
|
||||
# free RAM — critically low RAM means OOM or swap imminent
|
||||
# ZFS ARC pinned — ARC not releasing after reclaim means memory is stuck
|
||||
# CPU temperature — sustained tjmax causes throttling or kernel panic
|
||||
# load average — sustained high load means something is stuck or runaway
|
||||
# zombie processes — large zombie count indicates serious process management failure
|
||||
# Docker daemon — unresponsive daemon means containers cannot be managed
|
||||
# Required containers — stopped containers that should be running (after watchdog skip list)
|
||||
#
|
||||
# Abort conditions (toggleable — true = abort, false = reboot anyway):
|
||||
# ZFS pool unhealthy — reboot with bad pool risks data loss
|
||||
# Parity running — aborting parity is better than crashing mid-check
|
||||
# Mover running — aborting move is better than crashing mid-move
|
||||
#
|
||||
# Reboot loop protection:
|
||||
# Tracks reboot timestamps in persistent log on /boot/
|
||||
# Rolling window — old entries purge automatically after SYS_WATCHDOG_REBOOT_WINDOW_HRS
|
||||
# If reboot count hits SYS_WATCHDOG_REBOOT_LIMIT in window → shutdown instead of reboot
|
||||
#
|
||||
# All configuration in Master.conf under System Watchdog section.
|
||||
# Supports --dry-run to show triggered conditions without rebooting.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# Convert window hours to seconds for internal use
|
||||
SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 ))
|
||||
|
||||
TOTAL_CORES=$(nproc)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
# Ensure state and persistent files exist
|
||||
touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || {
|
||||
error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
touch "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || {
|
||||
error "Cannot create reboot log: $SYS_WATCHDOG_REBOOT_LOG"
|
||||
exit 1
|
||||
}
|
||||
|
||||
touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || {
|
||||
error "Cannot create failed container list: $SYS_WATCHDOG_FAILED_FILE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HEALTH rootfs threshold: ${SYS_WATCHDOG_ROOTFS_PCT}% toggle: $SYS_WATCHDOG_CHECK_ROOTFS"
|
||||
echo "$ICON_HEALTH log threshold: ${SYS_WATCHDOG_LOG_PCT}% toggle: $SYS_WATCHDOG_CHECK_LOG"
|
||||
echo "$ICON_MEM RAM threshold: ${SYS_WATCHDOG_MEM_GB}GB free toggle: $SYS_WATCHDOG_CHECK_RAM"
|
||||
echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}% toggle: $SYS_WATCHDOG_CHECK_ARC"
|
||||
echo "$ICON_ZFS ARC release: ${SYS_WATCHDOG_ARC_RELEASE_PCT}%"
|
||||
echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C toggle: $SYS_WATCHDOG_CHECK_CPU_TEMP"
|
||||
echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x cores toggle: $SYS_WATCHDOG_CHECK_LOAD"
|
||||
echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT} toggle: $SYS_WATCHDOG_CHECK_ZOMBIES"
|
||||
echo "$ICON_CONTAINERS Docker daemon: toggle: $SYS_WATCHDOG_CHECK_DOCKER_DAEMON"
|
||||
echo "$ICON_CONTAINERS Containers: toggle: $SYS_WATCHDOG_CHECK_CONTAINERS"
|
||||
echo "$ICON_SHIELD Strike limit: $SYS_WATCHDOG_STRIKE_LIMIT"
|
||||
echo "$ICON_REBOOT_SMART Reboot limit: $SYS_WATCHDOG_REBOOT_LIMIT in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs"
|
||||
echo "$ICON_ZFS Abort ZFS unhealthy: $SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY"
|
||||
echo "$ICON_GEAR Abort on parity: $SYS_WATCHDOG_ABORT_ON_PARITY"
|
||||
echo "$ICON_MOVER Abort on mover: $SYS_WATCHDOG_ABORT_ON_MOVER"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboot will be executed"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# STATE HELPERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
get_strikes() {
|
||||
local key="$1"
|
||||
grep -E "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2
|
||||
}
|
||||
|
||||
set_strikes() {
|
||||
local key="$1" count="$2"
|
||||
grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
||||
echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
||||
mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE"
|
||||
}
|
||||
|
||||
# Returns current strike count, increments and saves, then echoes new count
|
||||
increment_strikes() {
|
||||
local key="$1"
|
||||
local current
|
||||
current=$(get_strikes "$key")
|
||||
[[ -z "$current" ]] && current=0
|
||||
((current++))
|
||||
set_strikes "$key" "$current"
|
||||
echo "$current"
|
||||
}
|
||||
|
||||
reset_strikes() {
|
||||
local key="$1"
|
||||
set_strikes "$key" 0
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# REBOOT LOG HELPERS
|
||||
# Tracks reboot timestamps for loop detection.
|
||||
# Rolling window — entries older than SYS_WATCHDOG_REBOOT_WINDOW are purged automatically.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
purge_old_reboots() {
|
||||
local now
|
||||
now=$(date +%s)
|
||||
local cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW ))
|
||||
|
||||
grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do
|
||||
[[ "$ts" -gt "$cutoff" ]] && echo "$ts"
|
||||
done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp"
|
||||
mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG"
|
||||
}
|
||||
|
||||
count_recent_reboots() {
|
||||
purge_old_reboots
|
||||
grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0
|
||||
}
|
||||
|
||||
log_reboot() {
|
||||
date +%s >> "$SYS_WATCHDOG_REBOOT_LOG"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ABORT CONDITION CHECKS
|
||||
# Run before any reboot is triggered — abort conditions prevent rebooting
|
||||
# when it would make things worse than letting the system run.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
check_abort_conditions() {
|
||||
local should_abort=false
|
||||
|
||||
# ZFS pool health
|
||||
if command -v zpool >/dev/null 2>&1; then
|
||||
local unhealthy
|
||||
unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true)
|
||||
if [[ -n "$unhealthy" ]]; then
|
||||
if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then
|
||||
error "$ICON_ZFS ZFS pool unhealthy — aborting reboot (ABORT_ON_ZFS_UNHEALTHY=true)"
|
||||
notify "System watchdog aborted reboot on $(hostname) — ZFS pool unhealthy" "System Watchdog" "warning"
|
||||
should_abort=true
|
||||
else
|
||||
warn "$ICON_ZFS ZFS pool unhealthy — continuing reboot (ABORT_ON_ZFS_UNHEALTHY=false)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Parity check running
|
||||
if [[ -f /var/local/emhttp/parity-date.txt ]]; then
|
||||
if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then
|
||||
if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then
|
||||
error "$ICON_DISK Parity check running — aborting reboot (ABORT_ON_PARITY=true)"
|
||||
notify "System watchdog aborted reboot on $(hostname) — parity check running" "System Watchdog" "warning"
|
||||
should_abort=true
|
||||
else
|
||||
warn "$ICON_DISK Parity check running — continuing reboot (ABORT_ON_PARITY=false)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Mover running
|
||||
if pgrep -f "mover" >/dev/null 2>&1; then
|
||||
if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then
|
||||
error "$ICON_MOVER Mover is running — aborting reboot (ABORT_ON_MOVER=true)"
|
||||
notify "System watchdog aborted reboot on $(hostname) — mover running" "System Watchdog" "warning"
|
||||
should_abort=true
|
||||
else
|
||||
warn "$ICON_MOVER Mover is running — continuing reboot (ABORT_ON_MOVER=false)"
|
||||
fi
|
||||
fi
|
||||
|
||||
[[ "$should_abort" == true ]] && return 1
|
||||
return 0
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# STRIKE-BASED CHECK HELPER
|
||||
# Runs a check function, increments strikes on trigger, resets on clear.
|
||||
# Returns 0 if strike limit hit (reboot trigger), 1 otherwise.
|
||||
# Usage: run_strike_check "key" "triggered (true/false)" "description"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
run_strike_check() {
|
||||
local key="$1" triggered="$2" description="$3"
|
||||
|
||||
if [[ "$triggered" == true ]]; then
|
||||
local strikes
|
||||
strikes=$(increment_strikes "$key")
|
||||
warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT"
|
||||
if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then
|
||||
error "$description hit strike limit — reboot triggered"
|
||||
reset_strikes "$key"
|
||||
return 0
|
||||
fi
|
||||
else
|
||||
local current
|
||||
current=$(get_strikes "$key")
|
||||
if [[ -n "$current" && "$current" -gt 0 ]]; then
|
||||
info "$description — recovered, resetting strikes"
|
||||
reset_strikes "$key"
|
||||
fi
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SHIELD Health Checks ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_SHIELD Health Checks — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
|
||||
TRIGGERS=()
|
||||
|
||||
# rootfs usage
|
||||
if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then
|
||||
info "$ICON_HEALTH Checking rootfs usage..."
|
||||
ROOTFS_USED=$(df / --output=pcent | tail -1 | tr -d ' %')
|
||||
TRIGGERED=false
|
||||
if [[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]]; then
|
||||
error "$ICON_HEALTH rootfs is ${ROOTFS_USED}% full — threshold ${SYS_WATCHDOG_ROOTFS_PCT}%"
|
||||
TRIGGERED=true
|
||||
else
|
||||
success "$ICON_HEALTH rootfs: ${ROOTFS_USED}% used"
|
||||
fi
|
||||
run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && TRIGGERS+=("rootfs=${ROOTFS_USED}%")
|
||||
fi
|
||||
|
||||
# /var/log usage
|
||||
if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then
|
||||
info "$ICON_HEALTH Checking /var/log usage..."
|
||||
LOG_USED=$(df -P /var/log | awk 'NR==2 {print $5}' | tr -d '%')
|
||||
TRIGGERED=false
|
||||
if [[ "$LOG_USED" -ge "$SYS_WATCHDOG_LOG_PCT" ]]; then
|
||||
error "$ICON_HEALTH /var/log is ${LOG_USED}% full — threshold ${SYS_WATCHDOG_LOG_PCT}%"
|
||||
TRIGGERED=true
|
||||
else
|
||||
success "$ICON_HEALTH /var/log: ${LOG_USED}% used"
|
||||
fi
|
||||
run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && TRIGGERS+=("log=${LOG_USED}%")
|
||||
fi
|
||||
|
||||
# Free RAM
|
||||
if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then
|
||||
info "$ICON_MEM Checking available RAM..."
|
||||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_GB=$((MEM_KB / 1024 / 1024))
|
||||
TRIGGERED=false
|
||||
if [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]]; then
|
||||
error "$ICON_MEM Available RAM: ${MEM_GB}GB — threshold ${SYS_WATCHDOG_MEM_GB}GB"
|
||||
TRIGGERED=true
|
||||
else
|
||||
success "$ICON_MEM Available RAM: ${MEM_GB}GB"
|
||||
fi
|
||||
run_strike_check "ram" "$TRIGGERED" "RAM ${MEM_GB}GB free" && TRIGGERS+=("low_ram=${MEM_GB}GB")
|
||||
fi
|
||||
|
||||
# ZFS ARC pinned
|
||||
if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||||
info "$ICON_ZFS Checking ZFS ARC..."
|
||||
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX ))
|
||||
TRIGGERED=false
|
||||
|
||||
if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then
|
||||
warn "$ICON_ZFS ARC at ${ARC_PCT}% — attempting reclaim..."
|
||||
sync
|
||||
echo 3 > /proc/sys/vm/drop_caches
|
||||
sleep 5
|
||||
ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||
ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX ))
|
||||
if [[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]]; then
|
||||
error "$ICON_ZFS ARC still ${ARC_AFTER_PCT}% after reclaim — threshold ${SYS_WATCHDOG_ARC_RELEASE_PCT}%"
|
||||
TRIGGERED=true
|
||||
else
|
||||
success "$ICON_ZFS ARC released to ${ARC_AFTER_PCT}% after reclaim"
|
||||
fi
|
||||
else
|
||||
success "$ICON_ZFS ARC: ${ARC_PCT}% of max"
|
||||
fi
|
||||
run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned" && TRIGGERS+=("arc_pinned=${ARC_PCT}%")
|
||||
elif [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]]; then
|
||||
info "$ICON_ZFS ZFS arcstats not available — skipping"
|
||||
fi
|
||||
|
||||
# CPU temperature
|
||||
if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then
|
||||
info "$ICON_GEAR Checking CPU temperature..."
|
||||
CPU_TEMP=""
|
||||
if command -v sensors >/dev/null 2>&1; then
|
||||
CPU_TEMP=$(sensors 2>/dev/null | grep -i "Package id 0\|Tctl\|CPU Temp" | awk '{print $NF}' | tr -d '+°C' | head -1)
|
||||
fi
|
||||
|
||||
if [[ -z "$CPU_TEMP" ]]; then
|
||||
info "$ICON_GEAR CPU temperature sensor not available — skipping"
|
||||
else
|
||||
CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP")
|
||||
TRIGGERED=false
|
||||
if [[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]]; then
|
||||
error "$ICON_GEAR CPU temp ${CPU_TEMP_INT}°C — threshold ${SYS_WATCHDOG_CPU_TEMP_MAX}°C"
|
||||
TRIGGERED=true
|
||||
else
|
||||
success "$ICON_GEAR CPU temp: ${CPU_TEMP_INT}°C"
|
||||
fi
|
||||
run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C")
|
||||
fi
|
||||
fi
|
||||
|
||||
# Load average
|
||||
if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then
|
||||
info "$ICON_GEAR Checking load average..."
|
||||
LOAD=$(awk '{print $1}' /proc/loadavg)
|
||||
LOAD_INT=$(printf "%.0f" "$LOAD")
|
||||
LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER ))
|
||||
TRIGGERED=false
|
||||
if [[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]]; then
|
||||
error "$ICON_GEAR Load average ${LOAD} — threshold ${LOAD_THRESHOLD} (${TOTAL_CORES} cores x ${SYS_WATCHDOG_LOAD_MULTIPLIER})"
|
||||
TRIGGERED=true
|
||||
else
|
||||
success "$ICON_GEAR Load average: ${LOAD}"
|
||||
fi
|
||||
run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && TRIGGERS+=("load=${LOAD}")
|
||||
fi
|
||||
|
||||
# Zombie processes
|
||||
if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then
|
||||
info "$ICON_GEAR Checking zombie processes..."
|
||||
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" || echo 0)
|
||||
TRIGGERED=false
|
||||
if [[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]]; then
|
||||
error "$ICON_GEAR Zombie processes: $ZOMBIE_COUNT — threshold $SYS_WATCHDOG_ZOMBIE_LIMIT"
|
||||
TRIGGERED=true
|
||||
else
|
||||
success "$ICON_GEAR Zombie processes: $ZOMBIE_COUNT"
|
||||
fi
|
||||
run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && TRIGGERS+=("zombies=${ZOMBIE_COUNT}")
|
||||
fi
|
||||
|
||||
# Docker daemon health
|
||||
if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then
|
||||
info "$ICON_CONTAINERS Checking Docker daemon..."
|
||||
TRIGGERED=false
|
||||
if ! timeout 10 docker ps >/dev/null 2>&1; then
|
||||
error "$ICON_CONTAINERS Docker daemon is not responding"
|
||||
TRIGGERED=true
|
||||
else
|
||||
success "$ICON_CONTAINERS Docker daemon is healthy"
|
||||
fi
|
||||
run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon unresponsive" && TRIGGERS+=("docker_daemon")
|
||||
fi
|
||||
|
||||
# Required containers from skip list
|
||||
if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||
info "$ICON_CONTAINERS Checking persistent failed containers..."
|
||||
FAILED_CONTAINERS=()
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
if [[ "$STATUS" != "true" ]]; then
|
||||
error "$ICON_NOT_RUNNING $container still not running (from skip list)"
|
||||
FAILED_CONTAINERS+=("$container")
|
||||
fi
|
||||
done < "$SYS_WATCHDOG_FAILED_FILE"
|
||||
|
||||
if [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]]; then
|
||||
TRIGGERED=true
|
||||
run_strike_check "failed_containers" "$TRIGGERED" "required containers stopped" && TRIGGERS+=("containers=${FAILED_CONTAINERS[*]}")
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# No triggers — exit cleanly
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ ${#TRIGGERS[@]} -eq 0 ]]; then
|
||||
echo ""
|
||||
success "No reboot conditions met — system is healthy"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_REBOOT_SMART Reboot Triggered ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_REBOOT_SMART Reboot Triggered ━━━"
|
||||
for t in "${TRIGGERS[@]}"; do
|
||||
echo " $ICON_REBOOT_SMART $t"
|
||||
done
|
||||
echo ""
|
||||
|
||||
# Check abort conditions before proceeding
|
||||
if ! check_abort_conditions; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Reboot loop protection — check recent reboot count
|
||||
RECENT_REBOOTS=$(count_recent_reboots)
|
||||
info "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT"
|
||||
|
||||
if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then
|
||||
error "Reboot limit hit — $RECENT_REBOOTS reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs — shutting down instead"
|
||||
notify "Reboot loop detected on $(hostname) — shutting down after $RECENT_REBOOTS reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs — conditions: ${TRIGGERS[*]}" "System Watchdog" "warning"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would shutdown now"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
sync
|
||||
/sbin/poweroff
|
||||
exit 0
|
||||
fi
|
||||
|
||||
notify "System watchdog reboot triggered on $(hostname) — conditions: ${TRIGGERS[*]}" "System Watchdog" "warning"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — reboot sequence would begin now"
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY SYSTEM WATCHDOG SUMMARY ━━━━━"
|
||||
echo "$ICON_REBOOT_SMART Triggers: ${TRIGGERS[*]}"
|
||||
echo "$ICON_REBOOT_SMART Recent reboots: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT"
|
||||
echo "$ICON_WARN Status: DRY RUN — no reboot executed"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Graceful shutdown sequence
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
info "Logging reboot timestamp..."
|
||||
log_reboot
|
||||
|
||||
info "Shutting down VMs..."
|
||||
if command -v virsh >/dev/null 2>&1; then
|
||||
for VM in $(virsh list --name 2>/dev/null); do
|
||||
[[ -z "$VM" ]] && continue
|
||||
info "Shutting down VM: $VM"
|
||||
virsh shutdown "$VM" >/dev/null 2>&1
|
||||
done
|
||||
info "Waiting 30s for VMs..."
|
||||
sleep 30
|
||||
fi
|
||||
|
||||
info "Stopping Docker containers..."
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
docker ps -q | xargs -r docker stop >/dev/null 2>&1
|
||||
success "Docker containers stopped"
|
||||
fi
|
||||
|
||||
info "Stopping User Scripts..."
|
||||
pkill -f "/tmp/user.scripts" 2>/dev/null || true
|
||||
|
||||
info "Syncing disks..."
|
||||
sync
|
||||
success "Disks synced"
|
||||
|
||||
echo ""
|
||||
echo "$ICON_REBOOT_SMART Rebooting system NOW..."
|
||||
sleep 5
|
||||
/sbin/reboot
|
||||
Reference in New Issue
Block a user