Added system watchdog. and way to many other changes
This commit is contained in:
@@ -22,7 +22,6 @@ parse_args "$@"
|
|||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||||
|
|
||||||
# ROOT CHECK
|
|
||||||
if [[ "$EUID" -ne 0 ]]; then
|
if [[ "$EUID" -ne 0 ]]; then
|
||||||
error "Must be run as root"
|
error "Must be run as root"
|
||||||
exit 1
|
exit 1
|
||||||
@@ -30,10 +29,9 @@ fi
|
|||||||
|
|
||||||
success "Running as root"
|
success "Running as root"
|
||||||
|
|
||||||
# Verify Docker is available
|
|
||||||
if ! command -v docker &>/dev/null; then
|
if ! command -v docker &>/dev/null; then
|
||||||
error "Docker command not found — check PATH or Docker installation"
|
error "Docker command not found — check PATH or Docker installation"
|
||||||
notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "alert"
|
notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "warning"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -60,9 +58,6 @@ fi
|
|||||||
# FUNCTIONS
|
# FUNCTIONS
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
# Attempts a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
|
|
||||||
# Returns 0 on success, 1 if all attempts fail.
|
|
||||||
# Usage: retry_docker docker restart Emby
|
|
||||||
retry_docker() {
|
retry_docker() {
|
||||||
local attempt=1
|
local attempt=1
|
||||||
|
|
||||||
@@ -84,7 +79,7 @@ retry_docker() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# ━━━ $ICON_CONTAINERS $ICON_STOP $ICON_START Daily Restart ━━━
|
# ━━━ $ICON_CONTAINERS Daily Restart ━━━
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_CONTAINERS Daily Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
echo "━━━ $ICON_CONTAINERS Daily Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||||
@@ -100,7 +95,6 @@ STARTED=()
|
|||||||
for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||||
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
||||||
|
|
||||||
# Verify container exists
|
|
||||||
if ! docker inspect "$container" &>/dev/null; then
|
if ! docker inspect "$container" &>/dev/null; then
|
||||||
error "$container does not exist — skipping"
|
error "$container does not exist — skipping"
|
||||||
FAILED+=("$container")
|
FAILED+=("$container")
|
||||||
@@ -121,7 +115,8 @@ for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
|||||||
echo "$ICON_STARTED $container restarted"
|
echo "$ICON_STARTED $container restarted"
|
||||||
RESTARTED+=("$container")
|
RESTARTED+=("$container")
|
||||||
else
|
else
|
||||||
error "Failed to restart $container"
|
error "Failed to restart $container after $RETRY_COUNT attempts"
|
||||||
|
notify "$container failed to restart on $(hostname)" "Docker Daily Restart" "warning"
|
||||||
FAILED+=("$container")
|
FAILED+=("$container")
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
@@ -136,7 +131,8 @@ for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
|||||||
echo "$ICON_STARTED $container started"
|
echo "$ICON_STARTED $container started"
|
||||||
STARTED+=("$container")
|
STARTED+=("$container")
|
||||||
else
|
else
|
||||||
error "Failed to start $container"
|
error "Failed to start $container after $RETRY_COUNT attempts"
|
||||||
|
notify "$container failed to start on $(hostname)" "Docker Daily Restart" "warning"
|
||||||
FAILED+=("$container")
|
FAILED+=("$container")
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
@@ -157,27 +153,19 @@ END=$(date +%s)
|
|||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
echo "━━━━━ $ICON_SUMMARY DAILY RESTART SUMMARY ━━━━━"
|
echo "━━━━━ $ICON_SUMMARY DAILY RESTART SUMMARY ━━━━━"
|
||||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||||
|
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||||
if [[ ${#RESTARTED[@]} -gt 0 ]]; then
|
[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}"
|
||||||
echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||||
fi
|
|
||||||
if [[ ${#STARTED[@]} -gt 0 ]]; then
|
|
||||||
echo "$ICON_STARTED Started: ${STARTED[*]}"
|
|
||||||
fi
|
|
||||||
if [[ ${#FAILED[@]} -gt 0 ]]; then
|
|
||||||
echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE"
|
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE"
|
||||||
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started" "Docker Daily Restart" "normal"
|
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Daily Restart" "normal"
|
||||||
else
|
else
|
||||||
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed"
|
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed"
|
||||||
notify "Daily restart completed with errors — failed: ${FAILED[*]}" "Docker Daily Restart" "alert"
|
notify "Daily restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Daily Restart" "warning"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||||
|
|||||||
@@ -2,24 +2,28 @@
|
|||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# --------------------------------- Docker Watchdog --------------------------------------------
|
# --------------------------------- Docker Watchdog --------------------------------------------
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# Self-healing watchdog for Docker containers — monitors memory, CPU and HTTP responsiveness.
|
# First line of defense — monitors Docker containers for memory, CPU, HTTP responsiveness,
|
||||||
# Restarts containers that exceed configured thresholds using a strike system for CPU and
|
# and unexpected stops. Restarts containers that exceed thresholds or go offline.
|
||||||
# responsiveness checks to avoid restarting on brief spikes.
|
#
|
||||||
|
# Works alongside system_watchdog.sh:
|
||||||
|
# docker_watchdog.sh — container level, minimal disruption, tries to self-heal
|
||||||
|
# system_watchdog.sh — system level, last resort, reboots when healing fails
|
||||||
#
|
#
|
||||||
# Behaviour:
|
# Behaviour:
|
||||||
# Memory — immediate restart if hard limit is exceeded
|
# Memory — immediate restart if hard limit exceeded
|
||||||
# CPU — strike system, restarts after CPU_FAIL_LIMIT consecutive over-threshold checks
|
# CPU — strike system, restarts after CPU_FAIL_LIMIT consecutive hits
|
||||||
# HTTP — strike system, restarts after RESP_FAIL_LIMIT consecutive failed curl checks
|
# HTTP — strike system, restarts after RESP_FAIL_LIMIT consecutive failures
|
||||||
|
# Required — strike system, restarts stopped containers, persistent skip list
|
||||||
|
# prevents reboot loops, auto-clears when container recovers
|
||||||
|
# Daemon — immediate notify if Docker daemon is unresponsive
|
||||||
#
|
#
|
||||||
# Strike system:
|
|
||||||
# Strikes persist between runs via WATCHDOG_STATE_FILE (/tmp — resets on reboot)
|
|
||||||
# Strike cadence depends on cron schedule:
|
# Strike cadence depends on cron schedule:
|
||||||
# Every 15min + 2 strikes = 30min sustained abuse before restart
|
# Every 15min + 2 strikes = 30min sustained before restart
|
||||||
# Every 10min + 2 strikes = 20min sustained abuse before restart
|
# Every 10min + 2 strikes = 20min sustained before restart
|
||||||
# Every 5min + 2 strikes = 10min sustained abuse before restart
|
# Every 5min + 2 strikes = 10min sustained before restart
|
||||||
#
|
#
|
||||||
# All configuration lives in Master.conf under the Docker Watchdog section.
|
# All configuration in Master.conf under Docker Watchdog section.
|
||||||
# Supports --dry-run to show what would be restarted without taking any action.
|
# Supports --dry-run to show what would happen without acting.
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
@@ -29,16 +33,18 @@ source "$SCRIPT_DIR/../common.sh"
|
|||||||
|
|
||||||
parse_args "$@"
|
parse_args "$@"
|
||||||
|
|
||||||
# Auto-detect total CPU cores for normalisation
|
|
||||||
TOTAL_CORES=$(nproc)
|
TOTAL_CORES=$(nproc)
|
||||||
|
|
||||||
|
# Persistent skip list — shared with system_watchdog.sh
|
||||||
|
# Containers in this list are skipped until they recover
|
||||||
|
SKIP_LIST_FILE="$SYS_WATCHDOG_FAILED_FILE"
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# ━━━ $ICON_GEAR Setup ━━━
|
# ━━━ $ICON_GEAR Setup ━━━
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||||
|
|
||||||
# ROOT CHECK
|
|
||||||
if [[ "$EUID" -ne 0 ]]; then
|
if [[ "$EUID" -ne 0 ]]; then
|
||||||
error "Must be run as root"
|
error "Must be run as root"
|
||||||
exit 1
|
exit 1
|
||||||
@@ -47,26 +53,31 @@ fi
|
|||||||
success "Running as root"
|
success "Running as root"
|
||||||
info "$ICON_WATCHDOG Watchdog initialising — $TOTAL_CORES cores detected"
|
info "$ICON_WATCHDOG Watchdog initialising — $TOTAL_CORES cores detected"
|
||||||
|
|
||||||
# Ensure state file exists
|
|
||||||
touch "$WATCHDOG_STATE_FILE" 2>/dev/null || {
|
touch "$WATCHDOG_STATE_FILE" 2>/dev/null || {
|
||||||
error "Cannot create state file: $WATCHDOG_STATE_FILE"
|
error "Cannot create state file: $WATCHDOG_STATE_FILE"
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
touch "$SKIP_LIST_FILE" 2>/dev/null || {
|
||||||
|
error "Cannot create skip list: $SKIP_LIST_FILE"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# ━━━ $ICON_SUMMARY Status ━━━
|
# ━━━ $ICON_SUMMARY Status ━━━
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
if [[ "$SHOW_STATUS" == true ]]; then
|
if [[ "$SHOW_STATUS" == true ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||||
echo "$ICON_WATCHDOG Containers monitored: ${!WATCHDOG_CONTAINERS[*]}"
|
echo "$ICON_WATCHDOG Monitored: ${!WATCHDOG_CONTAINERS[*]}"
|
||||||
echo "$ICON_MEM Soft mem threshold: ${SOFT_MEM_THRESHOLD}% of per-container limit"
|
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}"
|
||||||
echo "$ICON_ZFS CPU soft threshold: ${SOFT_CPU_THRESHOLD}%"
|
echo "$ICON_MEM Soft mem: ${SOFT_MEM_THRESHOLD}% of limit"
|
||||||
echo "$ICON_ZFS CPU hard threshold: ${HARD_CPU_THRESHOLD}%"
|
echo "$ICON_ZFS CPU soft: ${SOFT_CPU_THRESHOLD}%"
|
||||||
echo "$ICON_RETRY CPU fail limit: ${CPU_FAIL_LIMIT} strikes"
|
echo "$ICON_ZFS CPU hard: ${HARD_CPU_THRESHOLD}%"
|
||||||
echo "$ICON_PING Resp fail limit: ${RESP_FAIL_LIMIT} strikes"
|
echo "$ICON_RETRY CPU strikes: ${CPU_FAIL_LIMIT}"
|
||||||
|
echo "$ICON_RETRY Resp strikes: ${RESP_FAIL_LIMIT}"
|
||||||
echo "$ICON_TIME Curl timeout: ${CURL_TIMEOUT}s"
|
echo "$ICON_TIME Curl timeout: ${CURL_TIMEOUT}s"
|
||||||
echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
exit 0
|
exit 0
|
||||||
@@ -76,19 +87,13 @@ fi
|
|||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# STATE HELPERS
|
# STATE HELPERS
|
||||||
# Reads and writes per-container strike counts to the state file.
|
|
||||||
# State file format: container:metric:count
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
# Returns current strike count for a container/metric pair.
|
|
||||||
# Usage: get_strikes "Emby" "CPU"
|
|
||||||
get_strikes() {
|
get_strikes() {
|
||||||
local container="$1" metric="$2"
|
local container="$1" metric="$2"
|
||||||
grep -E "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f3
|
grep -E "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f3
|
||||||
}
|
}
|
||||||
|
|
||||||
# Sets strike count for a container/metric pair.
|
|
||||||
# Usage: set_strikes "Emby" "CPU" 2
|
|
||||||
set_strikes() {
|
set_strikes() {
|
||||||
local container="$1" metric="$2" count="$3"
|
local container="$1" metric="$2" count="$3"
|
||||||
grep -vE "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null > "${WATCHDOG_STATE_FILE}.tmp"
|
grep -vE "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null > "${WATCHDOG_STATE_FILE}.tmp"
|
||||||
@@ -96,19 +101,83 @@ set_strikes() {
|
|||||||
mv "${WATCHDOG_STATE_FILE}.tmp" "$WATCHDOG_STATE_FILE"
|
mv "${WATCHDOG_STATE_FILE}.tmp" "$WATCHDOG_STATE_FILE"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# SKIP LIST HELPERS
|
||||||
|
# Container skip list — persistent across reboots via /boot/
|
||||||
|
# Auto-clears entries when container is found running again.
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
is_in_skip_list() {
|
||||||
|
local container="$1"
|
||||||
|
grep -qE "^${container}$" "$SKIP_LIST_FILE" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
add_to_skip_list() {
|
||||||
|
local container="$1"
|
||||||
|
if ! is_in_skip_list "$container"; then
|
||||||
|
echo "$container" >> "$SKIP_LIST_FILE"
|
||||||
|
warn "$ICON_WATCHDOG $container added to persistent skip list"
|
||||||
|
notify "$container added to watchdog skip list on $(hostname) — manual check recommended" "Docker Watchdog" "warning"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
remove_from_skip_list() {
|
||||||
|
local container="$1"
|
||||||
|
grep -vE "^${container}$" "$SKIP_LIST_FILE" 2>/dev/null > "${SKIP_LIST_FILE}.tmp"
|
||||||
|
mv "${SKIP_LIST_FILE}.tmp" "$SKIP_LIST_FILE"
|
||||||
|
success "$ICON_WATCHDOG $container recovered — removed from skip list"
|
||||||
|
notify "$container recovered and removed from watchdog skip list on $(hostname)" "Docker Watchdog" "normal"
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# SKIP LIST AUTO-HEAL CHECK
|
||||||
|
# On every run check if any skipped containers are now running.
|
||||||
|
# If running remove from skip list — could have recovered after reboot or manual fix.
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
check_skip_list_recovery() {
|
||||||
|
[[ ! -s "$SKIP_LIST_FILE" ]] && return
|
||||||
|
|
||||||
|
info "$ICON_WATCHDOG Checking skip list for recovered containers..."
|
||||||
|
|
||||||
|
while IFS= read -r container; do
|
||||||
|
[[ -z "$container" ]] && continue
|
||||||
|
|
||||||
|
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||||
|
|
||||||
|
if [[ "$STATUS" == "true" ]]; then
|
||||||
|
remove_from_skip_list "$container"
|
||||||
|
else
|
||||||
|
log "$container still not running — remains on skip list"
|
||||||
|
fi
|
||||||
|
done < "$SKIP_LIST_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# DOCKER DAEMON HEALTH CHECK
|
||||||
|
# Verifies Docker daemon is responding before attempting any container operations.
|
||||||
|
# A hung daemon means all checks will fail — notify immediately and exit.
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
check_docker_daemon() {
|
||||||
|
info "$ICON_CONTAINERS Checking Docker daemon..."
|
||||||
|
|
||||||
|
if ! timeout 10 docker ps >/dev/null 2>&1; then
|
||||||
|
error "Docker daemon is not responding"
|
||||||
|
notify "Docker daemon unresponsive on $(hostname) — immediate attention required" "Docker Watchdog" "warning"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
success "Docker daemon is healthy"
|
||||||
|
}
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# HELPERS
|
# HELPERS
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
# Fetches memory and CPU stats for a container in a single docker stats call.
|
|
||||||
# Returns: MEM_USAGE|CPU_PERCENT
|
|
||||||
parse_stats() {
|
parse_stats() {
|
||||||
local container="$1"
|
local container="$1"
|
||||||
docker stats --no-stream --format "{{.MemUsage}}|{{.CPUPerc}}" "$container"
|
docker stats --no-stream --format "{{.MemUsage}}|{{.CPUPerc}}" "$container"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Converts a memory value and unit to MB.
|
|
||||||
# Supports KiB, MiB, GiB — returns UNKNOWN for unrecognised units.
|
|
||||||
convert_to_mb() {
|
convert_to_mb() {
|
||||||
local value="$1" unit="$2"
|
local value="$1" unit="$2"
|
||||||
case "$unit" in
|
case "$unit" in
|
||||||
@@ -119,32 +188,32 @@ convert_to_mb() {
|
|||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
# Restarts a container locally and sends a notification.
|
# Restarts a container and sends notification.
|
||||||
# In dry run mode reports what would happen without acting.
|
# Failed restarts also notify — system_watchdog.sh is the next line of defense.
|
||||||
restart_container() {
|
restart_container() {
|
||||||
local container="$1" reason="$2"
|
local container="$1" reason="$2"
|
||||||
|
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
warn "DRY RUN — would restart $container ($reason)"
|
warn "DRY RUN — would restart $container ($reason)"
|
||||||
return
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
info "Restarting $container ($reason)..."
|
info "Restarting $container ($reason)..."
|
||||||
if docker restart "$container" >/dev/null 2>&1; then
|
if docker restart "$container" >/dev/null 2>&1; then
|
||||||
echo "$ICON_STARTED $container restarted"
|
echo "$ICON_STARTED $container restarted"
|
||||||
notify "$container restarted — $reason" "Docker Watchdog" "warning"
|
notify "$container restarted on $(hostname) — $reason" "Docker Watchdog" "warning"
|
||||||
log "Restarted $container — reason: $reason"
|
return 0
|
||||||
else
|
else
|
||||||
error "Failed to restart $container"
|
error "Failed to restart $container"
|
||||||
notify "Failed to restart $container — $reason" "Docker Watchdog" "alert"
|
notify "Failed to restart $container on $(hostname) — $reason" "Docker Watchdog" "warning"
|
||||||
|
return 1
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# MEMORY CHECK
|
# MEMORY CHECK
|
||||||
# Compares current container memory usage against its configured hard limit.
|
# Restarts immediately if container exceeds hard memory limit.
|
||||||
# Restarts immediately if at or above 100% of limit.
|
# Warns if approaching soft threshold.
|
||||||
# Warns if at or above SOFT_MEM_THRESHOLD % of limit.
|
|
||||||
# Usage: check_memory "Emby" 16384
|
# Usage: check_memory "Emby" 16384
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
check_memory() {
|
check_memory() {
|
||||||
@@ -174,9 +243,8 @@ check_memory() {
|
|||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# CPU CHECK
|
# CPU CHECK
|
||||||
# Normalises CPU usage against total core count and applies the strike system.
|
# Strike system — restarts after CPU_FAIL_LIMIT consecutive over-threshold checks.
|
||||||
# Warns at SOFT_CPU_THRESHOLD, strikes at HARD_CPU_THRESHOLD.
|
# Resets strikes on recovery or restart.
|
||||||
# Restarts after CPU_FAIL_LIMIT consecutive strikes — resets strikes on restart or recovery.
|
|
||||||
# Usage: check_cpu "Emby"
|
# Usage: check_cpu "Emby"
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
check_cpu() {
|
check_cpu() {
|
||||||
@@ -193,24 +261,21 @@ check_cpu() {
|
|||||||
|
|
||||||
if (( cpu_int >= HARD_CPU_THRESHOLD )); then
|
if (( cpu_int >= HARD_CPU_THRESHOLD )); then
|
||||||
((violations++))
|
((violations++))
|
||||||
error "$ICON_ZFS $container CPU ${cpu_int}% — hard threshold hit ($violations/$CPU_FAIL_LIMIT strikes)"
|
error "$ICON_ZFS $container CPU ${cpu_int}% — hard threshold ($violations/$CPU_FAIL_LIMIT strikes)"
|
||||||
set_strikes "$container" "CPU" "$violations"
|
set_strikes "$container" "CPU" "$violations"
|
||||||
elif (( cpu_int >= SOFT_CPU_THRESHOLD )); then
|
elif (( cpu_int >= SOFT_CPU_THRESHOLD )); then
|
||||||
((violations++))
|
((violations++))
|
||||||
warn "$ICON_ZFS $container CPU ${cpu_int}% — soft threshold hit ($violations/$CPU_FAIL_LIMIT strikes)"
|
warn "$ICON_ZFS $container CPU ${cpu_int}% — soft threshold ($violations/$CPU_FAIL_LIMIT strikes)"
|
||||||
set_strikes "$container" "CPU" "$violations"
|
set_strikes "$container" "CPU" "$violations"
|
||||||
else
|
else
|
||||||
if (( violations > 0 )); then
|
[[ $violations -gt 0 ]] && info "$ICON_ZFS $container CPU ${cpu_int}% — recovered, resetting strikes"
|
||||||
info "$ICON_ZFS $container CPU ${cpu_int}% — recovered, resetting strikes"
|
[[ $violations -eq 0 ]] && success "$ICON_ZFS $container CPU ${cpu_int}%"
|
||||||
else
|
|
||||||
success "$ICON_ZFS $container CPU ${cpu_int}%"
|
|
||||||
fi
|
|
||||||
set_strikes "$container" "CPU" 0
|
set_strikes "$container" "CPU" 0
|
||||||
violations=0
|
violations=0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if (( violations >= CPU_FAIL_LIMIT )); then
|
if (( violations >= CPU_FAIL_LIMIT )); then
|
||||||
error "$ICON_ZFS $container hit CPU limit for $CPU_FAIL_LIMIT consecutive checks"
|
error "$ICON_ZFS $container CPU limit hit for $CPU_FAIL_LIMIT consecutive checks"
|
||||||
restart_container "$container" "sustained CPU abuse"
|
restart_container "$container" "sustained CPU abuse"
|
||||||
set_strikes "$container" "CPU" 0
|
set_strikes "$container" "CPU" 0
|
||||||
fi
|
fi
|
||||||
@@ -218,9 +283,8 @@ check_cpu() {
|
|||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# RESPONSIVENESS CHECK
|
# RESPONSIVENESS CHECK
|
||||||
# Sends an HTTP request to the container's configured URL.
|
# Strike system — restarts after RESP_FAIL_LIMIT consecutive failed HTTP checks.
|
||||||
# Skips containers with no URL defined in WATCHDOG_CONTAINER_URLS.
|
# Skips containers with no URL defined in WATCHDOG_CONTAINER_URLS.
|
||||||
# Applies the same strike system as CPU — restarts after RESP_FAIL_LIMIT consecutive failures.
|
|
||||||
# Usage: check_responsiveness "Emby"
|
# Usage: check_responsiveness "Emby"
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
check_responsiveness() {
|
check_responsiveness() {
|
||||||
@@ -238,11 +302,8 @@ check_responsiveness() {
|
|||||||
warn "$ICON_PING $container unresponsive at $url ($fails/$RESP_FAIL_LIMIT strikes)"
|
warn "$ICON_PING $container unresponsive at $url ($fails/$RESP_FAIL_LIMIT strikes)"
|
||||||
set_strikes "$container" "RESP" "$fails"
|
set_strikes "$container" "RESP" "$fails"
|
||||||
else
|
else
|
||||||
if (( fails > 0 )); then
|
[[ $fails -gt 0 ]] && info "$ICON_PING $container responsive again — resetting strikes"
|
||||||
info "$ICON_PING $container responsive again — resetting strikes"
|
[[ $fails -eq 0 ]] && success "$ICON_PING $container responsive at $url"
|
||||||
else
|
|
||||||
success "$ICON_PING $container responsive at $url"
|
|
||||||
fi
|
|
||||||
set_strikes "$container" "RESP" 0
|
set_strikes "$container" "RESP" 0
|
||||||
fails=0
|
fails=0
|
||||||
fi
|
fi
|
||||||
@@ -255,54 +316,131 @@ check_responsiveness() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# ━━━ $ICON_WATCHDOG Watchdog Check ━━━
|
# REQUIRED CONTAINER CHECK
|
||||||
|
# Monitors WATCHDOG_REQUIRED_CONTAINERS for unexpected stops.
|
||||||
|
# Strike system — attempts restart on each strike.
|
||||||
|
# After strike limit hit — adds to persistent skip list and notifies system_watchdog handoff.
|
||||||
|
# Skip list auto-clears at start of each run if container has recovered.
|
||||||
|
# Usage: check_required_containers
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
echo ""
|
check_required_containers() {
|
||||||
echo "━━━ $ICON_WATCHDOG Watchdog Check — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
[[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -eq 0 ]] && return
|
||||||
echo ""
|
|
||||||
|
|
||||||
SKIPPED=()
|
info "$ICON_CONTAINERS Checking required containers..."
|
||||||
RESTARTED=0
|
|
||||||
|
|
||||||
START=$(date +%s)
|
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
||||||
|
[[ -z "$container" ]] && continue
|
||||||
|
|
||||||
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
|
# Skip if on persistent skip list
|
||||||
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
if is_in_skip_list "$container"; then
|
||||||
|
warn "$ICON_NOT_RUNNING $container is on skip list — skipping until recovered"
|
||||||
# Verify container exists
|
|
||||||
if ! docker inspect "$container" &>/dev/null; then
|
|
||||||
warn "$container not found on this host — skipping"
|
|
||||||
SKIPPED+=("$container")
|
|
||||||
echo ""
|
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Verify container is running
|
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||||
if ! docker ps --filter "name=^/${container}$" --format "{{.Names}}" | grep -qw "$container"; then
|
|
||||||
warn "$ICON_NOT_RUNNING $container is not running — skipping"
|
if [[ "$STATUS" == "true" ]]; then
|
||||||
SKIPPED+=("$container")
|
success "$ICON_RUNNING $container is running"
|
||||||
|
set_strikes "$container" "STOP" 0
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$STATUS" == "unknown" ]]; then
|
||||||
|
warn "$container not found on this host — skipping"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Container is stopped — apply strike
|
||||||
|
local strikes
|
||||||
|
strikes=$(get_strikes "$container" "STOP")
|
||||||
|
[[ -z "$strikes" ]] && strikes=0
|
||||||
|
((strikes++))
|
||||||
|
|
||||||
|
warn "$ICON_NOT_RUNNING $container is stopped ($strikes/$SYS_WATCHDOG_STRIKE_LIMIT strikes)"
|
||||||
|
set_strikes "$container" "STOP" "$strikes"
|
||||||
|
|
||||||
|
# Attempt restart on each strike
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would attempt restart of $container"
|
||||||
|
else
|
||||||
|
if restart_container "$container" "unexpected stop"; then
|
||||||
|
set_strikes "$container" "STOP" 0
|
||||||
|
else
|
||||||
|
# Restart failed
|
||||||
|
if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then
|
||||||
|
error "$container failed to restart after $SYS_WATCHDOG_STRIKE_LIMIT attempts"
|
||||||
|
add_to_skip_list "$container"
|
||||||
|
set_strikes "$container" "STOP" 0
|
||||||
|
notify "$container handed off to system_watchdog on $(hostname) — added to skip list" "Docker Watchdog" "warning"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_WATCHDOG Watchdog Run ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_WATCHDOG Watchdog Run — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
START=$(date +%s)
|
||||||
|
SKIPPED=()
|
||||||
|
|
||||||
|
# Daemon check first — if daemon is down nothing else works
|
||||||
|
check_docker_daemon
|
||||||
|
|
||||||
|
# Auto-heal skip list before processing
|
||||||
|
check_skip_list_recovery
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# Resource monitoring — WATCHDOG_CONTAINERS
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
|
||||||
echo ""
|
echo ""
|
||||||
|
echo "━━━ $ICON_MEM Resource Monitoring ━━━"
|
||||||
|
|
||||||
|
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
|
||||||
|
echo ""
|
||||||
|
info "$ICON_CONTAINERS $container"
|
||||||
|
|
||||||
|
if ! docker inspect "$container" &>/dev/null; then
|
||||||
|
warn "$container not found — skipping"
|
||||||
|
SKIPPED+=("$container")
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! docker ps --filter "name=^/${container}$" --format "{{.Names}}" | grep -qw "$container"; then
|
||||||
|
warn "$ICON_NOT_RUNNING $container is not running — skipping resource checks"
|
||||||
|
SKIPPED+=("$container")
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
check_memory "$container" "${WATCHDOG_CONTAINERS[$container]}"
|
check_memory "$container" "${WATCHDOG_CONTAINERS[$container]}"
|
||||||
check_cpu "$container"
|
check_cpu "$container"
|
||||||
check_responsiveness "$container"
|
check_responsiveness "$container"
|
||||||
|
done
|
||||||
|
fi
|
||||||
|
|
||||||
echo ""
|
# -----------------------------------------------------------------------------------------------
|
||||||
done
|
# Required container monitoring — WATCHDOG_REQUIRED_CONTAINERS
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_CONTAINERS Required Container Check ━━━"
|
||||||
|
check_required_containers
|
||||||
|
|
||||||
END=$(date +%s)
|
END=$(date +%s)
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo ""
|
||||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG SUMMARY ━━━━━"
|
echo "━━━━━ $ICON_SUMMARY WATCHDOG SUMMARY ━━━━━"
|
||||||
echo "$ICON_TIME $(date '+%Y-%m-%d %H:%M:%S')"
|
echo "$ICON_TIME $(date '+%Y-%m-%d %H:%M:%S')"
|
||||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||||
echo "$ICON_WATCHDOG Monitored: ${#WATCHDOG_CONTAINERS[@]} containers"
|
echo "$ICON_WATCHDOG Monitored: ${#WATCHDOG_CONTAINERS[@]} containers"
|
||||||
echo "$ICON_NOT_RUNNING Skipped: ${#SKIPPED[@]} containers"
|
echo "$ICON_CONTAINERS Required: ${#WATCHDOG_REQUIRED_CONTAINERS[@]} containers"
|
||||||
if [[ "$DRY_RUN" == true ]]; then
|
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]}"
|
||||||
echo "$ICON_WARN Dry Run: no restarts executed"
|
[[ "$DRY_RUN" == true ]] && echo "$ICON_WARN Dry Run: no actions taken"
|
||||||
fi
|
|
||||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
@@ -0,0 +1,174 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# --------------------------------- Docker Weekly Restart --------------------------------------
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# Restarts or starts specified Docker containers with retry logic.
|
||||||
|
# Containers are configured in Master.conf under WEEKLY_RESTART_CONTAINERS.
|
||||||
|
# Uses global RETRY_COUNT and SLEEP from Master.conf for retry behaviour.
|
||||||
|
# Sends notifications on completion or failure via common.sh notify().
|
||||||
|
# Supports --dry-run to preview what would be restarted without taking action.
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
|
source "$SCRIPT_DIR/../Master.conf"
|
||||||
|
source "$SCRIPT_DIR/../common.sh"
|
||||||
|
|
||||||
|
parse_args "$@"
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_GEAR Setup ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||||
|
|
||||||
|
if [[ "$EUID" -ne 0 ]]; then
|
||||||
|
error "Must be run as root"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
success "Running as root"
|
||||||
|
|
||||||
|
if ! command -v docker &>/dev/null; then
|
||||||
|
error "Docker command not found — check PATH or Docker installation"
|
||||||
|
notify "Docker weekly restart failed — Docker not found on $(hostname)" "Docker Weekly Restart" "warning"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
success "Docker found"
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_SUMMARY Status ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
if [[ "$SHOW_STATUS" == true ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||||
|
echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
|
||||||
|
echo "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||||
|
echo "$ICON_TIME Sleep: ${SLEEP}s between retries"
|
||||||
|
echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||||
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# FUNCTIONS
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Attempts a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
|
||||||
|
# Returns 0 on success, 1 if all attempts fail.
|
||||||
|
retry_docker() {
|
||||||
|
local attempt=1
|
||||||
|
|
||||||
|
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
|
||||||
|
info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
||||||
|
|
||||||
|
if "$@"; then
|
||||||
|
success "Succeeded on attempt $attempt"
|
||||||
|
return 0
|
||||||
|
else
|
||||||
|
warn "Attempt $attempt failed"
|
||||||
|
(( attempt++ ))
|
||||||
|
[[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
error "Command failed after $RETRY_COUNT attempts: $*"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_CONTAINERS Weekly Restart ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_CONTAINERS Weekly Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||||
|
echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
|
||||||
|
echo "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
START=$(date +%s)
|
||||||
|
FAILED=()
|
||||||
|
RESTARTED=()
|
||||||
|
STARTED=()
|
||||||
|
|
||||||
|
for container in "${WEEKLY_RESTART_CONTAINERS[@]}"; do
|
||||||
|
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
||||||
|
|
||||||
|
if ! docker inspect "$container" &>/dev/null; then
|
||||||
|
error "$container does not exist — skipping"
|
||||||
|
FAILED+=("$container")
|
||||||
|
echo ""
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||||
|
|
||||||
|
case "$STATUS" in
|
||||||
|
true)
|
||||||
|
echo "$ICON_RUNNING $container is running — restarting..."
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would restart $container"
|
||||||
|
else
|
||||||
|
if retry_docker docker restart "$container"; then
|
||||||
|
echo "$ICON_STARTED $container restarted"
|
||||||
|
RESTARTED+=("$container")
|
||||||
|
else
|
||||||
|
error "Failed to restart $container after $RETRY_COUNT attempts"
|
||||||
|
notify "$container failed to restart on $(hostname)" "Docker Weekly Restart" "warning"
|
||||||
|
FAILED+=("$container")
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
false)
|
||||||
|
echo "$ICON_NOT_RUNNING $container is stopped — starting..."
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would start $container"
|
||||||
|
else
|
||||||
|
if retry_docker docker start "$container"; then
|
||||||
|
echo "$ICON_STARTED $container started"
|
||||||
|
STARTED+=("$container")
|
||||||
|
else
|
||||||
|
error "Failed to start $container after $RETRY_COUNT attempts"
|
||||||
|
notify "$container failed to start on $(hostname)" "Docker Weekly Restart" "warning"
|
||||||
|
FAILED+=("$container")
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
error "Unknown status for $container: $STATUS"
|
||||||
|
FAILED+=("$container")
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
done
|
||||||
|
|
||||||
|
END=$(date +%s)
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo "━━━━━ $ICON_SUMMARY WEEKLY RESTART SUMMARY ━━━━━"
|
||||||
|
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||||
|
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||||
|
[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}"
|
||||||
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||||
|
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||||
|
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE"
|
||||||
|
notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Weekly Restart" "normal"
|
||||||
|
else
|
||||||
|
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed"
|
||||||
|
notify "Weekly restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Weekly Restart" "warning"
|
||||||
|
fi
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
|
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||||
|
exit 0
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# --------------------------------- Media Permissions Script -----------------------------------
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# Applies permissions and ownership to all configured media shares.
|
||||||
|
# Shares, permissions mode and owner are configured in Master.conf.
|
||||||
|
# Supports --dry-run to preview what would be changed without making changes.
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
|
source "$SCRIPT_DIR/../Master.conf"
|
||||||
|
source "$SCRIPT_DIR/../common.sh"
|
||||||
|
|
||||||
|
parse_args "$@"
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_GEAR Setup ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||||
|
|
||||||
|
if [[ "$EUID" -ne 0 ]]; then
|
||||||
|
error "Must be run as root"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
success "Running as root"
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_SUMMARY Status ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
if [[ "$SHOW_STATUS" == true ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||||
|
echo "$ICON_PERMS Mode: $PERMISSIONS_MODE"
|
||||||
|
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||||
|
echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}"
|
||||||
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_PERMS Media Permissions ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_PERMS Media Permissions ━━━"
|
||||||
|
echo "$ICON_PERMS Mode: $PERMISSIONS_MODE"
|
||||||
|
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||||
|
echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
START=$(date +%s)
|
||||||
|
FAILED=()
|
||||||
|
UPDATED=()
|
||||||
|
SKIPPED=()
|
||||||
|
|
||||||
|
for SHARE in "${MEDIA_PERMISSION_SHARES[@]}"; do
|
||||||
|
SHARE_NAME=$(basename "$SHARE")
|
||||||
|
|
||||||
|
if [[ ! -d "$SHARE" ]]; then
|
||||||
|
warn "$ICON_PERMS $SHARE_NAME not found — skipping"
|
||||||
|
SKIPPED+=("$SHARE_NAME")
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would apply $PERMISSIONS_MODE $PERMISSIONS_OWNER to $SHARE"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
info "$ICON_PERMS Updating $SHARE_NAME..."
|
||||||
|
|
||||||
|
CHMOD_OK=true
|
||||||
|
CHOWN_OK=true
|
||||||
|
|
||||||
|
chmod -R "$PERMISSIONS_MODE" "$SHARE" 2>/dev/null || CHMOD_OK=false
|
||||||
|
chown -R "$PERMISSIONS_OWNER" "$SHARE" 2>/dev/null || CHOWN_OK=false
|
||||||
|
|
||||||
|
if [[ "$CHMOD_OK" == true && "$CHOWN_OK" == true ]]; then
|
||||||
|
echo "$ICON_UNLOCKED $SHARE_NAME — permissions applied"
|
||||||
|
UPDATED+=("$SHARE_NAME")
|
||||||
|
else
|
||||||
|
error "$SHARE_NAME — permissions failed (chmod=$CHMOD_OK chown=$CHOWN_OK)"
|
||||||
|
FAILED+=("$SHARE_NAME")
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
END=$(date +%s)
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SUMMARY MEDIA PERMISSIONS SUMMARY ━━━━━"
|
||||||
|
echo "$ICON_PERMS Mode: $PERMISSIONS_MODE"
|
||||||
|
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||||
|
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||||
|
echo ""
|
||||||
|
[[ ${#UPDATED[@]} -gt 0 ]] && echo " $ICON_UNLOCKED Updated: ${#UPDATED[@]}"
|
||||||
|
[[ ${#SKIPPED[@]} -gt 0 ]] && echo " $ICON_WARN Skipped: ${#SKIPPED[@]}"
|
||||||
|
[[ ${#FAILED[@]} -gt 0 ]] && echo " $ICON_ERROR Failed: ${#FAILED[@]} — ${FAILED[*]}"
|
||||||
|
echo ""
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||||
|
elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||||
|
echo "$ICON_ERROR Status: $ICON_ERROR SOME SHARES FAILED"
|
||||||
|
notify "Media permissions failed on $(hostname) — ${FAILED[*]}" "Media Permissions" "warning"
|
||||||
|
else
|
||||||
|
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||||
|
notify "Media permissions applied on $(hostname) — ${#UPDATED[@]} shares updated" "Media Permissions" "normal"
|
||||||
|
fi
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
|
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||||
|
exit 0
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# --------------------------------- Media Cleaner Script ---------------------------------------
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# Removes unwanted files from media share folders using configurable file patterns.
|
||||||
|
# Supports two profiles: anime and media — each with their own folder list and file patterns.
|
||||||
|
# Profiles and patterns are configured in Master.conf.
|
||||||
|
# Supports --dry-run to preview what would be deleted without making changes.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# media_cleaner.sh anime — clean anime shares
|
||||||
|
# media_cleaner.sh media — clean media shares
|
||||||
|
# media_cleaner.sh anime --dry-run — preview anime clean
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
|
source "$SCRIPT_DIR/../Master.conf"
|
||||||
|
source "$SCRIPT_DIR/../common.sh"
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# Separate profile argument from flags
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
PROFILE=""
|
||||||
|
RAW_ARGS=()
|
||||||
|
|
||||||
|
for ARG in "$@"; do
|
||||||
|
case "$ARG" in
|
||||||
|
--*|*=*) RAW_ARGS+=("$ARG") ;;
|
||||||
|
anime|media) PROFILE="$ARG" ;;
|
||||||
|
*) RAW_ARGS+=("$ARG") ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
parse_args "${RAW_ARGS[@]}"
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_GEAR Setup ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||||
|
|
||||||
|
if [[ "$EUID" -ne 0 ]]; then
|
||||||
|
error "Must be run as root"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
success "Running as root"
|
||||||
|
|
||||||
|
if ! command -v find >/dev/null 2>&1; then
|
||||||
|
error "find command not found — check findutils installation"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z "$PROFILE" ]]; then
|
||||||
|
error "No profile specified. Usage: media_cleaner.sh <anime|media> [--dry-run]"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Resolve profile folders and patterns
|
||||||
|
case "$PROFILE" in
|
||||||
|
anime)
|
||||||
|
CLEAN_FOLDERS=("${ANIME_CLEAN_FOLDERS[@]}")
|
||||||
|
FILE_PATTERNS=("${ANIME_FILE_PATTERNS[@]}")
|
||||||
|
;;
|
||||||
|
media)
|
||||||
|
CLEAN_FOLDERS=("${MEDIA_CLEAN_FOLDERS[@]}")
|
||||||
|
FILE_PATTERNS=("${MEDIA_FILE_PATTERNS[@]}")
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
error "Unknown profile: $PROFILE — must be anime or media"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
info "$ICON_GEAR Profile: $PROFILE"
|
||||||
|
info "$ICON_CLEAN Folders: ${#CLEAN_FOLDERS[@]}"
|
||||||
|
info "$ICON_TRASH Patterns: ${#FILE_PATTERNS[@]}"
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_SUMMARY Status ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
if [[ "$SHOW_STATUS" == true ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||||
|
echo "$ICON_GEAR Profile: $PROFILE"
|
||||||
|
echo "$ICON_CLEAN Folders: ${CLEAN_FOLDERS[*]}"
|
||||||
|
echo "$ICON_TRASH Patterns: ${FILE_PATTERNS[*]}"
|
||||||
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_CLEAN Media Cleaner ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_CLEAN Media Cleaner — $PROFILE ━━━"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
START=$(date +%s)
|
||||||
|
TOTAL_REMOVED=0
|
||||||
|
FAILED=()
|
||||||
|
SKIPPED=()
|
||||||
|
|
||||||
|
for FOLDER in "${CLEAN_FOLDERS[@]}"; do
|
||||||
|
FOLDER_NAME=$(basename "$FOLDER")
|
||||||
|
echo "━━━ $ICON_CLEAN $FOLDER_NAME ━━━"
|
||||||
|
|
||||||
|
if [[ ! -d "$FOLDER" ]]; then
|
||||||
|
warn "$FOLDER_NAME not found — skipping"
|
||||||
|
SKIPPED+=("$FOLDER_NAME")
|
||||||
|
echo ""
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Build find command dynamically from FILE_PATTERNS array
|
||||||
|
CMD=(find "$FOLDER" -type f \()
|
||||||
|
for ((i = 0; i < ${#FILE_PATTERNS[@]}; i++)); do
|
||||||
|
CMD+=(-iname "${FILE_PATTERNS[i]}")
|
||||||
|
if [[ $i -lt $(( ${#FILE_PATTERNS[@]} - 1 )) ]]; then
|
||||||
|
CMD+=(-o)
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
CMD+=(\))
|
||||||
|
|
||||||
|
# Count matching files before acting
|
||||||
|
FILE_COUNT=$("${CMD[@]}" 2>/dev/null | wc -l)
|
||||||
|
|
||||||
|
if [[ "$FILE_COUNT" -eq 0 ]]; then
|
||||||
|
success "$FOLDER_NAME — no matching files found"
|
||||||
|
echo ""
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
info "$ICON_TRASH $FILE_COUNT file(s) found in $FOLDER_NAME"
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — files that would be deleted:"
|
||||||
|
"${CMD[@]}" 2>/dev/null | while IFS= read -r f; do
|
||||||
|
echo " $ICON_TRASH $f"
|
||||||
|
done
|
||||||
|
else
|
||||||
|
CLEAN_CMD=("${CMD[@]}" -exec rm -f {} +)
|
||||||
|
if "${CLEAN_CMD[@]}" 2>/dev/null; then
|
||||||
|
success "$FOLDER_NAME — $FILE_COUNT file(s) removed"
|
||||||
|
TOTAL_REMOVED=$((TOTAL_REMOVED + FILE_COUNT))
|
||||||
|
else
|
||||||
|
error "$FOLDER_NAME — cleanup failed"
|
||||||
|
FAILED+=("$FOLDER_NAME")
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
done
|
||||||
|
|
||||||
|
END=$(date +%s)
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo "━━━━━ $ICON_SUMMARY MEDIA CLEANER SUMMARY ━━━━━"
|
||||||
|
echo "$ICON_GEAR Profile: $PROFILE"
|
||||||
|
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||||
|
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_WARN Skipped: ${SKIPPED[*]}"
|
||||||
|
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
echo "$ICON_WARN Status: DRY RUN — no files deleted"
|
||||||
|
elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||||
|
echo "$ICON_ERROR Status: $ICON_ERROR SOME FOLDERS FAILED"
|
||||||
|
notify "Media cleaner ($PROFILE) failed on $(hostname) — ${FAILED[*]}" "Media Cleaner" "warning"
|
||||||
|
else
|
||||||
|
echo "$ICON_TRASH Removed: $TOTAL_REMOVED file(s)"
|
||||||
|
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||||
|
notify "Media cleaner ($PROFILE) complete on $(hostname) — $TOTAL_REMOVED file(s) removed" "Media Cleaner" "normal"
|
||||||
|
fi
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
|
||||||
|
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||||
|
exit 0
|
||||||
+302
-104
@@ -1,63 +1,96 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# ----------------------------------------------------------------------------------------------
|
# ==============================================================================================
|
||||||
# ---------------------------- Master Variables for unRAID scripts -----------------------------
|
# ================================= MASTER CONFIGURATION =======================================
|
||||||
# ----------------------------------------------------------------------------------------------
|
# ==============================================================================================
|
||||||
|
# All user-facing variables for the unRAID script ecosystem.
|
||||||
|
# Scripts source this file — edit here, changes apply everywhere on next git pull.
|
||||||
|
#
|
||||||
|
# ── INDEX ─────────────────────────────────────────────────────────────────────────────────────
|
||||||
|
#
|
||||||
|
# Section Description
|
||||||
|
# ───────────────────────────────────────────────────────────────────────────────────────────
|
||||||
|
# HOST CONFIGURATION Server hostnames and SSH key paths
|
||||||
|
# LOGGING Enable or disable verbose logging
|
||||||
|
# NOTIFICATIONS unRAID native and Discord webhook settings
|
||||||
|
# GIT / REPO Gitea repository and SSH settings
|
||||||
|
#
|
||||||
|
# ── RSYNC ──────────────────────────────────────────────────────────────────────────────────
|
||||||
|
# RSYNC DEFAULTS Global fallback rsync settings
|
||||||
|
# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort
|
||||||
|
# DAILY SYNC SHARES Media shares synced by daily_sync.sh
|
||||||
|
# RSYNC PROFILE SYSTEM Per-profile overrides (appdata profiles)
|
||||||
|
#
|
||||||
|
# ── DOCKER ESSENTIALS ──────────────────────────────────────────────────────────────────────
|
||||||
|
# DOCKER DAILY RESTART Containers restarted daily
|
||||||
|
# DOCKER WEEKLY RESTART Containers restarted weekly
|
||||||
|
# DOCKER WATCHDOG Container health monitoring — memory, CPU, HTTP
|
||||||
|
#
|
||||||
|
# ── UNRAID ESSENTIALS ──────────────────────────────────────────────────────────────────────
|
||||||
|
# REBOOT User warning delay before scheduled reboot
|
||||||
|
# MOVER Mover stop timeout
|
||||||
|
# SYSLOG FILTER Docker veth noise filter file path
|
||||||
|
# PHP-FPM PHP-FPM max children config
|
||||||
|
# CLEAR LOGS System log file paths
|
||||||
|
#
|
||||||
|
# ── MEDIA ──────────────────────────────────────────────────────────────────────────────────
|
||||||
|
# MEDIA PERMISSIONS Share list, mode and owner for permissions script
|
||||||
|
# MEDIA CLEANER Anime and media folder lists and file patterns
|
||||||
|
#
|
||||||
|
# ── SYSTEM WATCHDOG ────────────────────────────────────────────────────────────────────────
|
||||||
|
# SYSTEM WATCHDOG System health monitoring — last line of defense
|
||||||
|
#
|
||||||
|
# ==============================================================================================
|
||||||
|
|
||||||
# ━━━ Host Configuration ━━━
|
# ━━━ Host Configuration ━━━
|
||||||
# List the Hostnames of both servers
|
# Hostnames must match Tailscale machine names exactly — case sensitive
|
||||||
HOST1="unRAID-Gmer4Lfe"
|
HOST1="unRAID-Gmer4Lfe"
|
||||||
HOST2="unRAID-Jayred365"
|
HOST2="unRAID-Jayred365"
|
||||||
# Assign SSH keys for each host pair (adjust paths as needed) HOST1 from above
|
|
||||||
# Must have its key added to HOST1_SSH_Key same applies for HOST2
|
# SSH keys for server-to-server rsync — each server needs the other's key authorised
|
||||||
HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
|
HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key"
|
||||||
HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
|
HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key"
|
||||||
|
|
||||||
# ━━━ Logging ━━━
|
# ━━━ Logging ━━━
|
||||||
ENABLE_LOGGING=true # false = only echo user-facing messages
|
# true = verbose [LOG] output in scripts / false = user-facing output only
|
||||||
|
ENABLE_LOGGING=true
|
||||||
|
|
||||||
# ━━━ Notifications ━━━
|
# ━━━ Notifications ━━━
|
||||||
# unRAID native notification system — set to true to enable
|
# unRAID native — configure Settings → Notification Settings for errors/warnings only
|
||||||
# Configure unRAID to send errors only: Settings → Notification Settings
|
|
||||||
NOTIFY_UNRAID=true
|
NOTIFY_UNRAID=true
|
||||||
# Discord webhook URL — leave blank to disable
|
# Discord webhook URL — leave blank to disable
|
||||||
DISCORD_WEBHOOK=""
|
DISCORD_WEBHOOK=""
|
||||||
|
|
||||||
# ━━━ Git, Pull & Execute Script ━━━
|
# ━━━ Git / Repo ━━━
|
||||||
REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git"
|
REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git"
|
||||||
TARGET_DIR="/mnt/user/appdata/unraid_scripts"
|
TARGET_DIR="/mnt/user/appdata/unraid_scripts"
|
||||||
GITEA_SSH_KEY="/root/.ssh/id_gitea_rsync"
|
GITEA_SSH_KEY="/root/.ssh/id_gitea_rsync"
|
||||||
SSH_PORT=221
|
SSH_PORT=221
|
||||||
|
|
||||||
# ━━━ Rsync Script Defaults ━━━
|
# ==============================================================================================
|
||||||
# These are the fallback values used when no matching profile is found.
|
# ── RSYNC ─────────────────────────────────────────────────────────────────────────────────────
|
||||||
# Any share whose directory basename does not match a profile key below
|
# ==============================================================================================
|
||||||
# will use these globals for all rsync behaviour.
|
|
||||||
BW_LIMIT=12500
|
# ━━━ Rsync Defaults ━━━
|
||||||
# Retry logic
|
# Global fallback values — used when no profile match is found for a share.
|
||||||
RETRY_COUNT=3
|
# Shares in DAILY_SYNC_SHARES always use these globals (no profile defined).
|
||||||
# Sleep between retries (seconds)
|
BW_LIMIT=12500 # network speed limit KB/s
|
||||||
SLEEP=300
|
RETRY_COUNT=3 # number of retry attempts on failure
|
||||||
# Container start/stop/restart before and after rsync
|
SLEEP=300 # seconds between retries
|
||||||
CRITICAL_CONTAINER_NAMES=()
|
CRITICAL_CONTAINER_NAMES=() # containers to stop before rsync
|
||||||
# Containers that need delayed before starting
|
DELAYED_CONTAINERS=() # containers needing delay before start
|
||||||
DELAYED_CONTAINERS=()
|
CONTAINER_DELAY=5 # seconds delay before starting delayed containers
|
||||||
# Delay in seconds between starting containers, useful for things like Authelia
|
EXCLUDE_DIRS=() # directories to exclude from transfer
|
||||||
CONTAINER_DELAY=5
|
|
||||||
# Directories to exclude during transfer
|
|
||||||
EXCLUDE_DIRS=()
|
|
||||||
# Default global rsync options
|
|
||||||
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)
|
DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file)
|
||||||
|
|
||||||
# ━━━ Remote Health Checks ━━━
|
# ━━━ Remote Health Checks ━━━
|
||||||
# Abort if remote rootfs usage is at or above this percentage.
|
# Abort rsync if remote rootfs exceeds this percentage.
|
||||||
# Protects against rsync writing to rootfs when the remote array is down or drives are missing.
|
# Protects against rsync filling rootfs when remote array is down or drives are missing.
|
||||||
# Recommended: 75 — gives headroom before the server becomes unstable
|
|
||||||
ROOTFS_WARN=75
|
ROOTFS_WARN=75
|
||||||
|
|
||||||
# ━━━ Daily Sync Shares ━━━
|
# ━━━ Daily Sync Shares ━━━
|
||||||
# Shares synced once daily by Orchestrators/daily_sync.sh
|
# Shares synced once daily by Orchestrators/daily_sync.sh
|
||||||
|
# No profile needed — all fall through to DEFAULT_RSYNC_OPTS above.
|
||||||
# Add or remove paths here to manage what gets synced.
|
# Add or remove paths here to manage what gets synced.
|
||||||
# These shares have no profile entry and fall through to DEFAULT_RSYNC_OPTS above.
|
|
||||||
DAILY_SYNC_SHARES=(
|
DAILY_SYNC_SHARES=(
|
||||||
/mnt/user/Anime_Movies-Old
|
/mnt/user/Anime_Movies-Old
|
||||||
/mnt/user/Anime_Shows-Old
|
/mnt/user/Anime_Shows-Old
|
||||||
@@ -73,81 +106,22 @@ DAILY_SYNC_SHARES=(
|
|||||||
/mnt/user/Tv_Shows
|
/mnt/user/Tv_Shows
|
||||||
)
|
)
|
||||||
|
|
||||||
# ━━━ unRAID Essential Scripts ━━━
|
# ━━━ Rsync Profile System ━━━
|
||||||
# unRAID reboot script user warning time (seconds)
|
|
||||||
REBOOT_SLEEP=300
|
|
||||||
# unRAID mover stop script timeout (seconds)
|
|
||||||
MOVER_STOP_TIMEOUT=300
|
|
||||||
# docker_syslog_filter.sh filter file path
|
|
||||||
FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
|
|
||||||
# php_fpm_max_children.sh php-fpm config file path
|
|
||||||
PHP_CONF="/etc/php-fpm.d/www.conf"
|
|
||||||
# php_fpm_max_children.sh max children value
|
|
||||||
PHP_MAX_CHILDREN=250
|
|
||||||
# clear_logs.sh system log file paths
|
|
||||||
LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
|
|
||||||
|
|
||||||
# ━━━ Docker Daily Restart ━━━
|
|
||||||
# Containers to restart daily — space-separated, case-sensitive
|
|
||||||
# These are the same containers as critical-data and other rsync profiles
|
|
||||||
DAILY_RESTART_CONTAINERS=(
|
|
||||||
"NginxProxyManager"
|
|
||||||
"Authelia"
|
|
||||||
"Dispatcharr-Iptv-Users"
|
|
||||||
"Dispatcharr"
|
|
||||||
"Dispatcharr-Basic"
|
|
||||||
"Code-Server"
|
|
||||||
)
|
|
||||||
|
|
||||||
# ━━━ Docker Watchdog ━━━
|
|
||||||
# Containers to monitor with their memory hard limits in MB
|
|
||||||
# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240 8GB=8192 6GB=6144 4GB=4096 1GB=1024
|
|
||||||
declare -A WATCHDOG_CONTAINERS=(
|
|
||||||
["Emby"]=16384
|
|
||||||
["jellyfin_with_request"]=12288
|
|
||||||
["LidaTube"]=6144
|
|
||||||
["Tdarr"]=6144
|
|
||||||
["Code-Server"]=1024
|
|
||||||
)
|
|
||||||
|
|
||||||
# Containers to check for HTTP responsiveness — omit a container to skip its check
|
|
||||||
declare -A WATCHDOG_CONTAINER_URLS=(
|
|
||||||
["Emby"]="http://localhost:8096"
|
|
||||||
["Jellyfin-Gmer4Lfe"]="http://localhost:8095"
|
|
||||||
)
|
|
||||||
|
|
||||||
# State file for tracking CPU and responsiveness strikes between runs
|
|
||||||
# Lives in /tmp — resets on reboot which is correct behaviour for strike tracking
|
|
||||||
WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
|
|
||||||
|
|
||||||
# CPU thresholds — normalised against total core count automatically at runtime
|
|
||||||
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
|
|
||||||
HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU
|
|
||||||
CPU_FAIL_LIMIT=2 # consecutive strikes before restart
|
|
||||||
|
|
||||||
# Memory threshold
|
|
||||||
SOFT_MEM_THRESHOLD=80 # warn at this % of per-container hard limit
|
|
||||||
|
|
||||||
# Responsiveness check settings
|
|
||||||
RESP_FAIL_LIMIT=2 # consecutive failures before restart
|
|
||||||
CURL_TIMEOUT=5 # seconds before curl gives up
|
|
||||||
|
|
||||||
# ━━━ Profile System ━━━
|
|
||||||
# Profiles are matched by directory basename (lowercased).
|
# Profiles are matched by directory basename (lowercased).
|
||||||
# Example: /mnt/user/appdata-Failover/Arrs_Stack → profile key = arrs_stack
|
# Example: /mnt/user/appdata-Failover/Arrs_Stack → profile key = arrs_stack
|
||||||
#
|
#
|
||||||
# How fallthrough works:
|
# How fallthrough works:
|
||||||
# - If a key exists in a profile array, that value is used
|
# - If a key exists in a profile array that value is used
|
||||||
# - If a key is missing, the global default above is used instead
|
# - If a key is missing the global default above is used instead
|
||||||
# - Shares in DAILY_SYNC_SHARES have no profile and always use globals
|
# - Shares in DAILY_SYNC_SHARES have no profile and always use globals
|
||||||
#
|
#
|
||||||
# To add a new profile:
|
# To add a new profile:
|
||||||
# 1. Add a key to each array below with your chosen profile name
|
# 1. Add a key to each array below with your chosen profile name
|
||||||
# 2. Call rsync.sh with a directory whose basename matches that key
|
# 2. Call rsync.sh with a directory whose basename matches that key
|
||||||
# 3. Any array you omit will fall back to its global default
|
# 3. Any array you omit falls back to its global default
|
||||||
#
|
#
|
||||||
# Note: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS —
|
# Note: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS —
|
||||||
# if you define it for a profile you must list all desired options explicitly
|
# list all desired options explicitly if you define a profile entry
|
||||||
|
|
||||||
# SPACE-SEPARATED STRINGS
|
# SPACE-SEPARATED STRINGS
|
||||||
declare -A PROFILE_RSYNC_OPTS=(
|
declare -A PROFILE_RSYNC_OPTS=(
|
||||||
@@ -191,7 +165,7 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=(
|
|||||||
[emby]=""
|
[emby]=""
|
||||||
)
|
)
|
||||||
|
|
||||||
# SPACE-SEPARATED STRINGS — containers that need a delay before starting
|
# SPACE-SEPARATED STRINGS — containers needing delay before starting
|
||||||
declare -A PROFILE_DELAYED_CONTAINERS=(
|
declare -A PROFILE_DELAYED_CONTAINERS=(
|
||||||
[arrs_stack]=""
|
[arrs_stack]=""
|
||||||
[critical-data]="Authelia"
|
[critical-data]="Authelia"
|
||||||
@@ -216,6 +190,230 @@ declare -A PROFILE_EXCLUDE_DIRS=(
|
|||||||
[important-data]="logs *.tmp"
|
[important-data]="logs *.tmp"
|
||||||
[emby]="logs *.tmp"
|
[emby]="logs *.tmp"
|
||||||
)
|
)
|
||||||
# ----------------------------------------------------------------------------------------------
|
|
||||||
# ---------------------- End Of User Variables, Please adjust above as needed ------------------
|
# ==============================================================================================
|
||||||
# ----------------------------------------------------------------------------------------------
|
# ── DOCKER ESSENTIALS ─────────────────────────────────────────────────────────────────────────
|
||||||
|
# ==============================================================================================
|
||||||
|
|
||||||
|
# ━━━ Docker Daily Restart ━━━
|
||||||
|
# Containers restarted every day — case-sensitive names
|
||||||
|
DAILY_RESTART_CONTAINERS=(
|
||||||
|
"NginxProxyManager"
|
||||||
|
"Authelia"
|
||||||
|
"Dispatcharr-Iptv-Users"
|
||||||
|
"Dispatcharr"
|
||||||
|
"Dispatcharr-Basic"
|
||||||
|
"Code-Server"
|
||||||
|
)
|
||||||
|
|
||||||
|
# ━━━ Docker Weekly Restart ━━━
|
||||||
|
# Containers restarted once per week — case-sensitive names
|
||||||
|
WEEKLY_RESTART_CONTAINERS=(
|
||||||
|
"NginxProxyManager"
|
||||||
|
"Authelia"
|
||||||
|
"Dispatcharr-Iptv-Users"
|
||||||
|
"Dispatcharr"
|
||||||
|
"Dispatcharr-Basic"
|
||||||
|
"Code-Server"
|
||||||
|
)
|
||||||
|
|
||||||
|
# ━━━ Docker Watchdog ━━━
|
||||||
|
# First line of defense — monitors and restarts unhealthy containers.
|
||||||
|
# Runs on a cron schedule (recommended every 15 minutes).
|
||||||
|
# Strike system prevents restarts on brief spikes.
|
||||||
|
|
||||||
|
# Containers to monitor with memory hard limits in MB
|
||||||
|
# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240
|
||||||
|
# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024
|
||||||
|
declare -A WATCHDOG_CONTAINERS=(
|
||||||
|
["Emby"]=16384
|
||||||
|
["LidaTube"]=6144
|
||||||
|
["Tdarr"]=6144
|
||||||
|
["Code-Server"]=1024
|
||||||
|
)
|
||||||
|
|
||||||
|
# Containers to check HTTP responsiveness — omit to skip
|
||||||
|
declare -A WATCHDOG_CONTAINER_URLS=(
|
||||||
|
["Emby"]="http://localhost:8096"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Containers that should always be running — monitored for unexpected stops
|
||||||
|
# Strike system used — persistent skip list prevents reboot loops
|
||||||
|
WATCHDOG_REQUIRED_CONTAINERS=(
|
||||||
|
"NginxProxyManager"
|
||||||
|
"Authelia"
|
||||||
|
"Emby"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Strike state file — /tmp resets on reboot, correct for strike tracking
|
||||||
|
WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db"
|
||||||
|
|
||||||
|
# CPU thresholds — normalised against total core count at runtime
|
||||||
|
SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU
|
||||||
|
HARD_CPU_THRESHOLD=90 # strike at this % of total system CPU
|
||||||
|
CPU_FAIL_LIMIT=2 # consecutive strikes before restart
|
||||||
|
|
||||||
|
# Memory threshold
|
||||||
|
SOFT_MEM_THRESHOLD=80 # warn at this % of per-container hard limit
|
||||||
|
|
||||||
|
# Responsiveness check
|
||||||
|
RESP_FAIL_LIMIT=2 # consecutive failures before restart
|
||||||
|
CURL_TIMEOUT=5 # seconds before curl gives up
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── UNRAID ESSENTIALS ─────────────────────────────────────────────────────────────────────────
|
||||||
|
# ==============================================================================================
|
||||||
|
|
||||||
|
# ━━━ Reboot ━━━
|
||||||
|
# User warning delay before scheduled reboot (seconds)
|
||||||
|
REBOOT_SLEEP=300
|
||||||
|
|
||||||
|
# ━━━ Mover ━━━
|
||||||
|
# Timeout before stopping the mover (seconds)
|
||||||
|
MOVER_STOP_TIMEOUT=300
|
||||||
|
|
||||||
|
# ━━━ Syslog Filter ━━━
|
||||||
|
# Path for the rsyslog Docker noise filter file
|
||||||
|
FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf"
|
||||||
|
|
||||||
|
# ━━━ PHP-FPM ━━━
|
||||||
|
# PHP-FPM config file path and max children value
|
||||||
|
PHP_CONF="/etc/php-fpm.d/www.conf"
|
||||||
|
PHP_MAX_CHILDREN=250
|
||||||
|
|
||||||
|
# ━━━ Clear Logs ━━━
|
||||||
|
# System log files to clear on each run
|
||||||
|
LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg)
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── MEDIA ─────────────────────────────────────────────────────────────────────────────────────
|
||||||
|
# ==============================================================================================
|
||||||
|
|
||||||
|
# ━━━ Media Permissions ━━━
|
||||||
|
# Mode and owner applied recursively to all listed shares
|
||||||
|
PERMISSIONS_MODE="777"
|
||||||
|
PERMISSIONS_OWNER="nobody:users"
|
||||||
|
|
||||||
|
MEDIA_PERMISSION_SHARES=(
|
||||||
|
/mnt/user/Anime_Movies
|
||||||
|
/mnt/user/Anime_Movies-Old
|
||||||
|
/mnt/user/Anime_Shows
|
||||||
|
/mnt/user/Anime_Shows-Old
|
||||||
|
/mnt/user/appcache
|
||||||
|
/mnt/user/Books
|
||||||
|
/mnt/user/Downloads
|
||||||
|
/mnt/user/Games
|
||||||
|
/mnt/user/Intros
|
||||||
|
/mnt/user/Kids_Movies
|
||||||
|
/mnt/user/Kids_Tv_Shows
|
||||||
|
/mnt/user/Movie_Recordings
|
||||||
|
/mnt/user/Movies
|
||||||
|
/mnt/user/Music
|
||||||
|
/mnt/user/Music_Videos
|
||||||
|
/mnt/user/Photo
|
||||||
|
/mnt/user/Sports
|
||||||
|
/mnt/user/stand-up_comedy
|
||||||
|
/mnt/user/Temp_Storage
|
||||||
|
/mnt/user/Tv_Recordings
|
||||||
|
/mnt/user/Tv_Shows
|
||||||
|
/mnt/user/YouTube
|
||||||
|
)
|
||||||
|
|
||||||
|
# ━━━ Media Cleaner ━━━
|
||||||
|
# Two profiles: anime and media — passed as argument to media_cleaner.sh
|
||||||
|
# Usage: media_cleaner.sh anime or media_cleaner.sh media
|
||||||
|
|
||||||
|
ANIME_CLEAN_FOLDERS=(
|
||||||
|
/mnt/user/Anime_Movies
|
||||||
|
/mnt/user/Anime_Movies-Old
|
||||||
|
/mnt/user/Anime_Shows
|
||||||
|
/mnt/user/Anime_Shows-Old
|
||||||
|
)
|
||||||
|
|
||||||
|
MEDIA_CLEAN_FOLDERS=(
|
||||||
|
/mnt/user/Kids_Movies
|
||||||
|
/mnt/user/Kids_Tv_Shows
|
||||||
|
/mnt/user/Movies
|
||||||
|
/mnt/user/Music
|
||||||
|
/mnt/user/Sports
|
||||||
|
/mnt/user/stand-up_comedy
|
||||||
|
/mnt/user/Tv_Shows
|
||||||
|
)
|
||||||
|
|
||||||
|
# Anime file patterns — junk files common in anime downloads
|
||||||
|
ANIME_FILE_PATTERNS=(
|
||||||
|
'*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
|
||||||
|
'*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
|
||||||
|
'*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp'
|
||||||
|
'*.log' '*.json'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Media file patterns — includes *.iso and *.lrc not needed in anime
|
||||||
|
MEDIA_FILE_PATTERNS=(
|
||||||
|
'*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk'
|
||||||
|
'*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*'
|
||||||
|
'*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp'
|
||||||
|
'*.log' '*.json' '*.iso' '*.lrc'
|
||||||
|
)
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ── SYSTEM WATCHDOG ───────────────────────────────────────────────────────────────────────────
|
||||||
|
# ==============================================================================================
|
||||||
|
# Last line of defense — reboots the system cleanly if it is about to become unstable.
|
||||||
|
# Designed to run on a cron schedule (recommended every 15-30 minutes).
|
||||||
|
# Works alongside docker_watchdog.sh — containers first, system second.
|
||||||
|
#
|
||||||
|
# Strike system — sustained threshold hits trigger reboot, not single spikes.
|
||||||
|
# Reboot loop protection — shuts down instead of rebooting if limit hit in window.
|
||||||
|
# Container skip list — persistent, auto-clears when container recovers.
|
||||||
|
|
||||||
|
# ━━━ System Watchdog State Files ━━━
|
||||||
|
# Strike counts — /tmp resets on reboot, correct for strike tracking
|
||||||
|
SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db"
|
||||||
|
# Persistent container skip list — survives reboots, auto-clears on recovery
|
||||||
|
SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db"
|
||||||
|
# Reboot timestamp log — survives reboots for loop detection
|
||||||
|
SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db"
|
||||||
|
|
||||||
|
# ━━━ Strike and Reboot Loop Settings ━━━
|
||||||
|
# Consecutive threshold hits before triggering reboot
|
||||||
|
SYS_WATCHDOG_STRIKE_LIMIT=2
|
||||||
|
# Maximum reboots allowed within the window before shutdown instead
|
||||||
|
SYS_WATCHDOG_REBOOT_LIMIT=3
|
||||||
|
# Window in hours — controls both reboot count window AND rolling log purge
|
||||||
|
# 12 = entries older than 12hrs purge automatically / 24 = entries older than 24hrs purge
|
||||||
|
SYS_WATCHDOG_REBOOT_WINDOW_HRS=12
|
||||||
|
|
||||||
|
# ━━━ Thresholds ━━━
|
||||||
|
# Set at "about to fall over" levels — not just high usage
|
||||||
|
SYS_WATCHDOG_ROOTFS_PCT=95 # rootfs usage % before strike
|
||||||
|
SYS_WATCHDOG_LOG_PCT=95 # /var/log usage % before strike
|
||||||
|
SYS_WATCHDOG_MEM_GB=4 # free RAM in GB below which strikes
|
||||||
|
SYS_WATCHDOG_ARC_PINNED_PCT=98 # ZFS ARC % of max before reclaim attempt
|
||||||
|
SYS_WATCHDOG_ARC_RELEASE_PCT=95 # ZFS ARC % after reclaim that still triggers
|
||||||
|
SYS_WATCHDOG_LOAD_MULTIPLIER=32 # strike if load avg > cores x this value
|
||||||
|
SYS_WATCHDOG_ZOMBIE_LIMIT=50 # strike if zombie process count exceeds this
|
||||||
|
SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — tjmax varies by CPU
|
||||||
|
|
||||||
|
# ━━━ Check Toggles ━━━
|
||||||
|
# true = run this check / false = skip entirely
|
||||||
|
SYS_WATCHDOG_CHECK_ROOTFS=true
|
||||||
|
SYS_WATCHDOG_CHECK_LOG=true
|
||||||
|
SYS_WATCHDOG_CHECK_RAM=true
|
||||||
|
SYS_WATCHDOG_CHECK_ARC=true
|
||||||
|
SYS_WATCHDOG_CHECK_CPU_TEMP=true
|
||||||
|
SYS_WATCHDOG_CHECK_LOAD=true
|
||||||
|
SYS_WATCHDOG_CHECK_ZOMBIES=true
|
||||||
|
SYS_WATCHDOG_CHECK_CONTAINERS=true
|
||||||
|
SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true
|
||||||
|
|
||||||
|
# ━━━ Abort Toggles ━━━
|
||||||
|
# true = abort reboot if condition is active / false = reboot anyway
|
||||||
|
# Default true = conservative / set false to reboot regardless
|
||||||
|
SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true
|
||||||
|
SYS_WATCHDOG_ABORT_ON_PARITY=false
|
||||||
|
SYS_WATCHDOG_ABORT_ON_MOVER=false
|
||||||
|
|
||||||
|
# ==============================================================================================
|
||||||
|
# ──────────────────────── End Of User Variables ───────────────────────────────────────────────
|
||||||
|
# ==============================================================================================
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ----------------------------
|
# ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ----------------------------
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# Version: 2.3
|
# Version: 2.4
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
# Changelog:
|
# Changelog:
|
||||||
# v1.0 — Initial stable framework
|
# v1.0 — Initial stable framework
|
||||||
@@ -37,6 +37,9 @@
|
|||||||
# v2.3 — ICON_NOTIFY added for notification operations
|
# v2.3 — ICON_NOTIFY added for notification operations
|
||||||
# notify() added — shared notification function supporting unRAID native and Discord
|
# notify() added — shared notification function supporting unRAID native and Discord
|
||||||
# NOTIFY_UNRAID and DISCORD_WEBHOOK configured in Master.conf
|
# NOTIFY_UNRAID and DISCORD_WEBHOOK configured in Master.conf
|
||||||
|
# v2.4 — ICON_CLEAN, ICON_TRASH added for media cleaner operations
|
||||||
|
# ICON_PERMS, ICON_UNLOCKED added for media permissions operations
|
||||||
|
# ICON_REBOOT_SMART added for smart conditional reboot
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
@@ -76,7 +79,8 @@ ICON_TIME="⏱️" # duration line
|
|||||||
|
|
||||||
# System Operations
|
# System Operations
|
||||||
ICON_MOVER="🔃" # mover operations
|
ICON_MOVER="🔃" # mover operations
|
||||||
ICON_REBOOT="⚡" # server reboot operations
|
ICON_REBOOT="⚡" # scheduled server reboot
|
||||||
|
ICON_REBOOT_SMART="🚨" # smart conditional reboot triggered
|
||||||
ICON_PLUGIN="🧩" # user scripts plugin operations
|
ICON_PLUGIN="🧩" # user scripts plugin operations
|
||||||
ICON_PHP="👥" # PHP-FPM operations
|
ICON_PHP="👥" # PHP-FPM operations
|
||||||
|
|
||||||
@@ -85,6 +89,12 @@ ICON_ZFS="📊" # ZFS ARC statistics
|
|||||||
ICON_MEM="🧠" # memory status
|
ICON_MEM="🧠" # memory status
|
||||||
ICON_WATCHDOG="🐾" # docker watchdog monitoring operations
|
ICON_WATCHDOG="🐾" # docker watchdog monitoring operations
|
||||||
|
|
||||||
|
# Media Operations
|
||||||
|
ICON_CLEAN="🧹" # media cleaner operations
|
||||||
|
ICON_TRASH="🗑️" # files being deleted
|
||||||
|
ICON_PERMS="🔐" # permissions operation / section header
|
||||||
|
ICON_UNLOCKED="🔓" # permissions successfully applied to a share
|
||||||
|
|
||||||
# Notifications
|
# Notifications
|
||||||
ICON_NOTIFY="🔔" # notification operations
|
ICON_NOTIFY="🔔" # notification operations
|
||||||
|
|
||||||
@@ -116,7 +126,7 @@ log() {
|
|||||||
# Discord: requires DISCORD_WEBHOOK to be set to a valid webhook URL.
|
# Discord: requires DISCORD_WEBHOOK to be set to a valid webhook URL.
|
||||||
# Severity levels: normal, warning, alert — maps to unRAID notification severity.
|
# Severity levels: normal, warning, alert — maps to unRAID notification severity.
|
||||||
# Usage: notify "message" "subject" "severity"
|
# Usage: notify "message" "subject" "severity"
|
||||||
# notify "Rsync failed: Movies" "Rsync Alert" "alert"
|
# notify "Rsync failed: Movies" "Rsync Alert" "warning"
|
||||||
# notify "Daily sync complete" "Daily Sync" "normal"
|
# notify "Daily sync complete" "Daily Sync" "normal"
|
||||||
# -----------------------------------------------------------------------------------------------
|
# -----------------------------------------------------------------------------------------------
|
||||||
notify() {
|
notify() {
|
||||||
@@ -139,8 +149,6 @@ notify() {
|
|||||||
|
|
||||||
# Discord webhook notification
|
# Discord webhook notification
|
||||||
if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then
|
if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then
|
||||||
local hostname
|
|
||||||
hostname=$(hostname)
|
|
||||||
local payload
|
local payload
|
||||||
payload=$(printf '{"content": "%s — **%s**\\n%s"}' \
|
payload=$(printf '{"content": "%s — **%s**\\n%s"}' \
|
||||||
"$ICON_NOTIFY" "$subject" "$message")
|
"$ICON_NOTIFY" "$subject" "$message")
|
||||||
|
|||||||
@@ -0,0 +1,495 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# --------------------------------- System Watchdog --------------------------------------------
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# Last line of defense — reboots the system cleanly if it is about to become unstable.
|
||||||
|
# Works alongside docker_watchdog.sh which handles container-level healing first.
|
||||||
|
#
|
||||||
|
# Checks (all toggleable in Master.conf):
|
||||||
|
# rootfs usage — high rootfs fills rapidly when array is down, crash imminent
|
||||||
|
# /var/log usage — log spam can fill rootfs, indicates something is broken
|
||||||
|
# free RAM — critically low RAM means OOM or swap imminent
|
||||||
|
# ZFS ARC pinned — ARC not releasing after reclaim means memory is stuck
|
||||||
|
# CPU temperature — sustained tjmax causes throttling or kernel panic
|
||||||
|
# load average — sustained high load means something is stuck or runaway
|
||||||
|
# zombie processes — large zombie count indicates serious process management failure
|
||||||
|
# Docker daemon — unresponsive daemon means containers cannot be managed
|
||||||
|
# Required containers — stopped containers that should be running (after watchdog skip list)
|
||||||
|
#
|
||||||
|
# Abort conditions (toggleable — true = abort, false = reboot anyway):
|
||||||
|
# ZFS pool unhealthy — reboot with bad pool risks data loss
|
||||||
|
# Parity running — aborting parity is better than crashing mid-check
|
||||||
|
# Mover running — aborting move is better than crashing mid-move
|
||||||
|
#
|
||||||
|
# Reboot loop protection:
|
||||||
|
# Tracks reboot timestamps in persistent log on /boot/
|
||||||
|
# Rolling window — old entries purge automatically after SYS_WATCHDOG_REBOOT_WINDOW_HRS
|
||||||
|
# If reboot count hits SYS_WATCHDOG_REBOOT_LIMIT in window → shutdown instead of reboot
|
||||||
|
#
|
||||||
|
# All configuration in Master.conf under System Watchdog section.
|
||||||
|
# Supports --dry-run to show triggered conditions without rebooting.
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
|
source "$SCRIPT_DIR/../Master.conf"
|
||||||
|
source "$SCRIPT_DIR/../common.sh"
|
||||||
|
|
||||||
|
parse_args "$@"
|
||||||
|
|
||||||
|
# Convert window hours to seconds for internal use
|
||||||
|
SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 ))
|
||||||
|
|
||||||
|
TOTAL_CORES=$(nproc)
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_GEAR Setup ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||||
|
|
||||||
|
if [[ "$EUID" -ne 0 ]]; then
|
||||||
|
error "Must be run as root"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
success "Running as root"
|
||||||
|
|
||||||
|
# Ensure state and persistent files exist
|
||||||
|
touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || {
|
||||||
|
error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
touch "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || {
|
||||||
|
error "Cannot create reboot log: $SYS_WATCHDOG_REBOOT_LOG"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || {
|
||||||
|
error "Cannot create failed container list: $SYS_WATCHDOG_FAILED_FILE"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_SUMMARY Status ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
if [[ "$SHOW_STATUS" == true ]]; then
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||||
|
echo "$ICON_HEALTH rootfs threshold: ${SYS_WATCHDOG_ROOTFS_PCT}% toggle: $SYS_WATCHDOG_CHECK_ROOTFS"
|
||||||
|
echo "$ICON_HEALTH log threshold: ${SYS_WATCHDOG_LOG_PCT}% toggle: $SYS_WATCHDOG_CHECK_LOG"
|
||||||
|
echo "$ICON_MEM RAM threshold: ${SYS_WATCHDOG_MEM_GB}GB free toggle: $SYS_WATCHDOG_CHECK_RAM"
|
||||||
|
echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}% toggle: $SYS_WATCHDOG_CHECK_ARC"
|
||||||
|
echo "$ICON_ZFS ARC release: ${SYS_WATCHDOG_ARC_RELEASE_PCT}%"
|
||||||
|
echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C toggle: $SYS_WATCHDOG_CHECK_CPU_TEMP"
|
||||||
|
echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x cores toggle: $SYS_WATCHDOG_CHECK_LOAD"
|
||||||
|
echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT} toggle: $SYS_WATCHDOG_CHECK_ZOMBIES"
|
||||||
|
echo "$ICON_CONTAINERS Docker daemon: toggle: $SYS_WATCHDOG_CHECK_DOCKER_DAEMON"
|
||||||
|
echo "$ICON_CONTAINERS Containers: toggle: $SYS_WATCHDOG_CHECK_CONTAINERS"
|
||||||
|
echo "$ICON_SHIELD Strike limit: $SYS_WATCHDOG_STRIKE_LIMIT"
|
||||||
|
echo "$ICON_REBOOT_SMART Reboot limit: $SYS_WATCHDOG_REBOOT_LIMIT in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs"
|
||||||
|
echo "$ICON_ZFS Abort ZFS unhealthy: $SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY"
|
||||||
|
echo "$ICON_GEAR Abort on parity: $SYS_WATCHDOG_ABORT_ON_PARITY"
|
||||||
|
echo "$ICON_MOVER Abort on mover: $SYS_WATCHDOG_ABORT_ON_MOVER"
|
||||||
|
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboot will be executed"
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# STATE HELPERS
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
get_strikes() {
|
||||||
|
local key="$1"
|
||||||
|
grep -E "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2
|
||||||
|
}
|
||||||
|
|
||||||
|
set_strikes() {
|
||||||
|
local key="$1" count="$2"
|
||||||
|
grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
||||||
|
echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp"
|
||||||
|
mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Returns current strike count, increments and saves, then echoes new count
|
||||||
|
increment_strikes() {
|
||||||
|
local key="$1"
|
||||||
|
local current
|
||||||
|
current=$(get_strikes "$key")
|
||||||
|
[[ -z "$current" ]] && current=0
|
||||||
|
((current++))
|
||||||
|
set_strikes "$key" "$current"
|
||||||
|
echo "$current"
|
||||||
|
}
|
||||||
|
|
||||||
|
reset_strikes() {
|
||||||
|
local key="$1"
|
||||||
|
set_strikes "$key" 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# REBOOT LOG HELPERS
|
||||||
|
# Tracks reboot timestamps for loop detection.
|
||||||
|
# Rolling window — entries older than SYS_WATCHDOG_REBOOT_WINDOW are purged automatically.
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
purge_old_reboots() {
|
||||||
|
local now
|
||||||
|
now=$(date +%s)
|
||||||
|
local cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW ))
|
||||||
|
|
||||||
|
grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do
|
||||||
|
[[ "$ts" -gt "$cutoff" ]] && echo "$ts"
|
||||||
|
done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp"
|
||||||
|
mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG"
|
||||||
|
}
|
||||||
|
|
||||||
|
count_recent_reboots() {
|
||||||
|
purge_old_reboots
|
||||||
|
grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0
|
||||||
|
}
|
||||||
|
|
||||||
|
log_reboot() {
|
||||||
|
date +%s >> "$SYS_WATCHDOG_REBOOT_LOG"
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ABORT CONDITION CHECKS
|
||||||
|
# Run before any reboot is triggered — abort conditions prevent rebooting
|
||||||
|
# when it would make things worse than letting the system run.
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
check_abort_conditions() {
|
||||||
|
local should_abort=false
|
||||||
|
|
||||||
|
# ZFS pool health
|
||||||
|
if command -v zpool >/dev/null 2>&1; then
|
||||||
|
local unhealthy
|
||||||
|
unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true)
|
||||||
|
if [[ -n "$unhealthy" ]]; then
|
||||||
|
if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then
|
||||||
|
error "$ICON_ZFS ZFS pool unhealthy — aborting reboot (ABORT_ON_ZFS_UNHEALTHY=true)"
|
||||||
|
notify "System watchdog aborted reboot on $(hostname) — ZFS pool unhealthy" "System Watchdog" "warning"
|
||||||
|
should_abort=true
|
||||||
|
else
|
||||||
|
warn "$ICON_ZFS ZFS pool unhealthy — continuing reboot (ABORT_ON_ZFS_UNHEALTHY=false)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Parity check running
|
||||||
|
if [[ -f /var/local/emhttp/parity-date.txt ]]; then
|
||||||
|
if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then
|
||||||
|
if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then
|
||||||
|
error "$ICON_DISK Parity check running — aborting reboot (ABORT_ON_PARITY=true)"
|
||||||
|
notify "System watchdog aborted reboot on $(hostname) — parity check running" "System Watchdog" "warning"
|
||||||
|
should_abort=true
|
||||||
|
else
|
||||||
|
warn "$ICON_DISK Parity check running — continuing reboot (ABORT_ON_PARITY=false)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Mover running
|
||||||
|
if pgrep -f "mover" >/dev/null 2>&1; then
|
||||||
|
if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then
|
||||||
|
error "$ICON_MOVER Mover is running — aborting reboot (ABORT_ON_MOVER=true)"
|
||||||
|
notify "System watchdog aborted reboot on $(hostname) — mover running" "System Watchdog" "warning"
|
||||||
|
should_abort=true
|
||||||
|
else
|
||||||
|
warn "$ICON_MOVER Mover is running — continuing reboot (ABORT_ON_MOVER=false)"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
[[ "$should_abort" == true ]] && return 1
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# STRIKE-BASED CHECK HELPER
|
||||||
|
# Runs a check function, increments strikes on trigger, resets on clear.
|
||||||
|
# Returns 0 if strike limit hit (reboot trigger), 1 otherwise.
|
||||||
|
# Usage: run_strike_check "key" "triggered (true/false)" "description"
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
run_strike_check() {
|
||||||
|
local key="$1" triggered="$2" description="$3"
|
||||||
|
|
||||||
|
if [[ "$triggered" == true ]]; then
|
||||||
|
local strikes
|
||||||
|
strikes=$(increment_strikes "$key")
|
||||||
|
warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT"
|
||||||
|
if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then
|
||||||
|
error "$description hit strike limit — reboot triggered"
|
||||||
|
reset_strikes "$key"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
local current
|
||||||
|
current=$(get_strikes "$key")
|
||||||
|
if [[ -n "$current" && "$current" -gt 0 ]]; then
|
||||||
|
info "$description — recovered, resetting strikes"
|
||||||
|
reset_strikes "$key"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_SHIELD Health Checks ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_SHIELD Health Checks — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
TRIGGERS=()
|
||||||
|
|
||||||
|
# rootfs usage
|
||||||
|
if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then
|
||||||
|
info "$ICON_HEALTH Checking rootfs usage..."
|
||||||
|
ROOTFS_USED=$(df / --output=pcent | tail -1 | tr -d ' %')
|
||||||
|
TRIGGERED=false
|
||||||
|
if [[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]]; then
|
||||||
|
error "$ICON_HEALTH rootfs is ${ROOTFS_USED}% full — threshold ${SYS_WATCHDOG_ROOTFS_PCT}%"
|
||||||
|
TRIGGERED=true
|
||||||
|
else
|
||||||
|
success "$ICON_HEALTH rootfs: ${ROOTFS_USED}% used"
|
||||||
|
fi
|
||||||
|
run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && TRIGGERS+=("rootfs=${ROOTFS_USED}%")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# /var/log usage
|
||||||
|
if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then
|
||||||
|
info "$ICON_HEALTH Checking /var/log usage..."
|
||||||
|
LOG_USED=$(df -P /var/log | awk 'NR==2 {print $5}' | tr -d '%')
|
||||||
|
TRIGGERED=false
|
||||||
|
if [[ "$LOG_USED" -ge "$SYS_WATCHDOG_LOG_PCT" ]]; then
|
||||||
|
error "$ICON_HEALTH /var/log is ${LOG_USED}% full — threshold ${SYS_WATCHDOG_LOG_PCT}%"
|
||||||
|
TRIGGERED=true
|
||||||
|
else
|
||||||
|
success "$ICON_HEALTH /var/log: ${LOG_USED}% used"
|
||||||
|
fi
|
||||||
|
run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && TRIGGERS+=("log=${LOG_USED}%")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Free RAM
|
||||||
|
if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then
|
||||||
|
info "$ICON_MEM Checking available RAM..."
|
||||||
|
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||||
|
MEM_GB=$((MEM_KB / 1024 / 1024))
|
||||||
|
TRIGGERED=false
|
||||||
|
if [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]]; then
|
||||||
|
error "$ICON_MEM Available RAM: ${MEM_GB}GB — threshold ${SYS_WATCHDOG_MEM_GB}GB"
|
||||||
|
TRIGGERED=true
|
||||||
|
else
|
||||||
|
success "$ICON_MEM Available RAM: ${MEM_GB}GB"
|
||||||
|
fi
|
||||||
|
run_strike_check "ram" "$TRIGGERED" "RAM ${MEM_GB}GB free" && TRIGGERS+=("low_ram=${MEM_GB}GB")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ZFS ARC pinned
|
||||||
|
if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then
|
||||||
|
info "$ICON_ZFS Checking ZFS ARC..."
|
||||||
|
ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||||
|
ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||||
|
ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX ))
|
||||||
|
TRIGGERED=false
|
||||||
|
|
||||||
|
if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then
|
||||||
|
warn "$ICON_ZFS ARC at ${ARC_PCT}% — attempting reclaim..."
|
||||||
|
sync
|
||||||
|
echo 3 > /proc/sys/vm/drop_caches
|
||||||
|
sleep 5
|
||||||
|
ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats)
|
||||||
|
ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX ))
|
||||||
|
if [[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]]; then
|
||||||
|
error "$ICON_ZFS ARC still ${ARC_AFTER_PCT}% after reclaim — threshold ${SYS_WATCHDOG_ARC_RELEASE_PCT}%"
|
||||||
|
TRIGGERED=true
|
||||||
|
else
|
||||||
|
success "$ICON_ZFS ARC released to ${ARC_AFTER_PCT}% after reclaim"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
success "$ICON_ZFS ARC: ${ARC_PCT}% of max"
|
||||||
|
fi
|
||||||
|
run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned" && TRIGGERS+=("arc_pinned=${ARC_PCT}%")
|
||||||
|
elif [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]]; then
|
||||||
|
info "$ICON_ZFS ZFS arcstats not available — skipping"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# CPU temperature
|
||||||
|
if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then
|
||||||
|
info "$ICON_GEAR Checking CPU temperature..."
|
||||||
|
CPU_TEMP=""
|
||||||
|
if command -v sensors >/dev/null 2>&1; then
|
||||||
|
CPU_TEMP=$(sensors 2>/dev/null | grep -i "Package id 0\|Tctl\|CPU Temp" | awk '{print $NF}' | tr -d '+°C' | head -1)
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z "$CPU_TEMP" ]]; then
|
||||||
|
info "$ICON_GEAR CPU temperature sensor not available — skipping"
|
||||||
|
else
|
||||||
|
CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP")
|
||||||
|
TRIGGERED=false
|
||||||
|
if [[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]]; then
|
||||||
|
error "$ICON_GEAR CPU temp ${CPU_TEMP_INT}°C — threshold ${SYS_WATCHDOG_CPU_TEMP_MAX}°C"
|
||||||
|
TRIGGERED=true
|
||||||
|
else
|
||||||
|
success "$ICON_GEAR CPU temp: ${CPU_TEMP_INT}°C"
|
||||||
|
fi
|
||||||
|
run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C")
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Load average
|
||||||
|
if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then
|
||||||
|
info "$ICON_GEAR Checking load average..."
|
||||||
|
LOAD=$(awk '{print $1}' /proc/loadavg)
|
||||||
|
LOAD_INT=$(printf "%.0f" "$LOAD")
|
||||||
|
LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER ))
|
||||||
|
TRIGGERED=false
|
||||||
|
if [[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]]; then
|
||||||
|
error "$ICON_GEAR Load average ${LOAD} — threshold ${LOAD_THRESHOLD} (${TOTAL_CORES} cores x ${SYS_WATCHDOG_LOAD_MULTIPLIER})"
|
||||||
|
TRIGGERED=true
|
||||||
|
else
|
||||||
|
success "$ICON_GEAR Load average: ${LOAD}"
|
||||||
|
fi
|
||||||
|
run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && TRIGGERS+=("load=${LOAD}")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Zombie processes
|
||||||
|
if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then
|
||||||
|
info "$ICON_GEAR Checking zombie processes..."
|
||||||
|
ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" || echo 0)
|
||||||
|
TRIGGERED=false
|
||||||
|
if [[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]]; then
|
||||||
|
error "$ICON_GEAR Zombie processes: $ZOMBIE_COUNT — threshold $SYS_WATCHDOG_ZOMBIE_LIMIT"
|
||||||
|
TRIGGERED=true
|
||||||
|
else
|
||||||
|
success "$ICON_GEAR Zombie processes: $ZOMBIE_COUNT"
|
||||||
|
fi
|
||||||
|
run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && TRIGGERS+=("zombies=${ZOMBIE_COUNT}")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Docker daemon health
|
||||||
|
if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then
|
||||||
|
info "$ICON_CONTAINERS Checking Docker daemon..."
|
||||||
|
TRIGGERED=false
|
||||||
|
if ! timeout 10 docker ps >/dev/null 2>&1; then
|
||||||
|
error "$ICON_CONTAINERS Docker daemon is not responding"
|
||||||
|
TRIGGERED=true
|
||||||
|
else
|
||||||
|
success "$ICON_CONTAINERS Docker daemon is healthy"
|
||||||
|
fi
|
||||||
|
run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon unresponsive" && TRIGGERS+=("docker_daemon")
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Required containers from skip list
|
||||||
|
if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then
|
||||||
|
info "$ICON_CONTAINERS Checking persistent failed containers..."
|
||||||
|
FAILED_CONTAINERS=()
|
||||||
|
while IFS= read -r container; do
|
||||||
|
[[ -z "$container" ]] && continue
|
||||||
|
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||||
|
if [[ "$STATUS" != "true" ]]; then
|
||||||
|
error "$ICON_NOT_RUNNING $container still not running (from skip list)"
|
||||||
|
FAILED_CONTAINERS+=("$container")
|
||||||
|
fi
|
||||||
|
done < "$SYS_WATCHDOG_FAILED_FILE"
|
||||||
|
|
||||||
|
if [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]]; then
|
||||||
|
TRIGGERED=true
|
||||||
|
run_strike_check "failed_containers" "$TRIGGERED" "required containers stopped" && TRIGGERS+=("containers=${FAILED_CONTAINERS[*]}")
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# No triggers — exit cleanly
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
if [[ ${#TRIGGERS[@]} -eq 0 ]]; then
|
||||||
|
echo ""
|
||||||
|
success "No reboot conditions met — system is healthy"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# ━━━ $ICON_REBOOT_SMART Reboot Triggered ━━━
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
echo ""
|
||||||
|
echo "━━━ $ICON_REBOOT_SMART Reboot Triggered ━━━"
|
||||||
|
for t in "${TRIGGERS[@]}"; do
|
||||||
|
echo " $ICON_REBOOT_SMART $t"
|
||||||
|
done
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
# Check abort conditions before proceeding
|
||||||
|
if ! check_abort_conditions; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Reboot loop protection — check recent reboot count
|
||||||
|
RECENT_REBOOTS=$(count_recent_reboots)
|
||||||
|
info "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT"
|
||||||
|
|
||||||
|
if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then
|
||||||
|
error "Reboot limit hit — $RECENT_REBOOTS reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs — shutting down instead"
|
||||||
|
notify "Reboot loop detected on $(hostname) — shutting down after $RECENT_REBOOTS reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs — conditions: ${TRIGGERS[*]}" "System Watchdog" "warning"
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — would shutdown now"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
sync
|
||||||
|
/sbin/poweroff
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
notify "System watchdog reboot triggered on $(hostname) — conditions: ${TRIGGERS[*]}" "System Watchdog" "warning"
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
warn "DRY RUN — reboot sequence would begin now"
|
||||||
|
echo ""
|
||||||
|
echo "━━━━━ $ICON_SUMMARY SYSTEM WATCHDOG SUMMARY ━━━━━"
|
||||||
|
echo "$ICON_REBOOT_SMART Triggers: ${TRIGGERS[*]}"
|
||||||
|
echo "$ICON_REBOOT_SMART Recent reboots: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT"
|
||||||
|
echo "$ICON_WARN Status: DRY RUN — no reboot executed"
|
||||||
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
# Graceful shutdown sequence
|
||||||
|
# -----------------------------------------------------------------------------------------------
|
||||||
|
info "Logging reboot timestamp..."
|
||||||
|
log_reboot
|
||||||
|
|
||||||
|
info "Shutting down VMs..."
|
||||||
|
if command -v virsh >/dev/null 2>&1; then
|
||||||
|
for VM in $(virsh list --name 2>/dev/null); do
|
||||||
|
[[ -z "$VM" ]] && continue
|
||||||
|
info "Shutting down VM: $VM"
|
||||||
|
virsh shutdown "$VM" >/dev/null 2>&1
|
||||||
|
done
|
||||||
|
info "Waiting 30s for VMs..."
|
||||||
|
sleep 30
|
||||||
|
fi
|
||||||
|
|
||||||
|
info "Stopping Docker containers..."
|
||||||
|
if command -v docker >/dev/null 2>&1; then
|
||||||
|
docker ps -q | xargs -r docker stop >/dev/null 2>&1
|
||||||
|
success "Docker containers stopped"
|
||||||
|
fi
|
||||||
|
|
||||||
|
info "Stopping User Scripts..."
|
||||||
|
pkill -f "/tmp/user.scripts" 2>/dev/null || true
|
||||||
|
|
||||||
|
info "Syncing disks..."
|
||||||
|
sync
|
||||||
|
success "Disks synced"
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "$ICON_REBOOT_SMART Rebooting system NOW..."
|
||||||
|
sleep 5
|
||||||
|
/sbin/reboot
|
||||||
Reference in New Issue
Block a user