From b7706f4ab4024c61f309a35736b65e89484fc489 Mon Sep 17 00:00:00 2001 From: FailedProxy Date: Fri, 10 Apr 2026 18:11:16 -0400 Subject: [PATCH] Added system watchdog. and way to many other changes --- Docker_essentials/docker_daily_restart.sh | 34 +- Docker_essentials/docker_watchdog.sh | 332 ++++++++---- Docker_essentials/docker_weekly_restart.sh | 174 ++++++ Docker_essentials/media_shares_permissions.sh | 119 +++++ Docker_essentials/remove_unwante_files.sh | 183 +++++++ Master.conf | 406 ++++++++++---- common.sh | 18 +- unRAID_Essentials/system_watchdog.sh | 495 ++++++++++++++++++ 8 files changed, 1532 insertions(+), 229 deletions(-) create mode 100644 Docker_essentials/docker_weekly_restart.sh create mode 100644 Docker_essentials/media_shares_permissions.sh create mode 100644 Docker_essentials/remove_unwante_files.sh create mode 100644 unRAID_Essentials/system_watchdog.sh diff --git a/Docker_essentials/docker_daily_restart.sh b/Docker_essentials/docker_daily_restart.sh index e516b7d..bc6e289 100644 --- a/Docker_essentials/docker_daily_restart.sh +++ b/Docker_essentials/docker_daily_restart.sh @@ -22,7 +22,6 @@ parse_args "$@" echo "" echo "━━━ $ICON_GEAR Setup ━━━" -# ROOT CHECK if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 @@ -30,10 +29,9 @@ fi success "Running as root" -# Verify Docker is available if ! command -v docker &>/dev/null; then error "Docker command not found — check PATH or Docker installation" - notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "alert" + notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "warning" exit 1 fi @@ -60,9 +58,6 @@ fi # FUNCTIONS # ----------------------------------------------------------------------------------------------- -# Attempts a docker command up to RETRY_COUNT times with SLEEP seconds between attempts. -# Returns 0 on success, 1 if all attempts fail. -# Usage: retry_docker docker restart Emby retry_docker() { local attempt=1 @@ -84,7 +79,7 @@ retry_docker() { } # ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_CONTAINERS $ICON_STOP $ICON_START Daily Restart ━━━ +# ━━━ $ICON_CONTAINERS Daily Restart ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_CONTAINERS Daily Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━" @@ -100,7 +95,6 @@ STARTED=() for container in "${DAILY_RESTART_CONTAINERS[@]}"; do echo "━━━ $ICON_CONTAINERS $container ━━━" - # Verify container exists if ! docker inspect "$container" &>/dev/null; then error "$container does not exist — skipping" FAILED+=("$container") @@ -121,7 +115,8 @@ for container in "${DAILY_RESTART_CONTAINERS[@]}"; do echo "$ICON_STARTED $container restarted" RESTARTED+=("$container") else - error "Failed to restart $container" + error "Failed to restart $container after $RETRY_COUNT attempts" + notify "$container failed to restart on $(hostname)" "Docker Daily Restart" "warning" FAILED+=("$container") fi fi @@ -136,7 +131,8 @@ for container in "${DAILY_RESTART_CONTAINERS[@]}"; do echo "$ICON_STARTED $container started" STARTED+=("$container") else - error "Failed to start $container" + error "Failed to start $container after $RETRY_COUNT attempts" + notify "$container failed to start on $(hostname)" "Docker Daily Restart" "warning" FAILED+=("$container") fi fi @@ -157,27 +153,19 @@ END=$(date +%s) # ----------------------------------------------------------------------------------------------- echo "━━━━━ $ICON_SUMMARY DAILY RESTART SUMMARY ━━━━━" echo "$ICON_TIME Duration: $(format_duration $((END - START)))" - -if [[ ${#RESTARTED[@]} -gt 0 ]]; then - echo "$ICON_STARTED Restarted: ${RESTARTED[*]}" -fi -if [[ ${#STARTED[@]} -gt 0 ]]; then - echo "$ICON_STARTED Started: ${STARTED[*]}" -fi -if [[ ${#FAILED[@]} -gt 0 ]]; then - echo "$ICON_ERROR Failed: ${FAILED[*]}" -fi +[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}" +[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" if [[ "$DRY_RUN" == true ]]; then echo "$ICON_WARN Status: DRY RUN — no changes made" elif [[ ${#FAILED[@]} -eq 0 ]]; then echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE" - notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started" "Docker Daily Restart" "normal" + notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Daily Restart" "normal" else echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed" - notify "Daily restart completed with errors — failed: ${FAILED[*]}" "Docker Daily Restart" "alert" + notify "Daily restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Daily Restart" "warning" fi - echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" [[ ${#FAILED[@]} -gt 0 ]] && exit 1 diff --git a/Docker_essentials/docker_watchdog.sh b/Docker_essentials/docker_watchdog.sh index 1c877de..fba76cc 100644 --- a/Docker_essentials/docker_watchdog.sh +++ b/Docker_essentials/docker_watchdog.sh @@ -2,24 +2,28 @@ # ----------------------------------------------------------------------------------------------- # --------------------------------- Docker Watchdog -------------------------------------------- # ----------------------------------------------------------------------------------------------- -# Self-healing watchdog for Docker containers — monitors memory, CPU and HTTP responsiveness. -# Restarts containers that exceed configured thresholds using a strike system for CPU and -# responsiveness checks to avoid restarting on brief spikes. +# First line of defense — monitors Docker containers for memory, CPU, HTTP responsiveness, +# and unexpected stops. Restarts containers that exceed thresholds or go offline. +# +# Works alongside system_watchdog.sh: +# docker_watchdog.sh — container level, minimal disruption, tries to self-heal +# system_watchdog.sh — system level, last resort, reboots when healing fails # # Behaviour: -# Memory — immediate restart if hard limit is exceeded -# CPU — strike system, restarts after CPU_FAIL_LIMIT consecutive over-threshold checks -# HTTP — strike system, restarts after RESP_FAIL_LIMIT consecutive failed curl checks +# Memory — immediate restart if hard limit exceeded +# CPU — strike system, restarts after CPU_FAIL_LIMIT consecutive hits +# HTTP — strike system, restarts after RESP_FAIL_LIMIT consecutive failures +# Required — strike system, restarts stopped containers, persistent skip list +# prevents reboot loops, auto-clears when container recovers +# Daemon — immediate notify if Docker daemon is unresponsive # -# Strike system: -# Strikes persist between runs via WATCHDOG_STATE_FILE (/tmp — resets on reboot) -# Strike cadence depends on cron schedule: -# Every 15min + 2 strikes = 30min sustained abuse before restart -# Every 10min + 2 strikes = 20min sustained abuse before restart -# Every 5min + 2 strikes = 10min sustained abuse before restart +# Strike cadence depends on cron schedule: +# Every 15min + 2 strikes = 30min sustained before restart +# Every 10min + 2 strikes = 20min sustained before restart +# Every 5min + 2 strikes = 10min sustained before restart # -# All configuration lives in Master.conf under the Docker Watchdog section. -# Supports --dry-run to show what would be restarted without taking any action. +# All configuration in Master.conf under Docker Watchdog section. +# Supports --dry-run to show what would happen without acting. # ----------------------------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -29,16 +33,18 @@ source "$SCRIPT_DIR/../common.sh" parse_args "$@" -# Auto-detect total CPU cores for normalisation TOTAL_CORES=$(nproc) +# Persistent skip list — shared with system_watchdog.sh +# Containers in this list are skipped until they recover +SKIP_LIST_FILE="$SYS_WATCHDOG_FAILED_FILE" + # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_GEAR Setup ━━━ # ----------------------------------------------------------------------------------------------- echo "" echo "━━━ $ICON_GEAR Setup ━━━" -# ROOT CHECK if [[ "$EUID" -ne 0 ]]; then error "Must be run as root" exit 1 @@ -47,27 +53,32 @@ fi success "Running as root" info "$ICON_WATCHDOG Watchdog initialising — $TOTAL_CORES cores detected" -# Ensure state file exists touch "$WATCHDOG_STATE_FILE" 2>/dev/null || { error "Cannot create state file: $WATCHDOG_STATE_FILE" exit 1 } +touch "$SKIP_LIST_FILE" 2>/dev/null || { + error "Cannot create skip list: $SKIP_LIST_FILE" + exit 1 +} + # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Status ━━━ # ----------------------------------------------------------------------------------------------- if [[ "$SHOW_STATUS" == true ]]; then echo "" echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" - echo "$ICON_WATCHDOG Containers monitored: ${!WATCHDOG_CONTAINERS[*]}" - echo "$ICON_MEM Soft mem threshold: ${SOFT_MEM_THRESHOLD}% of per-container limit" - echo "$ICON_ZFS CPU soft threshold: ${SOFT_CPU_THRESHOLD}%" - echo "$ICON_ZFS CPU hard threshold: ${HARD_CPU_THRESHOLD}%" - echo "$ICON_RETRY CPU fail limit: ${CPU_FAIL_LIMIT} strikes" - echo "$ICON_PING Resp fail limit: ${RESP_FAIL_LIMIT} strikes" - echo "$ICON_TIME Curl timeout: ${CURL_TIMEOUT}s" - echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" - echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "$ICON_WATCHDOG Monitored: ${!WATCHDOG_CONTAINERS[*]}" + echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}" + echo "$ICON_MEM Soft mem: ${SOFT_MEM_THRESHOLD}% of limit" + echo "$ICON_ZFS CPU soft: ${SOFT_CPU_THRESHOLD}%" + echo "$ICON_ZFS CPU hard: ${HARD_CPU_THRESHOLD}%" + echo "$ICON_RETRY CPU strikes: ${CPU_FAIL_LIMIT}" + echo "$ICON_RETRY Resp strikes: ${RESP_FAIL_LIMIT}" + echo "$ICON_TIME Curl timeout: ${CURL_TIMEOUT}s" + echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" + echo "$ICON_GEAR Dry Run: $DRY_RUN" echo "━━━━━━━━━━━━━━━━━━━━━━━" exit 0 fi @@ -76,19 +87,13 @@ fi # ----------------------------------------------------------------------------------------------- # STATE HELPERS -# Reads and writes per-container strike counts to the state file. -# State file format: container:metric:count # ----------------------------------------------------------------------------------------------- -# Returns current strike count for a container/metric pair. -# Usage: get_strikes "Emby" "CPU" get_strikes() { local container="$1" metric="$2" grep -E "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f3 } -# Sets strike count for a container/metric pair. -# Usage: set_strikes "Emby" "CPU" 2 set_strikes() { local container="$1" metric="$2" count="$3" grep -vE "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null > "${WATCHDOG_STATE_FILE}.tmp" @@ -96,19 +101,83 @@ set_strikes() { mv "${WATCHDOG_STATE_FILE}.tmp" "$WATCHDOG_STATE_FILE" } +# ----------------------------------------------------------------------------------------------- +# SKIP LIST HELPERS +# Container skip list — persistent across reboots via /boot/ +# Auto-clears entries when container is found running again. +# ----------------------------------------------------------------------------------------------- + +is_in_skip_list() { + local container="$1" + grep -qE "^${container}$" "$SKIP_LIST_FILE" 2>/dev/null +} + +add_to_skip_list() { + local container="$1" + if ! is_in_skip_list "$container"; then + echo "$container" >> "$SKIP_LIST_FILE" + warn "$ICON_WATCHDOG $container added to persistent skip list" + notify "$container added to watchdog skip list on $(hostname) — manual check recommended" "Docker Watchdog" "warning" + fi +} + +remove_from_skip_list() { + local container="$1" + grep -vE "^${container}$" "$SKIP_LIST_FILE" 2>/dev/null > "${SKIP_LIST_FILE}.tmp" + mv "${SKIP_LIST_FILE}.tmp" "$SKIP_LIST_FILE" + success "$ICON_WATCHDOG $container recovered — removed from skip list" + notify "$container recovered and removed from watchdog skip list on $(hostname)" "Docker Watchdog" "normal" +} + +# ----------------------------------------------------------------------------------------------- +# SKIP LIST AUTO-HEAL CHECK +# On every run check if any skipped containers are now running. +# If running remove from skip list — could have recovered after reboot or manual fix. +# ----------------------------------------------------------------------------------------------- +check_skip_list_recovery() { + [[ ! -s "$SKIP_LIST_FILE" ]] && return + + info "$ICON_WATCHDOG Checking skip list for recovered containers..." + + while IFS= read -r container; do + [[ -z "$container" ]] && continue + + STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") + + if [[ "$STATUS" == "true" ]]; then + remove_from_skip_list "$container" + else + log "$container still not running — remains on skip list" + fi + done < "$SKIP_LIST_FILE" +} + +# ----------------------------------------------------------------------------------------------- +# DOCKER DAEMON HEALTH CHECK +# Verifies Docker daemon is responding before attempting any container operations. +# A hung daemon means all checks will fail — notify immediately and exit. +# ----------------------------------------------------------------------------------------------- +check_docker_daemon() { + info "$ICON_CONTAINERS Checking Docker daemon..." + + if ! timeout 10 docker ps >/dev/null 2>&1; then + error "Docker daemon is not responding" + notify "Docker daemon unresponsive on $(hostname) — immediate attention required" "Docker Watchdog" "warning" + exit 1 + fi + + success "Docker daemon is healthy" +} + # ----------------------------------------------------------------------------------------------- # HELPERS # ----------------------------------------------------------------------------------------------- -# Fetches memory and CPU stats for a container in a single docker stats call. -# Returns: MEM_USAGE|CPU_PERCENT parse_stats() { local container="$1" docker stats --no-stream --format "{{.MemUsage}}|{{.CPUPerc}}" "$container" } -# Converts a memory value and unit to MB. -# Supports KiB, MiB, GiB — returns UNKNOWN for unrecognised units. convert_to_mb() { local value="$1" unit="$2" case "$unit" in @@ -119,32 +188,32 @@ convert_to_mb() { esac } -# Restarts a container locally and sends a notification. -# In dry run mode reports what would happen without acting. +# Restarts a container and sends notification. +# Failed restarts also notify — system_watchdog.sh is the next line of defense. restart_container() { local container="$1" reason="$2" if [[ "$DRY_RUN" == true ]]; then warn "DRY RUN — would restart $container ($reason)" - return + return 0 fi info "Restarting $container ($reason)..." if docker restart "$container" >/dev/null 2>&1; then echo "$ICON_STARTED $container restarted" - notify "$container restarted — $reason" "Docker Watchdog" "warning" - log "Restarted $container — reason: $reason" + notify "$container restarted on $(hostname) — $reason" "Docker Watchdog" "warning" + return 0 else error "Failed to restart $container" - notify "Failed to restart $container — $reason" "Docker Watchdog" "alert" + notify "Failed to restart $container on $(hostname) — $reason" "Docker Watchdog" "warning" + return 1 fi } # ----------------------------------------------------------------------------------------------- # MEMORY CHECK -# Compares current container memory usage against its configured hard limit. -# Restarts immediately if at or above 100% of limit. -# Warns if at or above SOFT_MEM_THRESHOLD % of limit. +# Restarts immediately if container exceeds hard memory limit. +# Warns if approaching soft threshold. # Usage: check_memory "Emby" 16384 # ----------------------------------------------------------------------------------------------- check_memory() { @@ -174,9 +243,8 @@ check_memory() { # ----------------------------------------------------------------------------------------------- # CPU CHECK -# Normalises CPU usage against total core count and applies the strike system. -# Warns at SOFT_CPU_THRESHOLD, strikes at HARD_CPU_THRESHOLD. -# Restarts after CPU_FAIL_LIMIT consecutive strikes — resets strikes on restart or recovery. +# Strike system — restarts after CPU_FAIL_LIMIT consecutive over-threshold checks. +# Resets strikes on recovery or restart. # Usage: check_cpu "Emby" # ----------------------------------------------------------------------------------------------- check_cpu() { @@ -193,24 +261,21 @@ check_cpu() { if (( cpu_int >= HARD_CPU_THRESHOLD )); then ((violations++)) - error "$ICON_ZFS $container CPU ${cpu_int}% — hard threshold hit ($violations/$CPU_FAIL_LIMIT strikes)" + error "$ICON_ZFS $container CPU ${cpu_int}% — hard threshold ($violations/$CPU_FAIL_LIMIT strikes)" set_strikes "$container" "CPU" "$violations" elif (( cpu_int >= SOFT_CPU_THRESHOLD )); then ((violations++)) - warn "$ICON_ZFS $container CPU ${cpu_int}% — soft threshold hit ($violations/$CPU_FAIL_LIMIT strikes)" + warn "$ICON_ZFS $container CPU ${cpu_int}% — soft threshold ($violations/$CPU_FAIL_LIMIT strikes)" set_strikes "$container" "CPU" "$violations" else - if (( violations > 0 )); then - info "$ICON_ZFS $container CPU ${cpu_int}% — recovered, resetting strikes" - else - success "$ICON_ZFS $container CPU ${cpu_int}%" - fi + [[ $violations -gt 0 ]] && info "$ICON_ZFS $container CPU ${cpu_int}% — recovered, resetting strikes" + [[ $violations -eq 0 ]] && success "$ICON_ZFS $container CPU ${cpu_int}%" set_strikes "$container" "CPU" 0 violations=0 fi if (( violations >= CPU_FAIL_LIMIT )); then - error "$ICON_ZFS $container hit CPU limit for $CPU_FAIL_LIMIT consecutive checks" + error "$ICON_ZFS $container CPU limit hit for $CPU_FAIL_LIMIT consecutive checks" restart_container "$container" "sustained CPU abuse" set_strikes "$container" "CPU" 0 fi @@ -218,9 +283,8 @@ check_cpu() { # ----------------------------------------------------------------------------------------------- # RESPONSIVENESS CHECK -# Sends an HTTP request to the container's configured URL. +# Strike system — restarts after RESP_FAIL_LIMIT consecutive failed HTTP checks. # Skips containers with no URL defined in WATCHDOG_CONTAINER_URLS. -# Applies the same strike system as CPU — restarts after RESP_FAIL_LIMIT consecutive failures. # Usage: check_responsiveness "Emby" # ----------------------------------------------------------------------------------------------- check_responsiveness() { @@ -238,11 +302,8 @@ check_responsiveness() { warn "$ICON_PING $container unresponsive at $url ($fails/$RESP_FAIL_LIMIT strikes)" set_strikes "$container" "RESP" "$fails" else - if (( fails > 0 )); then - info "$ICON_PING $container responsive again — resetting strikes" - else - success "$ICON_PING $container responsive at $url" - fi + [[ $fails -gt 0 ]] && info "$ICON_PING $container responsive again — resetting strikes" + [[ $fails -eq 0 ]] && success "$ICON_PING $container responsive at $url" set_strikes "$container" "RESP" 0 fails=0 fi @@ -255,54 +316,131 @@ check_responsiveness() { } # ----------------------------------------------------------------------------------------------- -# ━━━ $ICON_WATCHDOG Watchdog Check ━━━ +# REQUIRED CONTAINER CHECK +# Monitors WATCHDOG_REQUIRED_CONTAINERS for unexpected stops. +# Strike system — attempts restart on each strike. +# After strike limit hit — adds to persistent skip list and notifies system_watchdog handoff. +# Skip list auto-clears at start of each run if container has recovered. +# Usage: check_required_containers +# ----------------------------------------------------------------------------------------------- +check_required_containers() { + [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -eq 0 ]] && return + + info "$ICON_CONTAINERS Checking required containers..." + + for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do + [[ -z "$container" ]] && continue + + # Skip if on persistent skip list + if is_in_skip_list "$container"; then + warn "$ICON_NOT_RUNNING $container is on skip list — skipping until recovered" + continue + fi + + STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") + + if [[ "$STATUS" == "true" ]]; then + success "$ICON_RUNNING $container is running" + set_strikes "$container" "STOP" 0 + continue + fi + + if [[ "$STATUS" == "unknown" ]]; then + warn "$container not found on this host — skipping" + continue + fi + + # Container is stopped — apply strike + local strikes + strikes=$(get_strikes "$container" "STOP") + [[ -z "$strikes" ]] && strikes=0 + ((strikes++)) + + warn "$ICON_NOT_RUNNING $container is stopped ($strikes/$SYS_WATCHDOG_STRIKE_LIMIT strikes)" + set_strikes "$container" "STOP" "$strikes" + + # Attempt restart on each strike + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would attempt restart of $container" + else + if restart_container "$container" "unexpected stop"; then + set_strikes "$container" "STOP" 0 + else + # Restart failed + if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then + error "$container failed to restart after $SYS_WATCHDOG_STRIKE_LIMIT attempts" + add_to_skip_list "$container" + set_strikes "$container" "STOP" 0 + notify "$container handed off to system_watchdog on $(hostname) — added to skip list" "Docker Watchdog" "warning" + fi + fi + fi + done +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_WATCHDOG Watchdog Run ━━━ # ----------------------------------------------------------------------------------------------- echo "" -echo "━━━ $ICON_WATCHDOG Watchdog Check — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "━━━ $ICON_WATCHDOG Watchdog Run — $(date '+%Y-%m-%d %H:%M:%S') ━━━" echo "" -SKIPPED=() -RESTARTED=0 - START=$(date +%s) +SKIPPED=() -for container in "${!WATCHDOG_CONTAINERS[@]}"; do - echo "━━━ $ICON_CONTAINERS $container ━━━" +# Daemon check first — if daemon is down nothing else works +check_docker_daemon - # Verify container exists - if ! docker inspect "$container" &>/dev/null; then - warn "$container not found on this host — skipping" - SKIPPED+=("$container") - echo "" - continue - fi - - # Verify container is running - if ! docker ps --filter "name=^/${container}$" --format "{{.Names}}" | grep -qw "$container"; then - warn "$ICON_NOT_RUNNING $container is not running — skipping" - SKIPPED+=("$container") - echo "" - continue - fi - - check_memory "$container" "${WATCHDOG_CONTAINERS[$container]}" - check_cpu "$container" - check_responsiveness "$container" +# Auto-heal skip list before processing +check_skip_list_recovery +# ----------------------------------------------------------------------------------------------- +# Resource monitoring — WATCHDOG_CONTAINERS +# ----------------------------------------------------------------------------------------------- +if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then echo "" -done + echo "━━━ $ICON_MEM Resource Monitoring ━━━" + + for container in "${!WATCHDOG_CONTAINERS[@]}"; do + echo "" + info "$ICON_CONTAINERS $container" + + if ! docker inspect "$container" &>/dev/null; then + warn "$container not found — skipping" + SKIPPED+=("$container") + continue + fi + + if ! docker ps --filter "name=^/${container}$" --format "{{.Names}}" | grep -qw "$container"; then + warn "$ICON_NOT_RUNNING $container is not running — skipping resource checks" + SKIPPED+=("$container") + continue + fi + + check_memory "$container" "${WATCHDOG_CONTAINERS[$container]}" + check_cpu "$container" + check_responsiveness "$container" + done +fi + +# ----------------------------------------------------------------------------------------------- +# Required container monitoring — WATCHDOG_REQUIRED_CONTAINERS +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_CONTAINERS Required Container Check ━━━" +check_required_containers END=$(date +%s) # ----------------------------------------------------------------------------------------------- # ━━━ $ICON_SUMMARY Summary ━━━ # ----------------------------------------------------------------------------------------------- +echo "" echo "━━━━━ $ICON_SUMMARY WATCHDOG SUMMARY ━━━━━" -echo "$ICON_TIME $(date '+%Y-%m-%d %H:%M:%S')" -echo "$ICON_TIME Duration: $(format_duration $((END - START)))" -echo "$ICON_WATCHDOG Monitored: ${#WATCHDOG_CONTAINERS[@]} containers" -echo "$ICON_NOT_RUNNING Skipped: ${#SKIPPED[@]} containers" -if [[ "$DRY_RUN" == true ]]; then - echo "$ICON_WARN Dry Run: no restarts executed" -fi +echo "$ICON_TIME $(date '+%Y-%m-%d %H:%M:%S')" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "$ICON_WATCHDOG Monitored: ${#WATCHDOG_CONTAINERS[@]} containers" +echo "$ICON_CONTAINERS Required: ${#WATCHDOG_REQUIRED_CONTAINERS[@]} containers" +[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]}" +[[ "$DRY_RUN" == true ]] && echo "$ICON_WARN Dry Run: no actions taken" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" \ No newline at end of file diff --git a/Docker_essentials/docker_weekly_restart.sh b/Docker_essentials/docker_weekly_restart.sh new file mode 100644 index 0000000..bd89b3b --- /dev/null +++ b/Docker_essentials/docker_weekly_restart.sh @@ -0,0 +1,174 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Docker Weekly Restart -------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Restarts or starts specified Docker containers with retry logic. +# Containers are configured in Master.conf under WEEKLY_RESTART_CONTAINERS. +# Uses global RETRY_COUNT and SLEEP from Master.conf for retry behaviour. +# Sends notifications on completion or failure via common.sh notify(). +# Supports --dry-run to preview what would be restarted without taking action. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +if ! command -v docker &>/dev/null; then + error "Docker command not found — check PATH or Docker installation" + notify "Docker weekly restart failed — Docker not found on $(hostname)" "Docker Weekly Restart" "warning" + exit 1 +fi + +success "Docker found" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}" + echo "$ICON_RETRY Retries: $RETRY_COUNT" + echo "$ICON_TIME Sleep: ${SLEEP}s between retries" + echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted" + +# ----------------------------------------------------------------------------------------------- +# FUNCTIONS +# ----------------------------------------------------------------------------------------------- + +# Attempts a docker command up to RETRY_COUNT times with SLEEP seconds between attempts. +# Returns 0 on success, 1 if all attempts fail. +retry_docker() { + local attempt=1 + + while [[ "$attempt" -le "$RETRY_COUNT" ]]; do + info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*" + + if "$@"; then + success "Succeeded on attempt $attempt" + return 0 + else + warn "Attempt $attempt failed" + (( attempt++ )) + [[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP" + fi + done + + error "Command failed after $RETRY_COUNT attempts: $*" + return 1 +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_CONTAINERS Weekly Restart ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_CONTAINERS Weekly Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}" +echo "$ICON_RETRY Retries: $RETRY_COUNT" +echo "" + +START=$(date +%s) +FAILED=() +RESTARTED=() +STARTED=() + +for container in "${WEEKLY_RESTART_CONTAINERS[@]}"; do + echo "━━━ $ICON_CONTAINERS $container ━━━" + + if ! docker inspect "$container" &>/dev/null; then + error "$container does not exist — skipping" + FAILED+=("$container") + echo "" + continue + fi + + STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null) + + case "$STATUS" in + true) + echo "$ICON_RUNNING $container is running — restarting..." + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would restart $container" + else + if retry_docker docker restart "$container"; then + echo "$ICON_STARTED $container restarted" + RESTARTED+=("$container") + else + error "Failed to restart $container after $RETRY_COUNT attempts" + notify "$container failed to restart on $(hostname)" "Docker Weekly Restart" "warning" + FAILED+=("$container") + fi + fi + ;; + false) + echo "$ICON_NOT_RUNNING $container is stopped — starting..." + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would start $container" + else + if retry_docker docker start "$container"; then + echo "$ICON_STARTED $container started" + STARTED+=("$container") + else + error "Failed to start $container after $RETRY_COUNT attempts" + notify "$container failed to start on $(hostname)" "Docker Weekly Restart" "warning" + FAILED+=("$container") + fi + fi + ;; + *) + error "Unknown status for $container: $STATUS" + FAILED+=("$container") + ;; + esac + + echo "" +done + +END=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "━━━━━ $ICON_SUMMARY WEEKLY RESTART SUMMARY ━━━━━" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}" +[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" + +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no changes made" +elif [[ ${#FAILED[@]} -eq 0 ]]; then + echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE" + notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Weekly Restart" "normal" +else + echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed" + notify "Weekly restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Weekly Restart" "warning" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Docker_essentials/media_shares_permissions.sh b/Docker_essentials/media_shares_permissions.sh new file mode 100644 index 0000000..cac5063 --- /dev/null +++ b/Docker_essentials/media_shares_permissions.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Media Permissions Script ----------------------------------- +# ----------------------------------------------------------------------------------------------- +# Applies permissions and ownership to all configured media shares. +# Shares, permissions mode and owner are configured in Master.conf. +# Supports --dry-run to preview what would be changed without making changes. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_PERMS Mode: $PERMISSIONS_MODE" + echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER" + echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_PERMS Media Permissions ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_PERMS Media Permissions ━━━" +echo "$ICON_PERMS Mode: $PERMISSIONS_MODE" +echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER" +echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}" +echo "" + +START=$(date +%s) +FAILED=() +UPDATED=() +SKIPPED=() + +for SHARE in "${MEDIA_PERMISSION_SHARES[@]}"; do + SHARE_NAME=$(basename "$SHARE") + + if [[ ! -d "$SHARE" ]]; then + warn "$ICON_PERMS $SHARE_NAME not found — skipping" + SKIPPED+=("$SHARE_NAME") + continue + fi + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would apply $PERMISSIONS_MODE $PERMISSIONS_OWNER to $SHARE" + continue + fi + + info "$ICON_PERMS Updating $SHARE_NAME..." + + CHMOD_OK=true + CHOWN_OK=true + + chmod -R "$PERMISSIONS_MODE" "$SHARE" 2>/dev/null || CHMOD_OK=false + chown -R "$PERMISSIONS_OWNER" "$SHARE" 2>/dev/null || CHOWN_OK=false + + if [[ "$CHMOD_OK" == true && "$CHOWN_OK" == true ]]; then + echo "$ICON_UNLOCKED $SHARE_NAME — permissions applied" + UPDATED+=("$SHARE_NAME") + else + error "$SHARE_NAME — permissions failed (chmod=$CHMOD_OK chown=$CHOWN_OK)" + FAILED+=("$SHARE_NAME") + fi +done + +END=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━━━ $ICON_SUMMARY MEDIA PERMISSIONS SUMMARY ━━━━━" +echo "$ICON_PERMS Mode: $PERMISSIONS_MODE" +echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +echo "" +[[ ${#UPDATED[@]} -gt 0 ]] && echo " $ICON_UNLOCKED Updated: ${#UPDATED[@]}" +[[ ${#SKIPPED[@]} -gt 0 ]] && echo " $ICON_WARN Skipped: ${#SKIPPED[@]}" +[[ ${#FAILED[@]} -gt 0 ]] && echo " $ICON_ERROR Failed: ${#FAILED[@]} — ${FAILED[*]}" +echo "" +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no changes made" +elif [[ ${#FAILED[@]} -gt 0 ]]; then + echo "$ICON_ERROR Status: $ICON_ERROR SOME SHARES FAILED" + notify "Media permissions failed on $(hostname) — ${FAILED[*]}" "Media Permissions" "warning" +else + echo "$ICON_DONE Status: $ICON_SUCCESS DONE" + notify "Media permissions applied on $(hostname) — ${#UPDATED[@]} shares updated" "Media Permissions" "normal" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Docker_essentials/remove_unwante_files.sh b/Docker_essentials/remove_unwante_files.sh new file mode 100644 index 0000000..42901bf --- /dev/null +++ b/Docker_essentials/remove_unwante_files.sh @@ -0,0 +1,183 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- Media Cleaner Script --------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Removes unwanted files from media share folders using configurable file patterns. +# Supports two profiles: anime and media — each with their own folder list and file patterns. +# Profiles and patterns are configured in Master.conf. +# Supports --dry-run to preview what would be deleted without making changes. +# +# Usage: +# media_cleaner.sh anime — clean anime shares +# media_cleaner.sh media — clean media shares +# media_cleaner.sh anime --dry-run — preview anime clean +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +# ----------------------------------------------------------------------------------------------- +# Separate profile argument from flags +# ----------------------------------------------------------------------------------------------- +PROFILE="" +RAW_ARGS=() + +for ARG in "$@"; do + case "$ARG" in + --*|*=*) RAW_ARGS+=("$ARG") ;; + anime|media) PROFILE="$ARG" ;; + *) RAW_ARGS+=("$ARG") ;; + esac +done + +parse_args "${RAW_ARGS[@]}" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +if ! command -v find >/dev/null 2>&1; then + error "find command not found — check findutils installation" + exit 1 +fi + +if [[ -z "$PROFILE" ]]; then + error "No profile specified. Usage: media_cleaner.sh [--dry-run]" + exit 1 +fi + +# Resolve profile folders and patterns +case "$PROFILE" in + anime) + CLEAN_FOLDERS=("${ANIME_CLEAN_FOLDERS[@]}") + FILE_PATTERNS=("${ANIME_FILE_PATTERNS[@]}") + ;; + media) + CLEAN_FOLDERS=("${MEDIA_CLEAN_FOLDERS[@]}") + FILE_PATTERNS=("${MEDIA_FILE_PATTERNS[@]}") + ;; + *) + error "Unknown profile: $PROFILE — must be anime or media" + exit 1 + ;; +esac + +info "$ICON_GEAR Profile: $PROFILE" +info "$ICON_CLEAN Folders: ${#CLEAN_FOLDERS[@]}" +info "$ICON_TRASH Patterns: ${#FILE_PATTERNS[@]}" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_GEAR Profile: $PROFILE" + echo "$ICON_CLEAN Folders: ${CLEAN_FOLDERS[*]}" + echo "$ICON_TRASH Patterns: ${FILE_PATTERNS[*]}" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted" + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_CLEAN Media Cleaner ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_CLEAN Media Cleaner — $PROFILE ━━━" +echo "" + +START=$(date +%s) +TOTAL_REMOVED=0 +FAILED=() +SKIPPED=() + +for FOLDER in "${CLEAN_FOLDERS[@]}"; do + FOLDER_NAME=$(basename "$FOLDER") + echo "━━━ $ICON_CLEAN $FOLDER_NAME ━━━" + + if [[ ! -d "$FOLDER" ]]; then + warn "$FOLDER_NAME not found — skipping" + SKIPPED+=("$FOLDER_NAME") + echo "" + continue + fi + + # Build find command dynamically from FILE_PATTERNS array + CMD=(find "$FOLDER" -type f \() + for ((i = 0; i < ${#FILE_PATTERNS[@]}; i++)); do + CMD+=(-iname "${FILE_PATTERNS[i]}") + if [[ $i -lt $(( ${#FILE_PATTERNS[@]} - 1 )) ]]; then + CMD+=(-o) + fi + done + CMD+=(\)) + + # Count matching files before acting + FILE_COUNT=$("${CMD[@]}" 2>/dev/null | wc -l) + + if [[ "$FILE_COUNT" -eq 0 ]]; then + success "$FOLDER_NAME — no matching files found" + echo "" + continue + fi + + info "$ICON_TRASH $FILE_COUNT file(s) found in $FOLDER_NAME" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — files that would be deleted:" + "${CMD[@]}" 2>/dev/null | while IFS= read -r f; do + echo " $ICON_TRASH $f" + done + else + CLEAN_CMD=("${CMD[@]}" -exec rm -f {} +) + if "${CLEAN_CMD[@]}" 2>/dev/null; then + success "$FOLDER_NAME — $FILE_COUNT file(s) removed" + TOTAL_REMOVED=$((TOTAL_REMOVED + FILE_COUNT)) + else + error "$FOLDER_NAME — cleanup failed" + FAILED+=("$FOLDER_NAME") + fi + fi + + echo "" +done + +END=$(date +%s) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Summary ━━━ +# ----------------------------------------------------------------------------------------------- +echo "━━━━━ $ICON_SUMMARY MEDIA CLEANER SUMMARY ━━━━━" +echo "$ICON_GEAR Profile: $PROFILE" +echo "$ICON_TIME Duration: $(format_duration $((END - START)))" +[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_WARN Skipped: ${SKIPPED[*]}" +[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}" + +if [[ "$DRY_RUN" == true ]]; then + echo "$ICON_WARN Status: DRY RUN — no files deleted" +elif [[ ${#FAILED[@]} -gt 0 ]]; then + echo "$ICON_ERROR Status: $ICON_ERROR SOME FOLDERS FAILED" + notify "Media cleaner ($PROFILE) failed on $(hostname) — ${FAILED[*]}" "Media Cleaner" "warning" +else + echo "$ICON_TRASH Removed: $TOTAL_REMOVED file(s)" + echo "$ICON_DONE Status: $ICON_SUCCESS DONE" + notify "Media cleaner ($PROFILE) complete on $(hostname) — $TOTAL_REMOVED file(s) removed" "Media Cleaner" "normal" +fi +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +[[ ${#FAILED[@]} -gt 0 ]] && exit 1 +exit 0 \ No newline at end of file diff --git a/Master.conf b/Master.conf index 921973d..d520203 100644 --- a/Master.conf +++ b/Master.conf @@ -1,63 +1,96 @@ #!/bin/bash -# ---------------------------------------------------------------------------------------------- -# ---------------------------- Master Variables for unRAID scripts ----------------------------- -# ---------------------------------------------------------------------------------------------- +# ============================================================================================== +# ================================= MASTER CONFIGURATION ======================================= +# ============================================================================================== +# All user-facing variables for the unRAID script ecosystem. +# Scripts source this file — edit here, changes apply everywhere on next git pull. +# +# ── INDEX ───────────────────────────────────────────────────────────────────────────────────── +# +# Section Description +# ─────────────────────────────────────────────────────────────────────────────────────────── +# HOST CONFIGURATION Server hostnames and SSH key paths +# LOGGING Enable or disable verbose logging +# NOTIFICATIONS unRAID native and Discord webhook settings +# GIT / REPO Gitea repository and SSH settings +# +# ── RSYNC ────────────────────────────────────────────────────────────────────────────────── +# RSYNC DEFAULTS Global fallback rsync settings +# REMOTE HEALTH CHECKS Rootfs threshold for pre-flight abort +# DAILY SYNC SHARES Media shares synced by daily_sync.sh +# RSYNC PROFILE SYSTEM Per-profile overrides (appdata profiles) +# +# ── DOCKER ESSENTIALS ────────────────────────────────────────────────────────────────────── +# DOCKER DAILY RESTART Containers restarted daily +# DOCKER WEEKLY RESTART Containers restarted weekly +# DOCKER WATCHDOG Container health monitoring — memory, CPU, HTTP +# +# ── UNRAID ESSENTIALS ────────────────────────────────────────────────────────────────────── +# REBOOT User warning delay before scheduled reboot +# MOVER Mover stop timeout +# SYSLOG FILTER Docker veth noise filter file path +# PHP-FPM PHP-FPM max children config +# CLEAR LOGS System log file paths +# +# ── MEDIA ────────────────────────────────────────────────────────────────────────────────── +# MEDIA PERMISSIONS Share list, mode and owner for permissions script +# MEDIA CLEANER Anime and media folder lists and file patterns +# +# ── SYSTEM WATCHDOG ──────────────────────────────────────────────────────────────────────── +# SYSTEM WATCHDOG System health monitoring — last line of defense +# +# ============================================================================================== # ━━━ Host Configuration ━━━ -# List the Hostnames of both servers +# Hostnames must match Tailscale machine names exactly — case sensitive HOST1="unRAID-Gmer4Lfe" HOST2="unRAID-Jayred365" -# Assign SSH keys for each host pair (adjust paths as needed) HOST1 from above -# Must have its key added to HOST1_SSH_Key same applies for HOST2 + +# SSH keys for server-to-server rsync — each server needs the other's key authorised HOST1_SSH_KEY="/root/.ssh/Gmer4Lfe-rsync-key" HOST2_SSH_KEY="/root/.ssh/Jayred365-rsync-key" # ━━━ Logging ━━━ - ENABLE_LOGGING=true # false = only echo user-facing messages +# true = verbose [LOG] output in scripts / false = user-facing output only + ENABLE_LOGGING=true # ━━━ Notifications ━━━ -# unRAID native notification system — set to true to enable -# Configure unRAID to send errors only: Settings → Notification Settings +# unRAID native — configure Settings → Notification Settings for errors/warnings only NOTIFY_UNRAID=true # Discord webhook URL — leave blank to disable DISCORD_WEBHOOK="" -# ━━━ Git, Pull & Execute Script ━━━ -REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git" -TARGET_DIR="/mnt/user/appdata/unraid_scripts" -GITEA_SSH_KEY="/root/.ssh/id_gitea_rsync" -SSH_PORT=221 +# ━━━ Git / Repo ━━━ + REPO_SSH="git@192.168.50.2:FailedProxy/Unraid_Scripts.git" + TARGET_DIR="/mnt/user/appdata/unraid_scripts" + GITEA_SSH_KEY="/root/.ssh/id_gitea_rsync" + SSH_PORT=221 -# ━━━ Rsync Script Defaults ━━━ -# These are the fallback values used when no matching profile is found. -# Any share whose directory basename does not match a profile key below -# will use these globals for all rsync behaviour. - BW_LIMIT=12500 -# Retry logic - RETRY_COUNT=3 -# Sleep between retries (seconds) - SLEEP=300 -# Container start/stop/restart before and after rsync - CRITICAL_CONTAINER_NAMES=() -# Containers that need delayed before starting - DELAYED_CONTAINERS=() -# Delay in seconds between starting containers, useful for things like Authelia - CONTAINER_DELAY=5 -# Directories to exclude during transfer - EXCLUDE_DIRS=() -# Default global rsync options +# ============================================================================================== +# ── RSYNC ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Rsync Defaults ━━━ +# Global fallback values — used when no profile match is found for a share. +# Shares in DAILY_SYNC_SHARES always use these globals (no profile defined). + BW_LIMIT=12500 # network speed limit KB/s + RETRY_COUNT=3 # number of retry attempts on failure + SLEEP=300 # seconds between retries + CRITICAL_CONTAINER_NAMES=() # containers to stop before rsync + DELAYED_CONTAINERS=() # containers needing delay before start + CONTAINER_DELAY=5 # seconds delay before starting delayed containers + EXCLUDE_DIRS=() # directories to exclude from transfer DEFAULT_RSYNC_OPTS=(-av --info=progress2 --human-readable --bwlimit="$BW_LIMIT" --delete --inplace --no-whole-file) # ━━━ Remote Health Checks ━━━ -# Abort if remote rootfs usage is at or above this percentage. -# Protects against rsync writing to rootfs when the remote array is down or drives are missing. -# Recommended: 75 — gives headroom before the server becomes unstable +# Abort rsync if remote rootfs exceeds this percentage. +# Protects against rsync filling rootfs when remote array is down or drives are missing. ROOTFS_WARN=75 # ━━━ Daily Sync Shares ━━━ # Shares synced once daily by Orchestrators/daily_sync.sh +# No profile needed — all fall through to DEFAULT_RSYNC_OPTS above. # Add or remove paths here to manage what gets synced. -# These shares have no profile entry and fall through to DEFAULT_RSYNC_OPTS above. DAILY_SYNC_SHARES=( /mnt/user/Anime_Movies-Old /mnt/user/Anime_Shows-Old @@ -73,81 +106,22 @@ DAILY_SYNC_SHARES=( /mnt/user/Tv_Shows ) -# ━━━ unRAID Essential Scripts ━━━ -# unRAID reboot script user warning time (seconds) - REBOOT_SLEEP=300 -# unRAID mover stop script timeout (seconds) - MOVER_STOP_TIMEOUT=300 -# docker_syslog_filter.sh filter file path - FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" -# php_fpm_max_children.sh php-fpm config file path - PHP_CONF="/etc/php-fpm.d/www.conf" -# php_fpm_max_children.sh max children value - PHP_MAX_CHILDREN=250 -# clear_logs.sh system log file paths - LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) - -# ━━━ Docker Daily Restart ━━━ -# Containers to restart daily — space-separated, case-sensitive -# These are the same containers as critical-data and other rsync profiles -DAILY_RESTART_CONTAINERS=( - "NginxProxyManager" - "Authelia" - "Dispatcharr-Iptv-Users" - "Dispatcharr" - "Dispatcharr-Basic" - "Code-Server" -) - -# ━━━ Docker Watchdog ━━━ -# Containers to monitor with their memory hard limits in MB -# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240 8GB=8192 6GB=6144 4GB=4096 1GB=1024 -declare -A WATCHDOG_CONTAINERS=( - ["Emby"]=16384 - ["jellyfin_with_request"]=12288 - ["LidaTube"]=6144 - ["Tdarr"]=6144 - ["Code-Server"]=1024 -) - -# Containers to check for HTTP responsiveness — omit a container to skip its check -declare -A WATCHDOG_CONTAINER_URLS=( - ["Emby"]="http://localhost:8096" - ["Jellyfin-Gmer4Lfe"]="http://localhost:8095" -) - -# State file for tracking CPU and responsiveness strikes between runs -# Lives in /tmp — resets on reboot which is correct behaviour for strike tracking - WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" - -# CPU thresholds — normalised against total core count automatically at runtime - SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU - HARD_CPU_THRESHOLD=85 # strike at this % of total system CPU - CPU_FAIL_LIMIT=2 # consecutive strikes before restart - -# Memory threshold - SOFT_MEM_THRESHOLD=80 # warn at this % of per-container hard limit - -# Responsiveness check settings - RESP_FAIL_LIMIT=2 # consecutive failures before restart - CURL_TIMEOUT=5 # seconds before curl gives up - -# ━━━ Profile System ━━━ +# ━━━ Rsync Profile System ━━━ # Profiles are matched by directory basename (lowercased). # Example: /mnt/user/appdata-Failover/Arrs_Stack → profile key = arrs_stack # # How fallthrough works: -# - If a key exists in a profile array, that value is used -# - If a key is missing, the global default above is used instead +# - If a key exists in a profile array that value is used +# - If a key is missing the global default above is used instead # - Shares in DAILY_SYNC_SHARES have no profile and always use globals # # To add a new profile: # 1. Add a key to each array below with your chosen profile name # 2. Call rsync.sh with a directory whose basename matches that key -# 3. Any array you omit will fall back to its global default +# 3. Any array you omit falls back to its global default # # Note: PROFILE_RSYNC_OPTS does NOT inherit from DEFAULT_RSYNC_OPTS — -# if you define it for a profile you must list all desired options explicitly +# list all desired options explicitly if you define a profile entry # SPACE-SEPARATED STRINGS declare -A PROFILE_RSYNC_OPTS=( @@ -191,7 +165,7 @@ declare -A PROFILE_CRITICAL_CONTAINER_NAMES=( [emby]="" ) -# SPACE-SEPARATED STRINGS — containers that need a delay before starting +# SPACE-SEPARATED STRINGS — containers needing delay before starting declare -A PROFILE_DELAYED_CONTAINERS=( [arrs_stack]="" [critical-data]="Authelia" @@ -216,6 +190,230 @@ declare -A PROFILE_EXCLUDE_DIRS=( [important-data]="logs *.tmp" [emby]="logs *.tmp" ) -# ---------------------------------------------------------------------------------------------- -# ---------------------- End Of User Variables, Please adjust above as needed ------------------ -# ---------------------------------------------------------------------------------------------- \ No newline at end of file + +# ============================================================================================== +# ── DOCKER ESSENTIALS ───────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Docker Daily Restart ━━━ +# Containers restarted every day — case-sensitive names +DAILY_RESTART_CONTAINERS=( + "NginxProxyManager" + "Authelia" + "Dispatcharr-Iptv-Users" + "Dispatcharr" + "Dispatcharr-Basic" + "Code-Server" +) + +# ━━━ Docker Weekly Restart ━━━ +# Containers restarted once per week — case-sensitive names +WEEKLY_RESTART_CONTAINERS=( + "NginxProxyManager" + "Authelia" + "Dispatcharr-Iptv-Users" + "Dispatcharr" + "Dispatcharr-Basic" + "Code-Server" +) + +# ━━━ Docker Watchdog ━━━ +# First line of defense — monitors and restarts unhealthy containers. +# Runs on a cron schedule (recommended every 15 minutes). +# Strike system prevents restarts on brief spikes. + +# Containers to monitor with memory hard limits in MB +# 20GB=20480 16GB=16384 14GB=14336 12GB=12288 10GB=10240 +# 8GB=8192 6GB=6144 4GB=4096 2GB=2048 1GB=1024 +declare -A WATCHDOG_CONTAINERS=( + ["Emby"]=16384 + ["LidaTube"]=6144 + ["Tdarr"]=6144 + ["Code-Server"]=1024 +) + +# Containers to check HTTP responsiveness — omit to skip +declare -A WATCHDOG_CONTAINER_URLS=( + ["Emby"]="http://localhost:8096" +) + +# Containers that should always be running — monitored for unexpected stops +# Strike system used — persistent skip list prevents reboot loops +WATCHDOG_REQUIRED_CONTAINERS=( + "NginxProxyManager" + "Authelia" + "Emby" +) + +# Strike state file — /tmp resets on reboot, correct for strike tracking + WATCHDOG_STATE_FILE="/tmp/container_watchdog_state.db" + +# CPU thresholds — normalised against total core count at runtime + SOFT_CPU_THRESHOLD=80 # warn at this % of total system CPU + HARD_CPU_THRESHOLD=90 # strike at this % of total system CPU + CPU_FAIL_LIMIT=2 # consecutive strikes before restart + +# Memory threshold + SOFT_MEM_THRESHOLD=80 # warn at this % of per-container hard limit + +# Responsiveness check + RESP_FAIL_LIMIT=2 # consecutive failures before restart + CURL_TIMEOUT=5 # seconds before curl gives up + +# ============================================================================================== +# ── UNRAID ESSENTIALS ───────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Reboot ━━━ +# User warning delay before scheduled reboot (seconds) + REBOOT_SLEEP=300 + +# ━━━ Mover ━━━ +# Timeout before stopping the mover (seconds) + MOVER_STOP_TIMEOUT=300 + +# ━━━ Syslog Filter ━━━ +# Path for the rsyslog Docker noise filter file + FILTER_FILE="/etc/rsyslog.d/ignore-docker-veth.conf" + +# ━━━ PHP-FPM ━━━ +# PHP-FPM config file path and max children value + PHP_CONF="/etc/php-fpm.d/www.conf" + PHP_MAX_CHILDREN=250 + +# ━━━ Clear Logs ━━━ +# System log files to clear on each run + LOG_FILES=(/var/log/syslog /var/log/messages /var/log/dmesg) + +# ============================================================================================== +# ── MEDIA ───────────────────────────────────────────────────────────────────────────────────── +# ============================================================================================== + +# ━━━ Media Permissions ━━━ +# Mode and owner applied recursively to all listed shares + PERMISSIONS_MODE="777" + PERMISSIONS_OWNER="nobody:users" + +MEDIA_PERMISSION_SHARES=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Shows + /mnt/user/Anime_Shows-Old + /mnt/user/appcache + /mnt/user/Books + /mnt/user/Downloads + /mnt/user/Games + /mnt/user/Intros + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movie_Recordings + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Music_Videos + /mnt/user/Photo + /mnt/user/Sports + /mnt/user/stand-up_comedy + /mnt/user/Temp_Storage + /mnt/user/Tv_Recordings + /mnt/user/Tv_Shows + /mnt/user/YouTube +) + +# ━━━ Media Cleaner ━━━ +# Two profiles: anime and media — passed as argument to media_cleaner.sh +# Usage: media_cleaner.sh anime or media_cleaner.sh media + +ANIME_CLEAN_FOLDERS=( + /mnt/user/Anime_Movies + /mnt/user/Anime_Movies-Old + /mnt/user/Anime_Shows + /mnt/user/Anime_Shows-Old +) + +MEDIA_CLEAN_FOLDERS=( + /mnt/user/Kids_Movies + /mnt/user/Kids_Tv_Shows + /mnt/user/Movies + /mnt/user/Music + /mnt/user/Sports + /mnt/user/stand-up_comedy + /mnt/user/Tv_Shows +) + +# Anime file patterns — junk files common in anime downloads +ANIME_FILE_PATTERNS=( + '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' + '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' + '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' + '*.log' '*.json' +) + +# Media file patterns — includes *.iso and *.lrc not needed in anime +MEDIA_FILE_PATTERNS=( + '*.sfv' '*.md5' '*.sha1' '*.txt' '*.url' '*.lnk' + '*.rar' '*.zip' '*.info' '*.torrent' '*.sample*' '*.proof*' + '*sync-conflict*' '*.scr' '*.srr' '*.exe' '*.webp' + '*.log' '*.json' '*.iso' '*.lrc' +) + +# ============================================================================================== +# ── SYSTEM WATCHDOG ─────────────────────────────────────────────────────────────────────────── +# ============================================================================================== +# Last line of defense — reboots the system cleanly if it is about to become unstable. +# Designed to run on a cron schedule (recommended every 15-30 minutes). +# Works alongside docker_watchdog.sh — containers first, system second. +# +# Strike system — sustained threshold hits trigger reboot, not single spikes. +# Reboot loop protection — shuts down instead of rebooting if limit hit in window. +# Container skip list — persistent, auto-clears when container recovers. + +# ━━━ System Watchdog State Files ━━━ +# Strike counts — /tmp resets on reboot, correct for strike tracking + SYS_WATCHDOG_STATE_FILE="/tmp/system_watchdog_state.db" +# Persistent container skip list — survives reboots, auto-clears on recovery + SYS_WATCHDOG_FAILED_FILE="/boot/config/system_watchdog_failed.db" +# Reboot timestamp log — survives reboots for loop detection + SYS_WATCHDOG_REBOOT_LOG="/boot/config/system_watchdog_reboots.db" + +# ━━━ Strike and Reboot Loop Settings ━━━ +# Consecutive threshold hits before triggering reboot + SYS_WATCHDOG_STRIKE_LIMIT=2 +# Maximum reboots allowed within the window before shutdown instead + SYS_WATCHDOG_REBOOT_LIMIT=3 +# Window in hours — controls both reboot count window AND rolling log purge +# 12 = entries older than 12hrs purge automatically / 24 = entries older than 24hrs purge + SYS_WATCHDOG_REBOOT_WINDOW_HRS=12 + +# ━━━ Thresholds ━━━ +# Set at "about to fall over" levels — not just high usage + SYS_WATCHDOG_ROOTFS_PCT=95 # rootfs usage % before strike + SYS_WATCHDOG_LOG_PCT=95 # /var/log usage % before strike + SYS_WATCHDOG_MEM_GB=4 # free RAM in GB below which strikes + SYS_WATCHDOG_ARC_PINNED_PCT=98 # ZFS ARC % of max before reclaim attempt + SYS_WATCHDOG_ARC_RELEASE_PCT=95 # ZFS ARC % after reclaim that still triggers + SYS_WATCHDOG_LOAD_MULTIPLIER=32 # strike if load avg > cores x this value + SYS_WATCHDOG_ZOMBIE_LIMIT=50 # strike if zombie process count exceeds this + SYS_WATCHDOG_CPU_TEMP_MAX=95 # degrees C — tjmax varies by CPU + +# ━━━ Check Toggles ━━━ +# true = run this check / false = skip entirely + SYS_WATCHDOG_CHECK_ROOTFS=true + SYS_WATCHDOG_CHECK_LOG=true + SYS_WATCHDOG_CHECK_RAM=true + SYS_WATCHDOG_CHECK_ARC=true + SYS_WATCHDOG_CHECK_CPU_TEMP=true + SYS_WATCHDOG_CHECK_LOAD=true + SYS_WATCHDOG_CHECK_ZOMBIES=true + SYS_WATCHDOG_CHECK_CONTAINERS=true + SYS_WATCHDOG_CHECK_DOCKER_DAEMON=true + +# ━━━ Abort Toggles ━━━ +# true = abort reboot if condition is active / false = reboot anyway +# Default true = conservative / set false to reboot regardless + SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY=true + SYS_WATCHDOG_ABORT_ON_PARITY=false + SYS_WATCHDOG_ABORT_ON_MOVER=false + +# ============================================================================================== +# ──────────────────────── End Of User Variables ─────────────────────────────────────────────── +# ============================================================================================== \ No newline at end of file diff --git a/common.sh b/common.sh index 0cd2a20..9962780 100644 --- a/common.sh +++ b/common.sh @@ -2,7 +2,7 @@ # ----------------------------------------------------------------------------------------------- # ----------------- UNRAID OPS COMMON LIBRARY (STABLE FRAMEWORK v1) ---------------------------- # ----------------------------------------------------------------------------------------------- -# Version: 2.3 +# Version: 2.4 # ----------------------------------------------------------------------------------------------- # Changelog: # v1.0 — Initial stable framework @@ -37,6 +37,9 @@ # v2.3 — ICON_NOTIFY added for notification operations # notify() added — shared notification function supporting unRAID native and Discord # NOTIFY_UNRAID and DISCORD_WEBHOOK configured in Master.conf +# v2.4 — ICON_CLEAN, ICON_TRASH added for media cleaner operations +# ICON_PERMS, ICON_UNLOCKED added for media permissions operations +# ICON_REBOOT_SMART added for smart conditional reboot # ----------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------- @@ -76,7 +79,8 @@ ICON_TIME="⏱️" # duration line # System Operations ICON_MOVER="🔃" # mover operations -ICON_REBOOT="⚡" # server reboot operations +ICON_REBOOT="⚡" # scheduled server reboot +ICON_REBOOT_SMART="🚨" # smart conditional reboot triggered ICON_PLUGIN="🧩" # user scripts plugin operations ICON_PHP="👥" # PHP-FPM operations @@ -85,6 +89,12 @@ ICON_ZFS="📊" # ZFS ARC statistics ICON_MEM="🧠" # memory status ICON_WATCHDOG="🐾" # docker watchdog monitoring operations +# Media Operations +ICON_CLEAN="🧹" # media cleaner operations +ICON_TRASH="🗑️" # files being deleted +ICON_PERMS="🔐" # permissions operation / section header +ICON_UNLOCKED="🔓" # permissions successfully applied to a share + # Notifications ICON_NOTIFY="🔔" # notification operations @@ -116,7 +126,7 @@ log() { # Discord: requires DISCORD_WEBHOOK to be set to a valid webhook URL. # Severity levels: normal, warning, alert — maps to unRAID notification severity. # Usage: notify "message" "subject" "severity" -# notify "Rsync failed: Movies" "Rsync Alert" "alert" +# notify "Rsync failed: Movies" "Rsync Alert" "warning" # notify "Daily sync complete" "Daily Sync" "normal" # ----------------------------------------------------------------------------------------------- notify() { @@ -139,8 +149,6 @@ notify() { # Discord webhook notification if [[ -n "${DISCORD_WEBHOOK:-}" ]]; then - local hostname - hostname=$(hostname) local payload payload=$(printf '{"content": "%s — **%s**\\n%s"}' \ "$ICON_NOTIFY" "$subject" "$message") diff --git a/unRAID_Essentials/system_watchdog.sh b/unRAID_Essentials/system_watchdog.sh new file mode 100644 index 0000000..17bf06e --- /dev/null +++ b/unRAID_Essentials/system_watchdog.sh @@ -0,0 +1,495 @@ +#!/bin/bash +# ----------------------------------------------------------------------------------------------- +# --------------------------------- System Watchdog -------------------------------------------- +# ----------------------------------------------------------------------------------------------- +# Last line of defense — reboots the system cleanly if it is about to become unstable. +# Works alongside docker_watchdog.sh which handles container-level healing first. +# +# Checks (all toggleable in Master.conf): +# rootfs usage — high rootfs fills rapidly when array is down, crash imminent +# /var/log usage — log spam can fill rootfs, indicates something is broken +# free RAM — critically low RAM means OOM or swap imminent +# ZFS ARC pinned — ARC not releasing after reclaim means memory is stuck +# CPU temperature — sustained tjmax causes throttling or kernel panic +# load average — sustained high load means something is stuck or runaway +# zombie processes — large zombie count indicates serious process management failure +# Docker daemon — unresponsive daemon means containers cannot be managed +# Required containers — stopped containers that should be running (after watchdog skip list) +# +# Abort conditions (toggleable — true = abort, false = reboot anyway): +# ZFS pool unhealthy — reboot with bad pool risks data loss +# Parity running — aborting parity is better than crashing mid-check +# Mover running — aborting move is better than crashing mid-move +# +# Reboot loop protection: +# Tracks reboot timestamps in persistent log on /boot/ +# Rolling window — old entries purge automatically after SYS_WATCHDOG_REBOOT_WINDOW_HRS +# If reboot count hits SYS_WATCHDOG_REBOOT_LIMIT in window → shutdown instead of reboot +# +# All configuration in Master.conf under System Watchdog section. +# Supports --dry-run to show triggered conditions without rebooting. +# ----------------------------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$SCRIPT_DIR/../Master.conf" +source "$SCRIPT_DIR/../common.sh" + +parse_args "$@" + +# Convert window hours to seconds for internal use +SYS_WATCHDOG_REBOOT_WINDOW=$(( SYS_WATCHDOG_REBOOT_WINDOW_HRS * 3600 )) + +TOTAL_CORES=$(nproc) + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_GEAR Setup ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_GEAR Setup ━━━" + +if [[ "$EUID" -ne 0 ]]; then + error "Must be run as root" + exit 1 +fi + +success "Running as root" + +# Ensure state and persistent files exist +touch "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null || { + error "Cannot create state file: $SYS_WATCHDOG_STATE_FILE" + exit 1 +} + +touch "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || { + error "Cannot create reboot log: $SYS_WATCHDOG_REBOOT_LOG" + exit 1 +} + +touch "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null || { + error "Cannot create failed container list: $SYS_WATCHDOG_FAILED_FILE" + exit 1 +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SUMMARY Status ━━━ +# ----------------------------------------------------------------------------------------------- +if [[ "$SHOW_STATUS" == true ]]; then + echo "" + echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━" + echo "$ICON_HEALTH rootfs threshold: ${SYS_WATCHDOG_ROOTFS_PCT}% toggle: $SYS_WATCHDOG_CHECK_ROOTFS" + echo "$ICON_HEALTH log threshold: ${SYS_WATCHDOG_LOG_PCT}% toggle: $SYS_WATCHDOG_CHECK_LOG" + echo "$ICON_MEM RAM threshold: ${SYS_WATCHDOG_MEM_GB}GB free toggle: $SYS_WATCHDOG_CHECK_RAM" + echo "$ICON_ZFS ARC pinned: ${SYS_WATCHDOG_ARC_PINNED_PCT}% toggle: $SYS_WATCHDOG_CHECK_ARC" + echo "$ICON_ZFS ARC release: ${SYS_WATCHDOG_ARC_RELEASE_PCT}%" + echo "$ICON_GEAR CPU temp max: ${SYS_WATCHDOG_CPU_TEMP_MAX}°C toggle: $SYS_WATCHDOG_CHECK_CPU_TEMP" + echo "$ICON_GEAR Load multiplier: ${SYS_WATCHDOG_LOAD_MULTIPLIER}x cores toggle: $SYS_WATCHDOG_CHECK_LOAD" + echo "$ICON_GEAR Zombie limit: ${SYS_WATCHDOG_ZOMBIE_LIMIT} toggle: $SYS_WATCHDOG_CHECK_ZOMBIES" + echo "$ICON_CONTAINERS Docker daemon: toggle: $SYS_WATCHDOG_CHECK_DOCKER_DAEMON" + echo "$ICON_CONTAINERS Containers: toggle: $SYS_WATCHDOG_CHECK_CONTAINERS" + echo "$ICON_SHIELD Strike limit: $SYS_WATCHDOG_STRIKE_LIMIT" + echo "$ICON_REBOOT_SMART Reboot limit: $SYS_WATCHDOG_REBOOT_LIMIT in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs" + echo "$ICON_ZFS Abort ZFS unhealthy: $SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" + echo "$ICON_GEAR Abort on parity: $SYS_WATCHDOG_ABORT_ON_PARITY" + echo "$ICON_MOVER Abort on mover: $SYS_WATCHDOG_ABORT_ON_MOVER" + echo "$ICON_GEAR Dry Run: $DRY_RUN" + echo "━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no reboot will be executed" + +# ----------------------------------------------------------------------------------------------- +# STATE HELPERS +# ----------------------------------------------------------------------------------------------- + +get_strikes() { + local key="$1" + grep -E "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f2 +} + +set_strikes() { + local key="$1" count="$2" + grep -vE "^${key}:" "$SYS_WATCHDOG_STATE_FILE" 2>/dev/null > "${SYS_WATCHDOG_STATE_FILE}.tmp" + echo "${key}:${count}" >> "${SYS_WATCHDOG_STATE_FILE}.tmp" + mv "${SYS_WATCHDOG_STATE_FILE}.tmp" "$SYS_WATCHDOG_STATE_FILE" +} + +# Returns current strike count, increments and saves, then echoes new count +increment_strikes() { + local key="$1" + local current + current=$(get_strikes "$key") + [[ -z "$current" ]] && current=0 + ((current++)) + set_strikes "$key" "$current" + echo "$current" +} + +reset_strikes() { + local key="$1" + set_strikes "$key" 0 +} + +# ----------------------------------------------------------------------------------------------- +# REBOOT LOG HELPERS +# Tracks reboot timestamps for loop detection. +# Rolling window — entries older than SYS_WATCHDOG_REBOOT_WINDOW are purged automatically. +# ----------------------------------------------------------------------------------------------- + +purge_old_reboots() { + local now + now=$(date +%s) + local cutoff=$(( now - SYS_WATCHDOG_REBOOT_WINDOW )) + + grep -v "^$" "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null | while IFS= read -r ts; do + [[ "$ts" -gt "$cutoff" ]] && echo "$ts" + done > "${SYS_WATCHDOG_REBOOT_LOG}.tmp" + mv "${SYS_WATCHDOG_REBOOT_LOG}.tmp" "$SYS_WATCHDOG_REBOOT_LOG" +} + +count_recent_reboots() { + purge_old_reboots + grep -c "." "$SYS_WATCHDOG_REBOOT_LOG" 2>/dev/null || echo 0 +} + +log_reboot() { + date +%s >> "$SYS_WATCHDOG_REBOOT_LOG" +} + +# ----------------------------------------------------------------------------------------------- +# ABORT CONDITION CHECKS +# Run before any reboot is triggered — abort conditions prevent rebooting +# when it would make things worse than letting the system run. +# ----------------------------------------------------------------------------------------------- + +check_abort_conditions() { + local should_abort=false + + # ZFS pool health + if command -v zpool >/dev/null 2>&1; then + local unhealthy + unhealthy=$(zpool list -H -o health 2>/dev/null | grep -v ONLINE || true) + if [[ -n "$unhealthy" ]]; then + if [[ "$SYS_WATCHDOG_ABORT_ON_ZFS_UNHEALTHY" == true ]]; then + error "$ICON_ZFS ZFS pool unhealthy — aborting reboot (ABORT_ON_ZFS_UNHEALTHY=true)" + notify "System watchdog aborted reboot on $(hostname) — ZFS pool unhealthy" "System Watchdog" "warning" + should_abort=true + else + warn "$ICON_ZFS ZFS pool unhealthy — continuing reboot (ABORT_ON_ZFS_UNHEALTHY=false)" + fi + fi + fi + + # Parity check running + if [[ -f /var/local/emhttp/parity-date.txt ]]; then + if grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null; then + if [[ "$SYS_WATCHDOG_ABORT_ON_PARITY" == true ]]; then + error "$ICON_DISK Parity check running — aborting reboot (ABORT_ON_PARITY=true)" + notify "System watchdog aborted reboot on $(hostname) — parity check running" "System Watchdog" "warning" + should_abort=true + else + warn "$ICON_DISK Parity check running — continuing reboot (ABORT_ON_PARITY=false)" + fi + fi + fi + + # Mover running + if pgrep -f "mover" >/dev/null 2>&1; then + if [[ "$SYS_WATCHDOG_ABORT_ON_MOVER" == true ]]; then + error "$ICON_MOVER Mover is running — aborting reboot (ABORT_ON_MOVER=true)" + notify "System watchdog aborted reboot on $(hostname) — mover running" "System Watchdog" "warning" + should_abort=true + else + warn "$ICON_MOVER Mover is running — continuing reboot (ABORT_ON_MOVER=false)" + fi + fi + + [[ "$should_abort" == true ]] && return 1 + return 0 +} + +# ----------------------------------------------------------------------------------------------- +# STRIKE-BASED CHECK HELPER +# Runs a check function, increments strikes on trigger, resets on clear. +# Returns 0 if strike limit hit (reboot trigger), 1 otherwise. +# Usage: run_strike_check "key" "triggered (true/false)" "description" +# ----------------------------------------------------------------------------------------------- +run_strike_check() { + local key="$1" triggered="$2" description="$3" + + if [[ "$triggered" == true ]]; then + local strikes + strikes=$(increment_strikes "$key") + warn "$description — strike $strikes/$SYS_WATCHDOG_STRIKE_LIMIT" + if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then + error "$description hit strike limit — reboot triggered" + reset_strikes "$key" + return 0 + fi + else + local current + current=$(get_strikes "$key") + if [[ -n "$current" && "$current" -gt 0 ]]; then + info "$description — recovered, resetting strikes" + reset_strikes "$key" + fi + fi + return 1 +} + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_SHIELD Health Checks ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_SHIELD Health Checks — $(date '+%Y-%m-%d %H:%M:%S') ━━━" +echo "" + +TRIGGERS=() + +# rootfs usage +if [[ "$SYS_WATCHDOG_CHECK_ROOTFS" == true ]]; then + info "$ICON_HEALTH Checking rootfs usage..." + ROOTFS_USED=$(df / --output=pcent | tail -1 | tr -d ' %') + TRIGGERED=false + if [[ "$ROOTFS_USED" -ge "$SYS_WATCHDOG_ROOTFS_PCT" ]]; then + error "$ICON_HEALTH rootfs is ${ROOTFS_USED}% full — threshold ${SYS_WATCHDOG_ROOTFS_PCT}%" + TRIGGERED=true + else + success "$ICON_HEALTH rootfs: ${ROOTFS_USED}% used" + fi + run_strike_check "rootfs" "$TRIGGERED" "rootfs ${ROOTFS_USED}%" && TRIGGERS+=("rootfs=${ROOTFS_USED}%") +fi + +# /var/log usage +if [[ "$SYS_WATCHDOG_CHECK_LOG" == true ]]; then + info "$ICON_HEALTH Checking /var/log usage..." + LOG_USED=$(df -P /var/log | awk 'NR==2 {print $5}' | tr -d '%') + TRIGGERED=false + if [[ "$LOG_USED" -ge "$SYS_WATCHDOG_LOG_PCT" ]]; then + error "$ICON_HEALTH /var/log is ${LOG_USED}% full — threshold ${SYS_WATCHDOG_LOG_PCT}%" + TRIGGERED=true + else + success "$ICON_HEALTH /var/log: ${LOG_USED}% used" + fi + run_strike_check "log" "$TRIGGERED" "/var/log ${LOG_USED}%" && TRIGGERS+=("log=${LOG_USED}%") +fi + +# Free RAM +if [[ "$SYS_WATCHDOG_CHECK_RAM" == true ]]; then + info "$ICON_MEM Checking available RAM..." + MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo) + MEM_GB=$((MEM_KB / 1024 / 1024)) + TRIGGERED=false + if [[ "$MEM_GB" -lt "$SYS_WATCHDOG_MEM_GB" ]]; then + error "$ICON_MEM Available RAM: ${MEM_GB}GB — threshold ${SYS_WATCHDOG_MEM_GB}GB" + TRIGGERED=true + else + success "$ICON_MEM Available RAM: ${MEM_GB}GB" + fi + run_strike_check "ram" "$TRIGGERED" "RAM ${MEM_GB}GB free" && TRIGGERS+=("low_ram=${MEM_GB}GB") +fi + +# ZFS ARC pinned +if [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]] && [[ -f /proc/spl/kstat/zfs/arcstats ]]; then + info "$ICON_ZFS Checking ZFS ARC..." + ARC_SIZE=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_MAX=$(awk '/^c_max / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_PCT=$(( ARC_SIZE * 100 / ARC_MAX )) + TRIGGERED=false + + if [[ "$ARC_PCT" -ge "$SYS_WATCHDOG_ARC_PINNED_PCT" ]]; then + warn "$ICON_ZFS ARC at ${ARC_PCT}% — attempting reclaim..." + sync + echo 3 > /proc/sys/vm/drop_caches + sleep 5 + ARC_AFTER=$(awk '/^size / {print $3}' /proc/spl/kstat/zfs/arcstats) + ARC_AFTER_PCT=$(( ARC_AFTER * 100 / ARC_MAX )) + if [[ "$ARC_AFTER_PCT" -ge "$SYS_WATCHDOG_ARC_RELEASE_PCT" ]]; then + error "$ICON_ZFS ARC still ${ARC_AFTER_PCT}% after reclaim — threshold ${SYS_WATCHDOG_ARC_RELEASE_PCT}%" + TRIGGERED=true + else + success "$ICON_ZFS ARC released to ${ARC_AFTER_PCT}% after reclaim" + fi + else + success "$ICON_ZFS ARC: ${ARC_PCT}% of max" + fi + run_strike_check "arc" "$TRIGGERED" "ZFS ARC pinned" && TRIGGERS+=("arc_pinned=${ARC_PCT}%") +elif [[ "$SYS_WATCHDOG_CHECK_ARC" == true ]]; then + info "$ICON_ZFS ZFS arcstats not available — skipping" +fi + +# CPU temperature +if [[ "$SYS_WATCHDOG_CHECK_CPU_TEMP" == true ]]; then + info "$ICON_GEAR Checking CPU temperature..." + CPU_TEMP="" + if command -v sensors >/dev/null 2>&1; then + CPU_TEMP=$(sensors 2>/dev/null | grep -i "Package id 0\|Tctl\|CPU Temp" | awk '{print $NF}' | tr -d '+°C' | head -1) + fi + + if [[ -z "$CPU_TEMP" ]]; then + info "$ICON_GEAR CPU temperature sensor not available — skipping" + else + CPU_TEMP_INT=$(printf "%.0f" "$CPU_TEMP") + TRIGGERED=false + if [[ "$CPU_TEMP_INT" -ge "$SYS_WATCHDOG_CPU_TEMP_MAX" ]]; then + error "$ICON_GEAR CPU temp ${CPU_TEMP_INT}°C — threshold ${SYS_WATCHDOG_CPU_TEMP_MAX}°C" + TRIGGERED=true + else + success "$ICON_GEAR CPU temp: ${CPU_TEMP_INT}°C" + fi + run_strike_check "cpu_temp" "$TRIGGERED" "CPU temp ${CPU_TEMP_INT}°C" && TRIGGERS+=("cpu_temp=${CPU_TEMP_INT}C") + fi +fi + +# Load average +if [[ "$SYS_WATCHDOG_CHECK_LOAD" == true ]]; then + info "$ICON_GEAR Checking load average..." + LOAD=$(awk '{print $1}' /proc/loadavg) + LOAD_INT=$(printf "%.0f" "$LOAD") + LOAD_THRESHOLD=$(( TOTAL_CORES * SYS_WATCHDOG_LOAD_MULTIPLIER )) + TRIGGERED=false + if [[ "$LOAD_INT" -ge "$LOAD_THRESHOLD" ]]; then + error "$ICON_GEAR Load average ${LOAD} — threshold ${LOAD_THRESHOLD} (${TOTAL_CORES} cores x ${SYS_WATCHDOG_LOAD_MULTIPLIER})" + TRIGGERED=true + else + success "$ICON_GEAR Load average: ${LOAD}" + fi + run_strike_check "load" "$TRIGGERED" "load avg ${LOAD}" && TRIGGERS+=("load=${LOAD}") +fi + +# Zombie processes +if [[ "$SYS_WATCHDOG_CHECK_ZOMBIES" == true ]]; then + info "$ICON_GEAR Checking zombie processes..." + ZOMBIE_COUNT=$(ps aux | awk '{print $8}' | grep -c "^Z$" || echo 0) + TRIGGERED=false + if [[ "$ZOMBIE_COUNT" -ge "$SYS_WATCHDOG_ZOMBIE_LIMIT" ]]; then + error "$ICON_GEAR Zombie processes: $ZOMBIE_COUNT — threshold $SYS_WATCHDOG_ZOMBIE_LIMIT" + TRIGGERED=true + else + success "$ICON_GEAR Zombie processes: $ZOMBIE_COUNT" + fi + run_strike_check "zombies" "$TRIGGERED" "zombies ${ZOMBIE_COUNT}" && TRIGGERS+=("zombies=${ZOMBIE_COUNT}") +fi + +# Docker daemon health +if [[ "$SYS_WATCHDOG_CHECK_DOCKER_DAEMON" == true ]]; then + info "$ICON_CONTAINERS Checking Docker daemon..." + TRIGGERED=false + if ! timeout 10 docker ps >/dev/null 2>&1; then + error "$ICON_CONTAINERS Docker daemon is not responding" + TRIGGERED=true + else + success "$ICON_CONTAINERS Docker daemon is healthy" + fi + run_strike_check "docker_daemon" "$TRIGGERED" "Docker daemon unresponsive" && TRIGGERS+=("docker_daemon") +fi + +# Required containers from skip list +if [[ "$SYS_WATCHDOG_CHECK_CONTAINERS" == true ]] && [[ -s "$SYS_WATCHDOG_FAILED_FILE" ]]; then + info "$ICON_CONTAINERS Checking persistent failed containers..." + FAILED_CONTAINERS=() + while IFS= read -r container; do + [[ -z "$container" ]] && continue + STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown") + if [[ "$STATUS" != "true" ]]; then + error "$ICON_NOT_RUNNING $container still not running (from skip list)" + FAILED_CONTAINERS+=("$container") + fi + done < "$SYS_WATCHDOG_FAILED_FILE" + + if [[ ${#FAILED_CONTAINERS[@]} -gt 0 ]]; then + TRIGGERED=true + run_strike_check "failed_containers" "$TRIGGERED" "required containers stopped" && TRIGGERS+=("containers=${FAILED_CONTAINERS[*]}") + fi +fi + +# ----------------------------------------------------------------------------------------------- +# No triggers — exit cleanly +# ----------------------------------------------------------------------------------------------- +if [[ ${#TRIGGERS[@]} -eq 0 ]]; then + echo "" + success "No reboot conditions met — system is healthy" + exit 0 +fi + +# ----------------------------------------------------------------------------------------------- +# ━━━ $ICON_REBOOT_SMART Reboot Triggered ━━━ +# ----------------------------------------------------------------------------------------------- +echo "" +echo "━━━ $ICON_REBOOT_SMART Reboot Triggered ━━━" +for t in "${TRIGGERS[@]}"; do + echo " $ICON_REBOOT_SMART $t" +done +echo "" + +# Check abort conditions before proceeding +if ! check_abort_conditions; then + exit 0 +fi + +# Reboot loop protection — check recent reboot count +RECENT_REBOOTS=$(count_recent_reboots) +info "Recent reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hr window: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT" + +if [[ "$RECENT_REBOOTS" -ge "$SYS_WATCHDOG_REBOOT_LIMIT" ]]; then + error "Reboot limit hit — $RECENT_REBOOTS reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs — shutting down instead" + notify "Reboot loop detected on $(hostname) — shutting down after $RECENT_REBOOTS reboots in ${SYS_WATCHDOG_REBOOT_WINDOW_HRS}hrs — conditions: ${TRIGGERS[*]}" "System Watchdog" "warning" + + if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — would shutdown now" + exit 0 + fi + + sync + /sbin/poweroff + exit 0 +fi + +notify "System watchdog reboot triggered on $(hostname) — conditions: ${TRIGGERS[*]}" "System Watchdog" "warning" + +if [[ "$DRY_RUN" == true ]]; then + warn "DRY RUN — reboot sequence would begin now" + echo "" + echo "━━━━━ $ICON_SUMMARY SYSTEM WATCHDOG SUMMARY ━━━━━" + echo "$ICON_REBOOT_SMART Triggers: ${TRIGGERS[*]}" + echo "$ICON_REBOOT_SMART Recent reboots: $RECENT_REBOOTS / $SYS_WATCHDOG_REBOOT_LIMIT" + echo "$ICON_WARN Status: DRY RUN — no reboot executed" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +fi + +# ----------------------------------------------------------------------------------------------- +# Graceful shutdown sequence +# ----------------------------------------------------------------------------------------------- +info "Logging reboot timestamp..." +log_reboot + +info "Shutting down VMs..." +if command -v virsh >/dev/null 2>&1; then + for VM in $(virsh list --name 2>/dev/null); do + [[ -z "$VM" ]] && continue + info "Shutting down VM: $VM" + virsh shutdown "$VM" >/dev/null 2>&1 + done + info "Waiting 30s for VMs..." + sleep 30 +fi + +info "Stopping Docker containers..." +if command -v docker >/dev/null 2>&1; then + docker ps -q | xargs -r docker stop >/dev/null 2>&1 + success "Docker containers stopped" +fi + +info "Stopping User Scripts..." +pkill -f "/tmp/user.scripts" 2>/dev/null || true + +info "Syncing disks..." +sync +success "Disks synced" + +echo "" +echo "$ICON_REBOOT_SMART Rebooting system NOW..." +sleep 5 +/sbin/reboot \ No newline at end of file