massive update. Master conf split, now modular with a load sceriprt to drive all configs to scripts. with unraid scpecific safeguard tests , and improved standardized ux. including dynamic host detect, who am i who else it there. EVERY SINGLE SCRIPT UPDATED. DEBATING THAT THIS IS ACUALLY V2
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -1,24 +1,64 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Docker Daily Restart ---------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Restarts or starts specified Docker containers with retry logic.
|
||||
# Containers are configured in Master.conf under DAILY_RESTART_CONTAINERS.
|
||||
# Uses global RETRY_COUNT and SLEEP from Master.conf for retry behaviour.
|
||||
# Sends notifications on completion or failure via common.sh notify().
|
||||
# Supports --dry-run to preview what would be restarted without taking action.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= Docker Daily Restart =======================================
|
||||
# ==============================================================================================
|
||||
# Restarts or starts all containers in HOST*_DAILY_RESTART_CONTAINERS.
|
||||
# Called by daily_sync_maintenance.sh via DAILY_MAINTENANCE_SCRIPTS every night at 1am.
|
||||
# Can also be run manually for ad hoc restarts.
|
||||
#
|
||||
# ── WHY DAILY RESTARTS ────────────────────────────────────────────────────────────────────────
|
||||
# Some containers degrade over time without a restart:
|
||||
# Dispatcharr — Live TV scheduler accumulates state and slows down
|
||||
# NginxProxyManager — connection table grows, occasional stale proxy entries
|
||||
# Authelia — session cache benefits from periodic clearing
|
||||
# Daily restart is intentional maintenance, not just housekeeping.
|
||||
#
|
||||
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
|
||||
# Running containers → docker restart (graceful stop + start)
|
||||
# Stopped containers → left stopped — was down intentionally, do not bring back up
|
||||
# Missing containers → logged and skipped — not treated as fatal
|
||||
# Each action uses RETRY_COUNT + SLEEP from master.conf for retry logic.
|
||||
#
|
||||
# The "was running → restart, was stopped → leave stopped" rule is consistent
|
||||
# across the entire ecosystem — container state is always respected.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Dependency ordering — containers restart in dependency-safe order using
|
||||
# HOST*_WATCHDOG_DEPENDENCIES from master_host*.conf. If Authelia depends on
|
||||
# Mariadb + Redis, those restart first with CONTAINER_DELAY before Authelia starts.
|
||||
#
|
||||
# Restart verification — after each restart, container state is checked after a short
|
||||
# settle period. If the container fails to stay running it is marked as failed and
|
||||
# a notification is sent rather than silently passing.
|
||||
#
|
||||
# Timeout protection — all docker commands are wrapped in a 30 second timeout.
|
||||
# A hung Docker daemon cannot cause this script to hang indefinitely.
|
||||
# Timed-out commands are retried per RETRY_COUNT before marking as failed.
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_DAILY_RESTART_CONTAINERS — list of containers to restart daily
|
||||
# Set by detect_hosts() alias → DAILY_RESTART_CONTAINERS used by this script
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# RETRY_COUNT — retry attempts before giving up on a container
|
||||
# SLEEP — seconds between retry attempts
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# docker_daily_restart.sh — normal restart
|
||||
# docker_daily_restart.sh --dry-run — preview without restarting
|
||||
# docker_daily_restart.sh --log — verbose output
|
||||
# docker_daily_restart.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
@@ -26,7 +66,6 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
acquire_lock
|
||||
@@ -36,30 +75,41 @@ if ! command -v docker &>/dev/null; then
|
||||
notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Docker found"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_DAILY_RESTART_CONTAINERS → DAILY_RESTART_CONTAINERS
|
||||
detect_hosts
|
||||
|
||||
if [[ ${#DAILY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
warn "DAILY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart"
|
||||
warn "Check HOST${MY_ID#HOST}_DAILY_RESTART_CONTAINERS in master_host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}"
|
||||
echo "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||
echo "$ICON_TIME Sleep: ${SLEEP}s between retries"
|
||||
echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||
echo "$ICON_TIME Sleep: ${SLEEP}s between retries"
|
||||
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Retries a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
|
||||
# Usage: retry_docker docker restart ContainerName
|
||||
retry_docker() {
|
||||
local attempt=1
|
||||
|
||||
@@ -80,9 +130,132 @@ retry_docker() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CONTAINERS Daily Restart ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Wraps docker commands with a 30 second timeout.
|
||||
# Prevents a hung Docker daemon from causing the script to hang indefinitely.
|
||||
# Usage: docker_cmd docker restart ContainerName
|
||||
DOCKER_TIMEOUT=30
|
||||
docker_cmd() {
|
||||
timeout "$DOCKER_TIMEOUT" "$@"
|
||||
local exit_code=$?
|
||||
if [[ "$exit_code" -eq 124 ]]; then
|
||||
error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*"
|
||||
return 1
|
||||
fi
|
||||
return "$exit_code"
|
||||
}
|
||||
|
||||
# Verifies a container is still running after restart.
|
||||
# Gives the container a short settle period before checking.
|
||||
# Returns 0 if running, 1 if crashed or stopped.
|
||||
RESTART_VERIFY_WAIT=5 # seconds to wait before checking state post-restart
|
||||
verify_running() {
|
||||
local container="$1"
|
||||
sleep "$RESTART_VERIFY_WAIT"
|
||||
local state
|
||||
state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
if [[ "$state" != "true" ]]; then
|
||||
error "$container failed to stay running after restart — may have crashed"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Builds a dependency-safe restart order from DAILY_RESTART_CONTAINERS.
|
||||
# Containers that are dependencies of others restart first.
|
||||
# Returns ordered list in ORDERED_RESTART array.
|
||||
build_restart_order() {
|
||||
ORDERED_RESTART=()
|
||||
local remaining=("${DAILY_RESTART_CONTAINERS[@]}")
|
||||
local placed=()
|
||||
|
||||
# First pass — add dependency containers that appear in our list
|
||||
for container in "${remaining[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local is_dependency=false
|
||||
# Check if this container is a dependency of any other in our list
|
||||
for dep_string in "${WATCHDOG_DEPENDENCIES[@]:-}"; do
|
||||
if [[ "$dep_string" == *"$container"* ]]; then
|
||||
is_dependency=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
# Also check associative array format
|
||||
for dependent in "${!WATCHDOG_DEPENDENCIES[@]:-}"; do
|
||||
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
|
||||
is_dependency=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$is_dependency" == true ]]; then
|
||||
# Check not already placed
|
||||
local already=false
|
||||
for p in "${placed[@]:-}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
||||
if [[ "$already" == false ]]; then
|
||||
ORDERED_RESTART+=("$container")
|
||||
placed+=("$container")
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Second pass — add remaining containers (dependents and independents)
|
||||
for container in "${remaining[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local already=false
|
||||
for p in "${placed[@]:-}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
||||
if [[ "$already" == false ]]; then
|
||||
ORDERED_RESTART+=("$container")
|
||||
placed+=("$container")
|
||||
fi
|
||||
done
|
||||
|
||||
log "Restart order: ${ORDERED_RESTART[*]}"
|
||||
}
|
||||
|
||||
# Checks if a container is a dependent of the previously restarted container.
|
||||
# If so, waits CONTAINER_DELAY before restarting to allow dependency to settle.
|
||||
# Usage: check_dependency_delay "$container" "$last_restarted"
|
||||
check_dependency_delay() {
|
||||
local container="$1"
|
||||
local last="$2"
|
||||
[[ -z "$last" ]] && return
|
||||
|
||||
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
|
||||
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
|
||||
info "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
|
||||
sleep "$CONTAINER_DELAY"
|
||||
fi
|
||||
}
|
||||
|
||||
# Retries a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
|
||||
# Uses docker_cmd wrapper for timeout protection on each attempt.
|
||||
# Usage: retry_docker docker restart ContainerName
|
||||
retry_docker() {
|
||||
local attempt=1
|
||||
|
||||
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
|
||||
info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
||||
|
||||
if docker_cmd "$@"; then
|
||||
success "Succeeded on attempt $attempt"
|
||||
return 0
|
||||
else
|
||||
warn "Attempt $attempt failed"
|
||||
(( attempt++ ))
|
||||
[[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP"
|
||||
fi
|
||||
done
|
||||
|
||||
error "Command failed after $RETRY_COUNT attempts: $*"
|
||||
return 1
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Daily Restart ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Daily Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_CONTAINERS Containers: ${DAILY_RESTART_CONTAINERS[*]}"
|
||||
@@ -92,30 +265,49 @@ echo ""
|
||||
START=$(date +%s)
|
||||
FAILED=()
|
||||
RESTARTED=()
|
||||
STARTED=()
|
||||
SKIPPED=()
|
||||
|
||||
for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
# Build dependency-safe restart order
|
||||
build_restart_order
|
||||
echo "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}"
|
||||
echo ""
|
||||
|
||||
LAST_RESTARTED=""
|
||||
|
||||
for container in "${ORDERED_RESTART[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
error "$container does not exist — skipping"
|
||||
FAILED+=("$container")
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
||||
warn "$container does not exist — skipping"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
|
||||
case "$STATUS" in
|
||||
true)
|
||||
echo "$ICON_RUNNING $container is running — restarting..."
|
||||
|
||||
# Wait if this container depends on the last one restarted
|
||||
check_dependency_delay "$container" "$LAST_RESTARTED"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container"
|
||||
RESTARTED+=("$container")
|
||||
else
|
||||
if retry_docker docker restart "$container"; then
|
||||
echo "$ICON_STARTED $container restarted"
|
||||
RESTARTED+=("$container")
|
||||
# Verify container stayed running after restart
|
||||
if verify_running "$container"; then
|
||||
echo "$ICON_STARTED $container restarted and running ✅"
|
||||
RESTARTED+=("$container")
|
||||
LAST_RESTARTED="$container"
|
||||
else
|
||||
error "$container restarted but crashed immediately"
|
||||
notify "$container crashed after restart on $(hostname)" "Docker Daily Restart" "warning"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
else
|
||||
error "Failed to restart $container after $RETRY_COUNT attempts"
|
||||
notify "$container failed to restart on $(hostname)" "Docker Daily Restart" "warning"
|
||||
@@ -124,20 +316,10 @@ for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
fi
|
||||
;;
|
||||
false)
|
||||
echo "$ICON_NOT_RUNNING $container is stopped — starting..."
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would start $container"
|
||||
else
|
||||
if retry_docker docker start "$container"; then
|
||||
echo "$ICON_STARTED $container started"
|
||||
STARTED+=("$container")
|
||||
else
|
||||
error "Failed to start $container after $RETRY_COUNT attempts"
|
||||
notify "$container failed to start on $(hostname)" "Docker Daily Restart" "warning"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
fi
|
||||
# Container was stopped — leave it stopped
|
||||
# Intentionally stopped containers are not restarted
|
||||
echo "$ICON_NOT_RUNNING $container is stopped — skipping (respecting stopped state)"
|
||||
SKIPPED+=("$container")
|
||||
;;
|
||||
*)
|
||||
error "Unknown status for $container: $STATUS"
|
||||
@@ -150,20 +332,21 @@ done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY RESTART SUMMARY ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||
[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]} (were stopped)"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE"
|
||||
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Daily Restart" "normal"
|
||||
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped (stopped) on $(hostname)" "Docker Daily Restart" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed"
|
||||
notify "Daily restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Daily Restart" "warning"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,56 +1,93 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Docker Watchdog --------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Two-tier self-healing container monitoring system — runs continuously as a background process.
|
||||
# Started by array_start.sh at array start — runs until array stops or SIGTERM received.
|
||||
# ==============================================================================================
|
||||
# ================================= Docker Watchdog ============================================
|
||||
# ==============================================================================================
|
||||
# Two-tier self-healing container monitoring system.
|
||||
# Runs continuously as a background process — started by array_start.sh at array start.
|
||||
# Shuts down cleanly on SIGTERM/SIGINT when array stops.
|
||||
#
|
||||
# Tier 1 — Strict monitoring (configured containers only)
|
||||
# Memory hard limits — immediate restart if exceeded
|
||||
# CPU thresholds — strike system, restart after CPU_FAIL_LIMIT strikes
|
||||
# HTTP responsiveness — strike system, restart after RESP_FAIL_LIMIT strikes
|
||||
# Required containers — must always be running, strike system with skip list
|
||||
# ── TIER 1 — STRICT MONITORING ────────────────────────────────────────────────────────────────
|
||||
# Applies only to explicitly configured containers (HOST*_WATCHDOG_CONTAINERS etc.)
|
||||
#
|
||||
# Tier 2 — Global health scan (all running containers)
|
||||
# Unhealthy status — Docker HEALTHCHECK unhealthy → restart
|
||||
# OOM killed — kernel killed container → restart + notify
|
||||
# Crash loop detection — RestartCount climbing → notify, critical above limit
|
||||
# Dead containers — remove and restart
|
||||
# Unexpected exits — non-zero exit code → restart
|
||||
# Memory hard limits — immediate restart if container exceeds configured MB ceiling
|
||||
# Memory soft threshold — warn at SOFT_MEM_THRESHOLD % of hard limit (no restart)
|
||||
# CPU thresholds — strike system: warn at SOFT_CPU_THRESHOLD, restart after
|
||||
# CPU_FAIL_LIMIT consecutive strikes at HARD_CPU_THRESHOLD
|
||||
# HTTP responsiveness — strike system: restart after RESP_FAIL_LIMIT consecutive failures
|
||||
# Required containers — must always be running; strike system before restart;
|
||||
# skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in window;
|
||||
# auto-clears when container recovers
|
||||
#
|
||||
# Cross-cutting intelligence:
|
||||
# Startup grace period — skip restarts while system is still booting
|
||||
# Dependency ordering — restart database before app
|
||||
# Restart loop protect — stop restarting after X restarts in X hours → skip list
|
||||
# Skip list auto-clear — clears when container recovers
|
||||
# Notification batching — one clean summary per cycle, not one ping per event
|
||||
# Quiet when healthy — only logs when something needs attention
|
||||
# Parity awareness — skips restarts during parity check
|
||||
# ── TIER 2 — GLOBAL HEALTH SCAN ───────────────────────────────────────────────────────────────
|
||||
# Scans ALL running containers when WATCHDOG_SCAN_ALL=true.
|
||||
# Containers in WATCHDOG_SCAN_IGNORE are excluded from Tier 2.
|
||||
#
|
||||
# Continuous loop:
|
||||
# Checks run every DOCKER_WATCHDOG_INTERVAL seconds (default 900 = 15min)
|
||||
# Clean shutdown on SIGTERM/SIGINT — sent by array stop
|
||||
# Variables scoped per-cycle — no state accumulation between cycles
|
||||
# Unhealthy status — Docker HEALTHCHECK unhealthy → safe_restart()
|
||||
# OOM killed — kernel OOM killed → safe_restart() + notify
|
||||
# OOM state tracked per-session to prevent restart loop
|
||||
# Crash loop detection — RestartCount climbing → notify; above WATCHDOG_CRASH_LIMIT
|
||||
# → safe_restart() → skip list if restart limit hit
|
||||
# Dead containers — safe_restart() via remove + start
|
||||
# Unexpected exits — non-zero exit code → safe_restart()
|
||||
#
|
||||
# State files:
|
||||
# WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot)
|
||||
# SYS_WATCHDOG_FAILED_FILE — persistent skip list (/boot — survives reboots)
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection
|
||||
# ── CROSS-CUTTING INTELLIGENCE ────────────────────────────────────────────────────────────────
|
||||
# Startup grace period — no restarts for WATCHDOG_STARTUP_GRACE seconds after boot
|
||||
# Dependency ordering — waits for dependencies before restarting a dependent container
|
||||
# Restart loop protect — skip list after WATCHDOG_CONTAINER_RESTART_LIMIT in rolling window
|
||||
# Skip list auto-clear — clears when container is seen running again
|
||||
# Notification batching — one summary per cycle, not one ping per event
|
||||
# Parity awareness — skips restart actions during parity check
|
||||
# Timeout protection — all docker commands wrapped in timeout — daemon hangs cannot
|
||||
# stall the watchdog and leave containers unmonitored
|
||||
# Docker daemon check — first check every cycle; hung daemon → strike system →
|
||||
# restart daemon via /etc/rc.d/rc.docker → verify recovery
|
||||
# system_watchdog.sh handles escalation if restart fails
|
||||
# Quiet when healthy — only logs when something needs attention (plus heartbeat)
|
||||
#
|
||||
# All configuration in Master.conf under Docker Watchdog section.
|
||||
# Supports --dry-run and --status.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ── STATE FILES ───────────────────────────────────────────────────────────────────────────────
|
||||
# WATCHDOG_STATE_FILE — strike counts (/tmp — resets on reboot, correct)
|
||||
# SYS_WATCHDOG_FAILED_FILE — skip list (/boot — survives reboots, intentional)
|
||||
# WATCHDOG_CONTAINER_RESTART_LOG — restart history for loop detection
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_WATCHDOG_CONTAINERS — memory hard limits per container
|
||||
# HOST*_WATCHDOG_CONTAINER_URLS — HTTP health check URLs
|
||||
# HOST*_WATCHDOG_REQUIRED_CONTAINERS — must always be running
|
||||
# HOST*_WATCHDOG_SCAN_IGNORE — skip in Tier 2 scan
|
||||
# HOST*_WATCHDOG_DEPENDENCIES — dependency ordering for restart decisions
|
||||
# All aliased by detect_hosts() — script uses unprefixed names
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# SOFT_CPU_THRESHOLD / HARD_CPU_THRESHOLD / CPU_FAIL_LIMIT
|
||||
# SOFT_MEM_THRESHOLD
|
||||
# RESP_FAIL_LIMIT / CURL_TIMEOUT
|
||||
# DOCKER_WATCHDOG_INTERVAL
|
||||
# DOCKER_WATCHDOG_HEARTBEAT / DOCKER_WATCHDOG_HEARTBEAT_HOURS
|
||||
# WATCHDOG_SCAN_ALL
|
||||
# WATCHDOG_RESTART_UNHEALTHY / WATCHDOG_RESTART_DEAD / WATCHDOG_RESTART_CRASHED
|
||||
# WATCHDOG_NOTIFY_OOM / WATCHDOG_NOTIFY_CRASHLOOP
|
||||
# WATCHDOG_CRASH_LIMIT
|
||||
# WATCHDOG_STARTUP_GRACE
|
||||
# WATCHDOG_CONTAINER_RESTART_LIMIT / WATCHDOG_CONTAINER_RESTART_WINDOW
|
||||
# WATCHDOG_BATCH_NOTIFY
|
||||
# WATCHDOG_STATE_FILE / SYS_WATCHDOG_FAILED_FILE / WATCHDOG_CONTAINER_RESTART_LOG
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# docker_watchdog.sh — normal start (continuous loop)
|
||||
# docker_watchdog.sh --dry-run — preview without restarting
|
||||
# docker_watchdog.sh --status — show config and exit
|
||||
# docker_watchdog.sh --log — verbose output
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup — runs once at start ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup — runs once at start ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
@@ -58,74 +95,70 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
# Continuous mode — skip gracefully if healthy instance already running
|
||||
acquire_lock "continuous"
|
||||
|
||||
# Select correct per-host watchdog lists — done once at startup
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_WATCHDOG_* arrays
|
||||
detect_hosts
|
||||
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
declare -A WATCHDOG_CONTAINER_URLS
|
||||
for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}"
|
||||
done
|
||||
else
|
||||
WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
declare -A WATCHDOG_CONTAINER_URLS
|
||||
for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}"
|
||||
done
|
||||
fi
|
||||
|
||||
info "Watchdog running as: $LOCAL_SERVER_NAME"
|
||||
info "Check interval: ${DOCKER_WATCHDOG_INTERVAL}s"
|
||||
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
error "Docker not found"
|
||||
error "Docker not found — cannot start watchdog"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Docker found"
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||
|
||||
# Validate unRAID-specific commands used by this script
|
||||
# If rc.docker is missing or changed, daemon restart will fail — better to know now
|
||||
validate_unraid_cmd "/etc/rc.d/rc.docker" "" "" "Docker rc.d script" || warn "rc.docker not found — daemon restart unavailable if needed"
|
||||
|
||||
validate_unraid_cmd "/usr/local/emhttp/plugins/dynamix/scripts/notify" "" "" "unRAID notify script" || warn "unRAID notify script not found — native notifications disabled"
|
||||
|
||||
# Ensure state files exist
|
||||
touch "$WATCHDOG_STATE_FILE" "$WATCHDOG_CONTAINER_RESTART_LOG" \
|
||||
"$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Timeout for all docker commands — prevents hung daemon from stalling the watchdog
|
||||
DOCKER_TIMEOUT=15
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_CONTAINERS Watched: ${!WATCHDOG_CONTAINERS[@]}"
|
||||
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}"
|
||||
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
|
||||
echo "$ICON_WATCHDOG Interval: ${DOCKER_WATCHDOG_INTERVAL}s"
|
||||
echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s"
|
||||
echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||
echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY"
|
||||
echo "$ICON_WATCHDOG Ignore list: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_CONTAINERS Watched: ${!WATCHDOG_CONTAINERS[*]:-none}"
|
||||
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]:-none}"
|
||||
echo "$ICON_WATCHDOG Scan all: $WATCHDOG_SCAN_ALL"
|
||||
echo "$ICON_WATCHDOG Ignore: ${WATCHDOG_SCAN_IGNORE[*]:-none}"
|
||||
echo "$ICON_WATCHDOG Interval: ${DOCKER_WATCHDOG_INTERVAL}s"
|
||||
echo "$ICON_WATCHDOG Startup grace: ${WATCHDOG_STARTUP_GRACE}s"
|
||||
echo "$ICON_WATCHDOG Restart limit: $WATCHDOG_CONTAINER_RESTART_LIMIT in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||
echo "$ICON_WATCHDOG Batch notify: $WATCHDOG_BATCH_NOTIFY"
|
||||
echo "$ICON_WATCHDOG Docker timeout: ${DOCKER_TIMEOUT}s"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
UPTIME_S=$(awk '{print int($1)}' /proc/uptime)
|
||||
echo "$ICON_TIME System uptime: $(format_duration $UPTIME_S)"
|
||||
echo "$ICON_TIME System uptime: $(format_duration $UPTIME_S)"
|
||||
[[ "$UPTIME_S" -lt "$WATCHDOG_STARTUP_GRACE" ]] && \
|
||||
warn "Within startup grace period — restarts suppressed"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# HELPERS — defined once, used every cycle
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ── HELPER FUNCTIONS ──────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Get strike count for a container from state file
|
||||
get_strikes() {
|
||||
grep "^${1}:" "${2}" 2>/dev/null | cut -d: -f2 || echo "0"
|
||||
}
|
||||
|
||||
# Set strike count for a container in state file
|
||||
set_strikes() {
|
||||
local container="$1" count="$2" file="$3"
|
||||
if grep -q "^${container}:" "$file" 2>/dev/null; then
|
||||
@@ -135,10 +168,12 @@ set_strikes() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Check if container is on the persistent skip list
|
||||
is_skipped() {
|
||||
grep -q "^${1}$" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
}
|
||||
|
||||
# Add container to persistent skip list — manual intervention required to recover
|
||||
add_to_skip_list() {
|
||||
local container="$1" reason="$2"
|
||||
if ! is_skipped "$container"; then
|
||||
@@ -148,11 +183,13 @@ add_to_skip_list() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Remove container from skip list — called when container is seen running again
|
||||
remove_from_skip_list() {
|
||||
sed -i "/^${1}$/d" "$SYS_WATCHDOG_FAILED_FILE" 2>/dev/null
|
||||
success "$1 removed from skip list — recovered"
|
||||
warn "$1 recovered — removed from skip list ✅"
|
||||
}
|
||||
|
||||
# Log a restart event to the rolling restart history file
|
||||
log_restart() {
|
||||
local container="$1"
|
||||
local now
|
||||
@@ -160,12 +197,14 @@ log_restart() {
|
||||
local cutoff
|
||||
cutoff=$(date -d "${WATCHDOG_CONTAINER_RESTART_WINDOW} hours ago" '+%Y-%m-%d %H:%M:%S')
|
||||
echo "${container}|${now}" >> "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
# Trim entries older than the rolling window
|
||||
local tmp="${WATCHDOG_CONTAINER_RESTART_LOG}.tmp"
|
||||
awk -F'|' -v cutoff="$cutoff" '$2 >= cutoff' \
|
||||
"$WATCHDOG_CONTAINER_RESTART_LOG" > "$tmp" && \
|
||||
mv "$tmp" "$WATCHDOG_CONTAINER_RESTART_LOG"
|
||||
}
|
||||
|
||||
# Get number of times a container was restarted within the rolling window
|
||||
get_restart_count() {
|
||||
local container="$1"
|
||||
local cutoff
|
||||
@@ -174,13 +213,15 @@ get_restart_count() {
|
||||
'$1==c && $2>=cutoff' "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null | wc -l
|
||||
}
|
||||
|
||||
# Check if all dependencies of a container are currently running.
|
||||
# Returns 0 if all deps running (or no deps), 1 if any dep is down.
|
||||
dependencies_satisfied() {
|
||||
local container="$1"
|
||||
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
|
||||
[[ -z "$deps" ]] && return 0
|
||||
for dep in $deps; do
|
||||
local status
|
||||
status=$(docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null)
|
||||
status=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$dep" 2>/dev/null)
|
||||
if [[ "$status" != "true" ]]; then
|
||||
warn "$container — dependency $dep is not running — skipping restart this cycle"
|
||||
return 1
|
||||
@@ -189,8 +230,18 @@ dependencies_satisfied() {
|
||||
return 0
|
||||
}
|
||||
|
||||
# Safe restart with all guards:
|
||||
# - Restart loop protection (skip list after limit)
|
||||
# - Dependency check (don't restart if deps down)
|
||||
# - Startup grace period (no restarts while booting)
|
||||
# - Dry run support
|
||||
# - Timeout protection on docker restart
|
||||
#
|
||||
# Returns: 0=restarted, 1=skipped, 2=added to skip list
|
||||
safe_restart() {
|
||||
local container="$1" reason="$2"
|
||||
|
||||
# Restart loop protection — skip list if over limit
|
||||
local restart_count
|
||||
restart_count=$(get_restart_count "$container")
|
||||
if [[ "$restart_count" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
|
||||
@@ -198,34 +249,42 @@ safe_restart() {
|
||||
"restarted $restart_count times in ${WATCHDOG_CONTAINER_RESTART_WINDOW}h"
|
||||
return 2
|
||||
fi
|
||||
|
||||
# Dependency check
|
||||
dependencies_satisfied "$container" || return 1
|
||||
|
||||
# Startup grace period
|
||||
local uptime_s
|
||||
uptime_s=$(awk '{print int($1)}' /proc/uptime)
|
||||
if [[ "$uptime_s" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
warn "$container — within startup grace period — skipping restart"
|
||||
warn "$container — within startup grace period (${uptime_s}s < ${WATCHDOG_STARTUP_GRACE}s) — skipping"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container ($reason)"
|
||||
return 0
|
||||
fi
|
||||
info "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]..."
|
||||
if docker restart "$container" >/dev/null 2>&1; then
|
||||
|
||||
log "Restarting $container ($reason) [restart $((restart_count + 1))/$WATCHDOG_CONTAINER_RESTART_LIMIT in window]"
|
||||
if timeout "$DOCKER_TIMEOUT" docker restart "$container" >/dev/null 2>&1; then
|
||||
success "$ICON_STARTED $container restarted"
|
||||
log_restart "$container"
|
||||
return 0
|
||||
else
|
||||
error "Failed to restart $container"
|
||||
error "Failed to restart $container (timeout or error)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Queue a notification event for batch sending at end of cycle
|
||||
queue_notify() {
|
||||
local message="$1" severity="${2:-warning}"
|
||||
NOTIFY_EVENTS+=("${severity}|${message}")
|
||||
log "Queued: $message"
|
||||
}
|
||||
|
||||
# Send all queued notifications — one batched summary or individual per event
|
||||
flush_notify() {
|
||||
[[ ${#NOTIFY_EVENTS[@]} -eq 0 ]] && return
|
||||
if [[ "$WATCHDOG_BATCH_NOTIFY" == "true" ]]; then
|
||||
@@ -251,28 +310,153 @@ flush_notify() {
|
||||
NOTIFY_EVENTS=()
|
||||
}
|
||||
|
||||
# Returns 0 if parity check is currently running
|
||||
is_parity_running() {
|
||||
grep -q "progress" /var/local/emhttp/parity-date.txt 2>/dev/null
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CLEAN SHUTDOWN — trap SIGTERM/SIGINT from array stop
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ── DOCKER DAEMON HEALTH CHECK ────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Checks Docker daemon responsiveness at the start of every cycle.
|
||||
# A hung daemon makes all container operations useless — check first, short-circuit if down.
|
||||
#
|
||||
# Strike system:
|
||||
# Each consecutive failed check adds a strike
|
||||
# At WATCHDOG_DAEMON_STRIKE_LIMIT → attempt daemon restart via rc.docker
|
||||
# After restart → wait WATCHDOG_DAEMON_RESTART_WAIT seconds → verify
|
||||
# If verified → clear strikes, continue cycle ✅
|
||||
# If still hung → notify critical, skip cycle → system_watchdog.sh escalates from here
|
||||
#
|
||||
# Returns: 0 = daemon healthy | 1 = daemon down, skip this cycle
|
||||
|
||||
WATCHDOG_DAEMON_STRIKE_LIMIT=3 # consecutive failed checks before restart attempt
|
||||
WATCHDOG_DAEMON_RESTART_WAIT=30 # seconds to wait after restart before verifying
|
||||
WATCHDOG_DAEMON_STRIKES=0 # persists across cycles — reset when daemon recovers
|
||||
WATCHDOG_DAEMON_RESTARTED=false # tracks if we already attempted restart this session
|
||||
|
||||
# ==============================================================================================
|
||||
# ── SYSTEM WATCHDOG COORDINATION ──────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
# Reads system_watchdog.sh state file to check if a RAM emergency shutdown is active.
|
||||
# During RAM emergency: system_watchdog.sh has stopped non-essential containers to free RAM.
|
||||
# docker_watchdog.sh must not restart them — that would undo the emergency shutdown and
|
||||
# prevent RAM from recovering, creating an infinite restart/shutdown loop.
|
||||
#
|
||||
# Returns:
|
||||
# 0 = normal — run all checks
|
||||
# 1 = RAM emergency active — defer container management this cycle
|
||||
|
||||
check_system_watchdog_state() {
|
||||
# Returns 0 = normal operation | 1 = defer, RAM emergency active
|
||||
local state_file="$SYS_WATCHDOG_STATE_FILE"
|
||||
|
||||
# No state file = system_watchdog not running or not yet written — assume normal
|
||||
[[ ! -f "$state_file" ]] && return 0
|
||||
|
||||
local mem_shutdown
|
||||
mem_shutdown=$(grep "^mem_shutdown_active=" "$state_file" 2>/dev/null | cut -d= -f2)
|
||||
|
||||
[[ "$mem_shutdown" != "true" ]] && return 0
|
||||
|
||||
# ── Stale state guard ─────────────────────────────────────────────────────────────────────
|
||||
# If mem_shutdown_active=true but state file hasn't been updated in > 2 hours,
|
||||
# system_watchdog.sh may have died — don't be silenced forever by a stale flag.
|
||||
local state_mtime now age_seconds stale_limit=7200 # 2 hours
|
||||
state_mtime=$(stat -c %Y "$state_file" 2>/dev/null || echo 0)
|
||||
now=$(date +%s)
|
||||
age_seconds=$(( now - state_mtime ))
|
||||
|
||||
if [[ "$age_seconds" -gt "$stale_limit" ]]; then
|
||||
warn "mem_shutdown_active=true but state file is ${age_seconds}s old — may be stale"
|
||||
warn "system_watchdog.sh may not be running — resuming normal container management"
|
||||
warn "If RAM is still low this will be caught on next system_watchdog.sh cycle"
|
||||
return 0 # Resume normal — don't defer indefinitely on stale state
|
||||
fi
|
||||
|
||||
return 1 # Defer — RAM emergency confirmed and state is fresh
|
||||
}
|
||||
|
||||
check_docker_daemon() {
|
||||
# docker info is more definitive than docker ps for daemon health
|
||||
if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
|
||||
if [[ "$WATCHDOG_DAEMON_STRIKES" -gt 0 ]]; then
|
||||
info "$ICON_STARTED Docker daemon recovered — clearing strikes"
|
||||
queue_notify "Docker daemon recovered on $(hostname)" "normal"
|
||||
WATCHDOG_DAEMON_STRIKES=0
|
||||
WATCHDOG_DAEMON_RESTARTED=false
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
WATCHDOG_DAEMON_STRIKES=$(( WATCHDOG_DAEMON_STRIKES + 1 ))
|
||||
warn "$ICON_WATCHDOG Docker daemon not responding (strike $WATCHDOG_DAEMON_STRIKES/$WATCHDOG_DAEMON_STRIKE_LIMIT)"
|
||||
|
||||
if [[ "$WATCHDOG_DAEMON_STRIKES" -lt "$WATCHDOG_DAEMON_STRIKE_LIMIT" ]]; then
|
||||
warn "Skipping monitoring cycle — waiting for daemon to recover"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "$WATCHDOG_DAEMON_RESTARTED" == true ]]; then
|
||||
error "Docker daemon still unresponsive after restart attempt"
|
||||
error "system_watchdog.sh will handle further escalation"
|
||||
queue_notify "Docker daemon hung on $(hostname) — restart failed — manual intervention needed" "critical"
|
||||
flush_notify
|
||||
return 1
|
||||
fi
|
||||
|
||||
error "Docker daemon unresponsive — attempting restart"
|
||||
notify "Docker daemon hung on $(hostname) — attempting restart" "Docker Watchdog" "warning"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart Docker daemon via /etc/rc.d/rc.docker restart"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Restart daemon — unRAID uses rc.d scripts, not systemd
|
||||
WATCHDOG_DAEMON_RESTARTED=true
|
||||
if /etc/rc.d/rc.docker restart >/dev/null 2>&1; then
|
||||
info "Docker daemon restart issued — waiting ${WATCHDOG_DAEMON_RESTART_WAIT}s..."
|
||||
sleep "$WATCHDOG_DAEMON_RESTART_WAIT"
|
||||
|
||||
if timeout "$DOCKER_TIMEOUT" docker info >/dev/null 2>&1; then
|
||||
success "Docker daemon restarted successfully ✅"
|
||||
notify "Docker daemon restarted successfully on $(hostname)" "Docker Watchdog" "normal"
|
||||
WATCHDOG_DAEMON_STRIKES=0
|
||||
WATCHDOG_DAEMON_RESTARTED=false
|
||||
return 0
|
||||
else
|
||||
error "Docker daemon did not recover after restart"
|
||||
queue_notify "Docker daemon restart failed on $(hostname) — system_watchdog.sh escalating" "critical"
|
||||
flush_notify
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
error "Failed to issue Docker daemon restart — /etc/rc.d/rc.docker not found or failed"
|
||||
queue_notify "Docker daemon restart command failed on $(hostname) — manual intervention needed" "critical"
|
||||
flush_notify
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ── CLEAN SHUTDOWN ────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
WATCHDOG_RUNNING=true
|
||||
|
||||
cleanup() {
|
||||
echo ""
|
||||
info "Docker watchdog received shutdown signal — stopping cleanly"
|
||||
warn "Docker watchdog received shutdown signal — stopping cleanly"
|
||||
WATCHDOG_RUNNING=false
|
||||
exit 0
|
||||
}
|
||||
|
||||
trap cleanup SIGTERM SIGINT
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ CONTINUOUS MONITORING LOOP ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
info "Docker watchdog started — checking every ${DOCKER_WATCHDOG_INTERVAL}s"
|
||||
# ==============================================================================================
|
||||
# ━━━ Continuous Monitoring Loop ━━━
|
||||
# ==============================================================================================
|
||||
info "$ICON_WATCHDOG Docker watchdog started — $MY_ID — checking every ${DOCKER_WATCHDOG_INTERVAL}s"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
CYCLE=0
|
||||
@@ -281,59 +465,79 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
((CYCLE++))
|
||||
CYCLE_START=$(date +%s)
|
||||
|
||||
# Re-source Master.conf each cycle — picks up any config changes without restart
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
# ── Re-source config each cycle ──────────────────────────────────────────────────────────
|
||||
# Picks up config changes (new containers, threshold adjustments) without restart.
|
||||
# detect_hosts() re-aliases all HOST*_WATCHDOG_* arrays after re-source.
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
detect_hosts
|
||||
|
||||
# Rebuild per-host lists after re-source in case they changed
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
WATCHDOG_REQUIRED_CONTAINERS=("${HOST1_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
for key in "${!HOST1_WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
WATCHDOG_CONTAINER_URLS["$key"]="${HOST1_WATCHDOG_CONTAINER_URLS[$key]}"
|
||||
done
|
||||
else
|
||||
WATCHDOG_REQUIRED_CONTAINERS=("${HOST2_WATCHDOG_REQUIRED_CONTAINERS[@]}")
|
||||
for key in "${!HOST2_WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
WATCHDOG_CONTAINER_URLS["$key"]="${HOST2_WATCHDOG_CONTAINER_URLS[$key]}"
|
||||
done
|
||||
fi
|
||||
|
||||
# Per-cycle variables — cleared each iteration, no accumulation
|
||||
# ── Per-cycle state — cleared each iteration ──────────────────────────────────────────────
|
||||
NOTIFY_EVENTS=()
|
||||
T1_RESTARTS=0
|
||||
T1_WARNINGS=0
|
||||
T2_RESTARTS=0
|
||||
T2_WARNINGS=0
|
||||
|
||||
# Rebuild ignore map each cycle in case config was updated
|
||||
# OOM handled set — tracks containers already handled for OOM this cycle
|
||||
# Prevents restart loop from OOMKilled flag persisting after restart
|
||||
declare -A OOM_HANDLED
|
||||
|
||||
# Rebuild ignore map each cycle (config may have changed)
|
||||
declare -A IGNORE_MAP
|
||||
for c in "${WATCHDOG_SCAN_IGNORE[@]}"; do
|
||||
for c in "${WATCHDOG_SCAN_IGNORE[@]:-}"; do
|
||||
[[ -n "$c" ]] && IGNORE_MAP["$c"]=1
|
||||
done
|
||||
|
||||
# Startup grace check
|
||||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||||
IN_GRACE_PERIOD=false
|
||||
if [[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]]; then
|
||||
IN_GRACE_PERIOD=true
|
||||
# ── Docker daemon health check — first check every cycle ────────────────────────────────
|
||||
# If daemon is hung all container operations will fail — check first, skip cycle if down
|
||||
if ! check_docker_daemon; then
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Parity check — skip restarts during parity to avoid I/O interference
|
||||
# ── Parity check — skip restarts during parity ───────────────────────────────────────────
|
||||
if is_parity_running; then
|
||||
log "Parity check in progress — skipping restart actions this cycle"
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL"
|
||||
continue
|
||||
fi
|
||||
|
||||
# ── TIER 1 — Strict Monitoring ──────────────────────────────────────────────────────────
|
||||
# ── RAM emergency check — system_watchdog.sh managing containers ───────────────────────────
|
||||
# If system_watchdog.sh has triggered an emergency RAM shutdown, defer all container
|
||||
# management this cycle. Docker daemon health checks continue — system still needs
|
||||
# monitoring even during RAM crisis. Restarts deferred to prevent undoing shutdown.
|
||||
if ! check_system_watchdog_state; then
|
||||
MEM_KB=$(awk '/MemAvailable/ {print $2}' /proc/meminfo)
|
||||
MEM_GB=$(( MEM_KB / 1024 / 1024 ))
|
||||
warn "RAM emergency active (${MEM_GB}GB free) — system_watchdog.sh managing containers"
|
||||
warn "Deferring all container restart logic this cycle"
|
||||
log "Waiting for RAM to recover above ${SYS_WATCHDOG_MEM_RECOVER_GB}GB before resuming"
|
||||
sleep "$DOCKER_WATCHDOG_INTERVAL" &
|
||||
wait $!
|
||||
continue
|
||||
fi
|
||||
|
||||
# Required containers
|
||||
# ── Startup grace period ──────────────────────────────────────────────────────────────────
|
||||
UPTIME_SECONDS=$(awk '{print int($1)}' /proc/uptime)
|
||||
IN_GRACE_PERIOD=false
|
||||
[[ "$UPTIME_SECONDS" -lt "$WATCHDOG_STARTUP_GRACE" ]] && IN_GRACE_PERIOD=true
|
||||
|
||||
# ==========================================================================================
|
||||
# ── TIER 1 — Strict Monitoring ────────────────────────────────────────────────────────────
|
||||
# ==========================================================================================
|
||||
|
||||
# ── Required containers ───────────────────────────────────────────────────────────────────
|
||||
# Must always be running — strike system before restart, skip list after limit
|
||||
if [[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -gt 0 ]]; then
|
||||
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' \
|
||||
"$container" 2>/dev/null)
|
||||
|
||||
if is_skipped "$container"; then
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
# Container recovered — remove from skip list and clear its history
|
||||
remove_from_skip_list "$container"
|
||||
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
sed -i "/^${container}|/d" "$WATCHDOG_CONTAINER_RESTART_LOG" 2>/dev/null
|
||||
@@ -345,6 +549,7 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
fi
|
||||
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
# Running — clear any strikes
|
||||
set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
else
|
||||
STRIKES=$(get_strikes "$container" "$WATCHDOG_STATE_FILE")
|
||||
@@ -360,7 +565,7 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
0) set_strikes "$container" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container was down and restarted on $(hostname)" "warning" ;;
|
||||
2) : ;;
|
||||
2) : ;; # Added to skip list — already notified
|
||||
*) queue_notify "$container failed to restart on $(hostname)" "warning" ;;
|
||||
esac
|
||||
fi
|
||||
@@ -368,9 +573,9 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
done
|
||||
fi
|
||||
|
||||
# Memory and CPU monitoring
|
||||
# ── Memory and CPU monitoring ─────────────────────────────────────────────────────────────
|
||||
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
|
||||
STATS=$(docker stats --no-stream \
|
||||
STATS=$(timeout "$DOCKER_TIMEOUT" docker stats --no-stream \
|
||||
--format "{{.Name}}|{{.MemUsage}}|{{.CPUPerc}}" 2>/dev/null)
|
||||
TOTAL_CORES=$(nproc 2>/dev/null || echo 1)
|
||||
|
||||
@@ -379,6 +584,7 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
CONTAINER_STATS=$(echo "$STATS" | grep "^${container}|" | head -1)
|
||||
[[ -z "$CONTAINER_STATS" ]] && continue
|
||||
|
||||
# ── Memory ────────────────────────────────────────────────────────────────────────
|
||||
MEM_USAGE=$(echo "$CONTAINER_STATS" | cut -d'|' -f2 | awk '{print $1}')
|
||||
MEM_UNIT=$(echo "$MEM_USAGE" | grep -oE '[A-Za-z]+')
|
||||
MEM_VALUE=$(echo "$MEM_USAGE" | grep -oE '[0-9.]+')
|
||||
@@ -390,34 +596,50 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
*) MEM_MB=0 ;;
|
||||
esac
|
||||
|
||||
# Soft memory threshold — warn when approaching hard limit
|
||||
SOFT_MEM_MB=$(awk "BEGIN {printf \"%.0f\", $MEM_LIMIT_MB * $SOFT_MEM_THRESHOLD / 100}")
|
||||
if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then
|
||||
# Hard limit exceeded — immediate restart
|
||||
error "$container — memory ${MEM_MB}MB exceeded hard limit ${MEM_LIMIT_MB}MB"
|
||||
safe_restart "$container" "memory hard limit exceeded"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container exceeded memory hard limit on $(hostname) — restarted" "warning"
|
||||
elif [[ "$MEM_MB" -ge "$SOFT_MEM_MB" ]]; then
|
||||
# Soft threshold — warn only, no restart
|
||||
warn "$container — memory ${MEM_MB}MB approaching limit (${SOFT_MEM_THRESHOLD}% of ${MEM_LIMIT_MB}MB)"
|
||||
((T1_WARNINGS++))
|
||||
fi
|
||||
|
||||
# ── CPU ───────────────────────────────────────────────────────────────────────────
|
||||
CPU_RAW=$(echo "$CONTAINER_STATS" | cut -d'|' -f3 | tr -d '%')
|
||||
CPU_NORM=$(awk "BEGIN {printf \"%.1f\", $CPU_RAW / $TOTAL_CORES}")
|
||||
CPU_INT=$(printf "%.0f" "$CPU_NORM")
|
||||
|
||||
if [[ "$MEM_MB" -ge "$MEM_LIMIT_MB" ]]; then
|
||||
error "$container — memory exceeded hard limit ${MEM_LIMIT_MB}MB"
|
||||
safe_restart "$container" "memory hard limit exceeded"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container exceeded memory limit on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
|
||||
if [[ "$CPU_INT" -ge "$HARD_CPU_THRESHOLD" ]]; then
|
||||
# Hard CPU threshold — strike system → restart
|
||||
CPU_STRIKES=$(get_strikes "${container}_cpu" "$WATCHDOG_STATE_FILE")
|
||||
CPU_STRIKES=$(( CPU_STRIKES + 1 ))
|
||||
set_strikes "${container}_cpu" "$CPU_STRIKES" "$WATCHDOG_STATE_FILE"
|
||||
warn "$container — CPU ${CPU_NORM}% (strike $CPU_STRIKES/$CPU_FAIL_LIMIT)"
|
||||
if [[ "$CPU_STRIKES" -ge "$CPU_FAIL_LIMIT" ]]; then
|
||||
safe_restart "$container" "CPU threshold exceeded"
|
||||
safe_restart "$container" "CPU hard threshold exceeded ${CPU_NORM}%"
|
||||
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container CPU ${CPU_NORM}% on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
elif [[ "$CPU_INT" -ge "$SOFT_CPU_THRESHOLD" ]]; then
|
||||
# Soft CPU threshold — warn only, no restart, clear strikes
|
||||
warn "$container — CPU ${CPU_NORM}% (above soft threshold ${SOFT_CPU_THRESHOLD}%)"
|
||||
((T1_WARNINGS++))
|
||||
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
|
||||
else
|
||||
# Normal — clear CPU strikes
|
||||
set_strikes "${container}_cpu" 0 "$WATCHDOG_STATE_FILE"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# HTTP responsiveness
|
||||
# ── HTTP responsiveness ───────────────────────────────────────────────────────────────────
|
||||
if [[ ${#WATCHDOG_CONTAINER_URLS[@]} -gt 0 ]]; then
|
||||
for container in "${!WATCHDOG_CONTAINER_URLS[@]}"; do
|
||||
URL="${WATCHDOG_CONTAINER_URLS[$container]}"
|
||||
@@ -430,102 +652,133 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
warn "$container — not responding at $URL (strike $HTTP_STRIKES/$RESP_FAIL_LIMIT)"
|
||||
((T1_WARNINGS++))
|
||||
if [[ "$HTTP_STRIKES" -ge "$RESP_FAIL_LIMIT" ]]; then
|
||||
safe_restart "$container" "HTTP unresponsive"
|
||||
set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning"
|
||||
result=0
|
||||
safe_restart "$container" "HTTP unresponsive at $URL" || result=$?
|
||||
if [[ $result -eq 0 ]]; then
|
||||
set_strikes "${container}_http" 0 "$WATCHDOG_STATE_FILE"
|
||||
((T1_RESTARTS++))
|
||||
queue_notify "$container unresponsive at $URL on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# ── TIER 2 — Global Health Scan ─────────────────────────────────────────────────────────
|
||||
# ==========================================================================================
|
||||
# ── TIER 2 — Global Health Scan ───────────────────────────────────────────────────────────
|
||||
# ==========================================================================================
|
||||
if [[ "$WATCHDOG_SCAN_ALL" == "true" ]]; then
|
||||
ALL_CONTAINERS=$(docker ps --format "{{.Names}}" 2>/dev/null)
|
||||
|
||||
# Unhealthy containers
|
||||
ALL_CONTAINERS=$(timeout "$DOCKER_TIMEOUT" docker ps --format "{{.Names}}" 2>/dev/null)
|
||||
|
||||
# ── Unhealthy containers ──────────────────────────────────────────────────────────────
|
||||
if [[ "$WATCHDOG_RESTART_UNHEALTHY" == "true" ]]; then
|
||||
UNHEALTHY=$(docker ps --filter health=unhealthy --format "{{.Names}}" 2>/dev/null)
|
||||
UNHEALTHY=$(timeout "$DOCKER_TIMEOUT" docker ps \
|
||||
--filter health=unhealthy --format "{{.Names}}" 2>/dev/null)
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
error "$container — unhealthy"
|
||||
error "$container — Docker HEALTHCHECK unhealthy"
|
||||
((T2_WARNINGS++))
|
||||
result=0
|
||||
safe_restart "$container" "unhealthy health status" || result=$?
|
||||
[[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
|
||||
if [[ $result -eq 0 ]]; then
|
||||
((T2_RESTARTS++))
|
||||
queue_notify "$container unhealthy on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
done <<< "$UNHEALTHY"
|
||||
fi
|
||||
|
||||
# OOM killed
|
||||
# ── OOM killed ────────────────────────────────────────────────────────────────────────
|
||||
# OOMKilled flag persists after restart — track handled containers per-cycle
|
||||
# to prevent the same container triggering a restart loop every cycle
|
||||
if [[ "$WATCHDOG_NOTIFY_OOM" == "true" ]]; then
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
OOM=$(docker inspect -f '{{.State.OOMKilled}}' "$container" 2>/dev/null)
|
||||
[[ -n "${OOM_HANDLED[$container]:-}" ]] && continue # already handled this cycle
|
||||
OOM=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.State.OOMKilled}}' "$container" 2>/dev/null)
|
||||
if [[ "$OOM" == "true" ]]; then
|
||||
error "$container — OOM killed"
|
||||
error "$container — OOM killed by kernel"
|
||||
((T2_WARNINGS++))
|
||||
OOM_HANDLED["$container"]=1
|
||||
result=0
|
||||
safe_restart "$container" "OOM killed" || result=$?
|
||||
[[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
|
||||
if [[ $result -eq 0 ]]; then
|
||||
((T2_RESTARTS++))
|
||||
queue_notify "$container OOM killed on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
done <<< "$ALL_CONTAINERS"
|
||||
fi
|
||||
|
||||
# Crash loop detection
|
||||
if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
RESTART_COUNT=$(docker inspect -f '{{.RestartCount}}' "$container" 2>/dev/null || echo 0)
|
||||
PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d: -f2 || echo 0)
|
||||
set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE"
|
||||
if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then
|
||||
((T2_WARNINGS++))
|
||||
if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then
|
||||
error "$container — crash loop CRITICAL: $RESTART_COUNT restarts"
|
||||
queue_notify "$container crash loop CRITICAL on $(hostname) — manual intervention needed" "critical"
|
||||
else
|
||||
warn "$container — restarted since last check (total: $RESTART_COUNT)"
|
||||
queue_notify "$container restarted on $(hostname) — count: $RESTART_COUNT" "warning"
|
||||
fi
|
||||
fi
|
||||
done <<< "$ALL_CONTAINERS"
|
||||
fi
|
||||
|
||||
# Dead containers
|
||||
# ── Crash loop detection ──────────────────────────────────────────────────────────────
|
||||
# Tracks Docker's own RestartCount climbing between cycles.
|
||||
# Below WATCHDOG_CRASH_LIMIT: notify only — Docker's restart policy is handling it.
|
||||
# At or above WATCHDOG_CRASH_LIMIT: safe_restart() which will add to skip list
|
||||
# if WATCHDOG_CONTAINER_RESTART_LIMIT is also hit — ensures eventual quarantine.
|
||||
if [[ "$WATCHDOG_NOTIFY_CRASHLOOP" == "true" ]]; then
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
RESTART_COUNT=$(timeout "$DOCKER_TIMEOUT" docker inspect -f \
|
||||
'{{.RestartCount}}' "$container" 2>/dev/null || echo 0)
|
||||
PREV_COUNT=$(grep "^${container}_docker:" "$WATCHDOG_STATE_FILE" \
|
||||
2>/dev/null | cut -d: -f2 || echo 0)
|
||||
set_strikes "${container}_docker" "$RESTART_COUNT" "$WATCHDOG_STATE_FILE"
|
||||
if [[ "$RESTART_COUNT" -gt "$PREV_COUNT" && "$RESTART_COUNT" -gt 0 ]]; then
|
||||
((T2_WARNINGS++))
|
||||
if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CRASH_LIMIT" ]]; then
|
||||
error "$container — crash loop CRITICAL: $RESTART_COUNT restarts"
|
||||
# Attempt restart via safe_restart — will add to skip list if over limit
|
||||
result=0
|
||||
safe_restart "$container" "crash loop — $RESTART_COUNT restarts" || result=$?
|
||||
case $result in
|
||||
0) ((T2_RESTARTS++))
|
||||
queue_notify "$container crash loop on $(hostname) — restarted" "critical" ;;
|
||||
2) : ;; # Added to skip list
|
||||
*) queue_notify "$container crash loop CRITICAL on $(hostname) — manual intervention needed" "critical" ;;
|
||||
esac
|
||||
else
|
||||
warn "$container — restarted since last check (Docker count: $RESTART_COUNT)"
|
||||
queue_notify "$container restarted on $(hostname) — Docker count: $RESTART_COUNT" "warning"
|
||||
fi
|
||||
fi
|
||||
done <<< "$ALL_CONTAINERS"
|
||||
fi
|
||||
|
||||
# ── Dead containers ───────────────────────────────────────────────────────────────────
|
||||
# Routes through safe_restart() — ensures restart loop protection applies
|
||||
if [[ "$WATCHDOG_RESTART_DEAD" == "true" ]]; then
|
||||
DEAD=$(docker ps -a --filter status=dead --format "{{.Names}}" 2>/dev/null)
|
||||
DEAD=$(timeout "$DOCKER_TIMEOUT" docker ps -a \
|
||||
--filter status=dead --format "{{.Names}}" 2>/dev/null)
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
error "$container — dead"
|
||||
((T2_WARNINGS++))
|
||||
RESTART_COUNT=$(get_restart_count "$container")
|
||||
if [[ "$RESTART_COUNT" -ge "$WATCHDOG_CONTAINER_RESTART_LIMIT" ]]; then
|
||||
add_to_skip_list "$container" "dead — restarted $RESTART_COUNT times"
|
||||
elif [[ "$DRY_RUN" == false ]]; then
|
||||
docker rm "$container" >/dev/null 2>&1
|
||||
if docker start "$container" >/dev/null 2>&1; then
|
||||
success "$container removed from dead state and restarted"
|
||||
log_restart "$container"
|
||||
((T2_RESTARTS++))
|
||||
queue_notify "$container was dead on $(hostname) — restarted" "warning"
|
||||
fi
|
||||
if [[ "$DRY_RUN" == false ]]; then
|
||||
timeout "$DOCKER_TIMEOUT" docker rm "$container" >/dev/null 2>&1
|
||||
fi
|
||||
result=0
|
||||
safe_restart "$container" "dead container" || result=$?
|
||||
if [[ $result -eq 0 ]]; then
|
||||
((T2_RESTARTS++))
|
||||
queue_notify "$container was dead on $(hostname) — removed and restarted" "warning"
|
||||
fi
|
||||
done <<< "$DEAD"
|
||||
fi
|
||||
|
||||
# Unexpected exits
|
||||
# ── Unexpected exits ──────────────────────────────────────────────────────────────────
|
||||
# Only non-zero exit codes — exit 0 is a clean stop, not a crash
|
||||
# Skips containers already covered by WATCHDOG_REQUIRED_CONTAINERS (handled in Tier 1)
|
||||
if [[ "$WATCHDOG_RESTART_CRASHED" == "true" ]]; then
|
||||
CRASHED=$(docker ps -a \
|
||||
CRASHED=$(timeout "$DOCKER_TIMEOUT" docker ps -a \
|
||||
--filter status=exited \
|
||||
--format "{{.Names}}|{{.Status}}" 2>/dev/null | \
|
||||
grep -v "Exited (0)")
|
||||
@@ -533,30 +786,34 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
[[ -z "$container" ]] && continue
|
||||
[[ -n "${IGNORE_MAP[$container]:-}" ]] && continue
|
||||
is_skipped "$container" && continue
|
||||
SKIP=false
|
||||
for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
||||
[[ "$container" == "$req" ]] && SKIP=true && break
|
||||
# Skip containers already monitored by required containers (Tier 1)
|
||||
local already_required=false
|
||||
for req in "${WATCHDOG_REQUIRED_CONTAINERS[@]:-}"; do
|
||||
[[ "$container" == "$req" ]] && already_required=true && break
|
||||
done
|
||||
[[ "$SKIP" == true ]] && continue
|
||||
[[ "$already_required" == true ]] && continue
|
||||
error "$container — $status (unexpected exit)"
|
||||
((T2_WARNINGS++))
|
||||
result=0
|
||||
safe_restart "$container" "unexpected exit" || result=$?
|
||||
[[ $result -eq 0 ]] && ((T2_RESTARTS++)) && \
|
||||
safe_restart "$container" "unexpected exit: $status" || result=$?
|
||||
if [[ $result -eq 0 ]]; then
|
||||
((T2_RESTARTS++))
|
||||
queue_notify "$container crashed on $(hostname) ($status) — restarted" "warning"
|
||||
fi
|
||||
done <<< "$CRASHED"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Send notifications if any events this cycle
|
||||
fi # WATCHDOG_SCAN_ALL
|
||||
|
||||
# ── Send notifications ────────────────────────────────────────────────────────────────────
|
||||
flush_notify
|
||||
|
||||
# Only log summary if something happened — quiet when all healthy
|
||||
# ── Cycle summary — quiet when healthy ───────────────────────────────────────────────────
|
||||
TOTAL_RESTARTS=$(( T1_RESTARTS + T2_RESTARTS ))
|
||||
TOTAL_WARNINGS=$(( T1_WARNINGS + T2_WARNINGS ))
|
||||
CYCLE_END=$(date +%s)
|
||||
|
||||
if [[ "$TOTAL_RESTARTS" -gt 0 || "$TOTAL_WARNINGS" -gt 0 ]]; then
|
||||
CYCLE_END=$(date +%s)
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Cycle $CYCLE — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_WATCHDOG T1: $T1_RESTARTS restarts / $T1_WARNINGS warnings"
|
||||
@@ -565,13 +822,16 @@ while [[ "$WATCHDOG_RUNNING" == true ]]; do
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
else
|
||||
log "Cycle $CYCLE — all healthy ($(date '+%H:%M:%S'))"
|
||||
# Heartbeat — periodic proof of life
|
||||
|
||||
# Heartbeat — periodic proof of life even when everything is healthy
|
||||
if [[ "${DOCKER_WATCHDOG_HEARTBEAT:-true}" == true ]]; then
|
||||
HB_SECONDS=$(( ${DOCKER_WATCHDOG_HEARTBEAT_HOURS:-1} * 3600 ))
|
||||
UPTIME_SECONDS=$(( CYCLE * DOCKER_WATCHDOG_INTERVAL ))
|
||||
if [[ "$HB_SECONDS" -gt 0 ]] && (( UPTIME_SECONDS % HB_SECONDS < DOCKER_WATCHDOG_INTERVAL )) && [[ "$UPTIME_SECONDS" -gt 0 ]]; then
|
||||
HB_UPTIME_HR=$(( UPTIME_SECONDS / 3600 ))
|
||||
info "♥ docker_watchdog alive — ~${HB_UPTIME_HR}hr uptime ($(date '+%H:%M:%S'))"
|
||||
UPTIME_APPROX=$(( CYCLE * DOCKER_WATCHDOG_INTERVAL ))
|
||||
if [[ "$HB_SECONDS" -gt 0 ]] && \
|
||||
(( UPTIME_APPROX % HB_SECONDS < DOCKER_WATCHDOG_INTERVAL )) && \
|
||||
[[ "$UPTIME_APPROX" -gt 0 ]]; then
|
||||
HB_UPTIME_HR=$(( UPTIME_APPROX / 3600 ))
|
||||
info "♥ docker_watchdog alive — $MY_ID — ~${HB_UPTIME_HR}hr uptime ($(date '+%H:%M:%S'))"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -1,24 +1,64 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Docker Weekly Restart --------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Restarts or starts specified Docker containers with retry logic.
|
||||
# Containers are configured in Master.conf under WEEKLY_RESTART_CONTAINERS.
|
||||
# Uses global RETRY_COUNT and SLEEP from Master.conf for retry behaviour.
|
||||
# Sends notifications on completion or failure via common.sh notify().
|
||||
# Supports --dry-run to preview what would be restarted without taking action.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ================================= Docker Weekly Restart ======================================
|
||||
# ==============================================================================================
|
||||
# Restarts all running containers in HOST*_WEEKLY_RESTART_CONTAINERS.
|
||||
# Called by weekly_sync_maintenance.sh via WEEKLY_MAINTENANCE_SCRIPTS every Sunday at 2:30am.
|
||||
# Can also be run manually for ad hoc weekly restarts.
|
||||
#
|
||||
# ── CONTEXT ───────────────────────────────────────────────────────────────────────────────────
|
||||
# weekly_sync_maintenance.sh stops containers before syncing and restarts them after.
|
||||
# This script runs AFTER that restart — targeting a different set of less critical services
|
||||
# that benefit from a weekly restart but don't need to be stopped for the sync itself.
|
||||
# These containers are typically already running when this script executes.
|
||||
#
|
||||
# ── BEHAVIOUR ─────────────────────────────────────────────────────────────────────────────────
|
||||
# Running containers → docker restart (graceful stop + start)
|
||||
# Stopped containers → left stopped — was down intentionally, do not bring back up
|
||||
# Missing containers → logged and skipped — not treated as fatal
|
||||
# Each action uses RETRY_COUNT + SLEEP from master.conf for retry logic.
|
||||
#
|
||||
# The "was running → restart, was stopped → leave stopped" rule is consistent
|
||||
# across the entire ecosystem — container state is always respected.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# Dependency ordering — containers restart in dependency-safe order using
|
||||
# HOST*_WATCHDOG_DEPENDENCIES from master_host*.conf. Dependencies restart
|
||||
# first with CONTAINER_DELAY before their dependents.
|
||||
#
|
||||
# Restart verification — after each restart, container state is checked after a short
|
||||
# settle period. If the container fails to stay running it is marked as failed and
|
||||
# a notification is sent rather than silently passing.
|
||||
#
|
||||
# Timeout protection — all docker commands are wrapped in a 30 second timeout.
|
||||
# A hung Docker daemon cannot cause this script to hang indefinitely.
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_WEEKLY_RESTART_CONTAINERS — list of containers to restart weekly
|
||||
# HOST*_WATCHDOG_DEPENDENCIES — dependency ordering for restart sequence
|
||||
# Set by detect_hosts() alias → WEEKLY_RESTART_CONTAINERS used by this script
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# RETRY_COUNT — retry attempts before giving up on a container
|
||||
# SLEEP — seconds between retry attempts
|
||||
# CONTAINER_DELAY — seconds to wait between dependency and dependent restart
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# docker_weekly_restart.sh — normal restart
|
||||
# docker_weekly_restart.sh --dry-run — preview without restarting
|
||||
# docker_weekly_restart.sh --log — verbose output
|
||||
# docker_weekly_restart.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
@@ -26,7 +66,6 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
acquire_lock
|
||||
@@ -36,19 +75,28 @@ if ! command -v docker &>/dev/null; then
|
||||
notify "Docker weekly restart failed — Docker not found on $(hostname)" "Docker Weekly Restart" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Docker found"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# detect_hosts() sets MY_ID and aliases HOST*_WEEKLY_RESTART_CONTAINERS → WEEKLY_RESTART_CONTAINERS
|
||||
detect_hosts
|
||||
|
||||
if [[ ${#WEEKLY_RESTART_CONTAINERS[@]} -eq 0 ]]; then
|
||||
warn "WEEKLY_RESTART_CONTAINERS is empty for $MY_ID — nothing to restart"
|
||||
warn "Check HOST*_WEEKLY_RESTART_CONTAINERS in master_host*.conf"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
|
||||
echo "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||
echo "$ICON_TIME Sleep: ${SLEEP}s between retries"
|
||||
echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
@@ -56,19 +104,32 @@ fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ── FUNCTIONS ─────────────────────────────────────────────────────────────────────────────────
|
||||
# ==============================================================================================
|
||||
|
||||
# Attempts a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
|
||||
# Returns 0 on success, 1 if all attempts fail.
|
||||
# Wraps docker commands with a 30 second timeout.
|
||||
# Prevents a hung Docker daemon from causing the script to hang indefinitely.
|
||||
DOCKER_TIMEOUT=30
|
||||
docker_cmd() {
|
||||
timeout "$DOCKER_TIMEOUT" "$@"
|
||||
local exit_code=$?
|
||||
if [[ "$exit_code" -eq 124 ]]; then
|
||||
error "Docker command timed out after ${DOCKER_TIMEOUT}s: $*"
|
||||
return 1
|
||||
fi
|
||||
return "$exit_code"
|
||||
}
|
||||
|
||||
# Retries a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
|
||||
# Uses docker_cmd wrapper for timeout protection on each attempt.
|
||||
retry_docker() {
|
||||
local attempt=1
|
||||
|
||||
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
|
||||
info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
||||
|
||||
if "$@"; then
|
||||
if docker_cmd "$@"; then
|
||||
success "Succeeded on attempt $attempt"
|
||||
return 0
|
||||
else
|
||||
@@ -82,9 +143,77 @@ retry_docker() {
|
||||
return 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CONTAINERS Weekly Restart ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Verifies a container is still running after restart.
|
||||
# Gives the container a short settle period before checking.
|
||||
RESTART_VERIFY_WAIT=5
|
||||
verify_running() {
|
||||
local container="$1"
|
||||
sleep "$RESTART_VERIFY_WAIT"
|
||||
local state
|
||||
state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
if [[ "$state" != "true" ]]; then
|
||||
error "$container failed to stay running after restart — may have crashed"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Builds a dependency-safe restart order from WEEKLY_RESTART_CONTAINERS.
|
||||
# Containers that are dependencies of others restart first.
|
||||
build_restart_order() {
|
||||
ORDERED_RESTART=()
|
||||
local remaining=("${WEEKLY_RESTART_CONTAINERS[@]}")
|
||||
local placed=()
|
||||
|
||||
# First pass — add dependency containers that appear in our list
|
||||
for container in "${remaining[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local is_dependency=false
|
||||
for dependent in "${!WATCHDOG_DEPENDENCIES[@]:-}"; do
|
||||
if [[ "${WATCHDOG_DEPENDENCIES[$dependent]}" == *"$container"* ]]; then
|
||||
is_dependency=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$is_dependency" == true ]]; then
|
||||
local already=false
|
||||
for p in "${placed[@]:-}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
||||
if [[ "$already" == false ]]; then
|
||||
ORDERED_RESTART+=("$container")
|
||||
placed+=("$container")
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Second pass — add remaining containers (dependents and independents)
|
||||
for container in "${remaining[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
local already=false
|
||||
for p in "${placed[@]:-}"; do [[ "$p" == "$container" ]] && already=true && break; done
|
||||
if [[ "$already" == false ]]; then
|
||||
ORDERED_RESTART+=("$container")
|
||||
placed+=("$container")
|
||||
fi
|
||||
done
|
||||
|
||||
log "Restart order: ${ORDERED_RESTART[*]}"
|
||||
}
|
||||
|
||||
# Waits CONTAINER_DELAY if this container depends on the last restarted one.
|
||||
check_dependency_delay() {
|
||||
local container="$1"
|
||||
local last="$2"
|
||||
[[ -z "$last" ]] && return
|
||||
local deps="${WATCHDOG_DEPENDENCIES[$container]:-}"
|
||||
if [[ -n "$deps" ]] && [[ "$deps" == *"$last"* ]]; then
|
||||
info "Waiting ${CONTAINER_DELAY}s — $container depends on $last..."
|
||||
sleep "$CONTAINER_DELAY"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ Weekly Restart ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Weekly Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
|
||||
@@ -94,30 +223,48 @@ echo ""
|
||||
START=$(date +%s)
|
||||
FAILED=()
|
||||
RESTARTED=()
|
||||
STARTED=()
|
||||
SKIPPED=()
|
||||
|
||||
for container in "${WEEKLY_RESTART_CONTAINERS[@]}"; do
|
||||
# Build dependency-safe restart order
|
||||
build_restart_order
|
||||
echo "$ICON_GEAR Restart order: ${ORDERED_RESTART[*]}"
|
||||
echo ""
|
||||
|
||||
LAST_RESTARTED=""
|
||||
|
||||
for container in "${ORDERED_RESTART[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
error "$container does not exist — skipping"
|
||||
FAILED+=("$container")
|
||||
if ! timeout "$DOCKER_TIMEOUT" docker inspect "$container" &>/dev/null; then
|
||||
warn "$container does not exist — skipping"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
STATUS=$(timeout "$DOCKER_TIMEOUT" docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
|
||||
case "$STATUS" in
|
||||
true)
|
||||
echo "$ICON_RUNNING $container is running — restarting..."
|
||||
|
||||
# Wait if this container depends on the last one restarted
|
||||
check_dependency_delay "$container" "$LAST_RESTARTED"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container"
|
||||
RESTARTED+=("$container")
|
||||
else
|
||||
if retry_docker docker restart "$container"; then
|
||||
echo "$ICON_STARTED $container restarted"
|
||||
RESTARTED+=("$container")
|
||||
if verify_running "$container"; then
|
||||
echo "$ICON_STARTED $container restarted and running ✅"
|
||||
RESTARTED+=("$container")
|
||||
LAST_RESTARTED="$container"
|
||||
else
|
||||
error "$container restarted but crashed immediately"
|
||||
notify "$container crashed after restart on $(hostname)" "Docker Weekly Restart" "warning"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
else
|
||||
error "Failed to restart $container after $RETRY_COUNT attempts"
|
||||
notify "$container failed to restart on $(hostname)" "Docker Weekly Restart" "warning"
|
||||
@@ -126,20 +273,10 @@ for container in "${WEEKLY_RESTART_CONTAINERS[@]}"; do
|
||||
fi
|
||||
;;
|
||||
false)
|
||||
echo "$ICON_NOT_RUNNING $container is stopped — starting..."
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would start $container"
|
||||
else
|
||||
if retry_docker docker start "$container"; then
|
||||
echo "$ICON_STARTED $container started"
|
||||
STARTED+=("$container")
|
||||
else
|
||||
error "Failed to start $container after $RETRY_COUNT attempts"
|
||||
notify "$container failed to start on $(hostname)" "Docker Weekly Restart" "warning"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
fi
|
||||
# Container was stopped — leave it stopped
|
||||
# Intentionally stopped containers are not restarted
|
||||
echo "$ICON_NOT_RUNNING $container is stopped — skipping (respecting stopped state)"
|
||||
SKIPPED+=("$container")
|
||||
;;
|
||||
*)
|
||||
error "Unknown status for $container: $STATUS"
|
||||
@@ -152,25 +289,26 @@ done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY RESTART SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||
[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]} (were stopped)"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE"
|
||||
notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Weekly Restart" "normal"
|
||||
notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#SKIPPED[@]} skipped (stopped) on $(hostname)" "Docker Weekly Restart" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed"
|
||||
notify "Weekly restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Weekly Restart" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -1,40 +1,70 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Downloaders Reset ------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Daily maintenance reset for all download clients.
|
||||
# Clears stuck states, purges old history, and prepares downloaders for a clean daily cycle.
|
||||
# Run before container restarts in daily_sync_maintenance.sh via MEDIA_MANAGEMENT_JOBS.
|
||||
# ==============================================================================================
|
||||
# ================================= Downloaders Reset ==========================================
|
||||
# ==============================================================================================
|
||||
# Maintenance reset for all download clients on this server.
|
||||
# Called every 15 minutes by critical_sync_maintenance.sh via CRITICAL_MAINTENANCE_SCRIPTS.
|
||||
# Can also be run manually for ad hoc cleanup.
|
||||
#
|
||||
# Downloaders covered:
|
||||
# slskd — clears stuck/errored searches, dead transfer records,
|
||||
# purges expired failed imports (albums Lidarr rejected)
|
||||
# SABnzbd — clears completed and failed history older than retention period,
|
||||
# removes stalled/paused queue items
|
||||
# qBittorrent — last chance failsafe delete for torrents older than
|
||||
# QBIT_FAILSAFE_MIN_DAYS regardless of ratio
|
||||
# ── DOWNLOADERS COVERED ───────────────────────────────────────────────────────────────────────
|
||||
# slskd
|
||||
# Stuck searches — clears Completed/Errored searches left by Soularr crashes
|
||||
# prevents 409 Conflict on next Soularr startup
|
||||
# Dead transfers — removes completed/errored/aborted transfer records per user
|
||||
# prevents Soularr 404 loop when polling a user whose transfer is gone
|
||||
# NEVER removes InProgress or Queued transfers
|
||||
# Failed imports — purges albums Soularr downloaded but Lidarr rejected
|
||||
# Soularr moves these to failed_imports/ and never cleans them up
|
||||
#
|
||||
# Safety:
|
||||
# Always --dry-run first before scheduling
|
||||
# slskd transfer cleanup skips any user with InProgress or Queued transfers
|
||||
# qBittorrent only deletes if torrent age exceeds QBIT_FAILSAFE_MIN_DAYS
|
||||
# SABnzbd only deletes history older than DOWNLOADER_RETENTION_DAYS
|
||||
# qBittorrent deleteFiles=false — removes from qBit, leaves files for arrs to manage
|
||||
# SABnzbd
|
||||
# Completed history — removes completed download records older than DOWNLOADER_RETENTION_DAYS
|
||||
# Failed history — removes failed download records older than DOWNLOADER_RETENTION_DAYS
|
||||
# Stalled queue — removes Paused or Stuck queue items no longer progressing
|
||||
# active downloading items are never touched
|
||||
#
|
||||
# Configuration in Master.conf under Docker Essentials — Downloaders Reset.
|
||||
# Supports --dry-run to preview without making changes.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# qBittorrent
|
||||
# Age failsafe — removes torrents older than QBIT_FAILSAFE_MIN_DAYS
|
||||
# deleteFiles=false — removes from qBit, leaves files for arrs to manage
|
||||
# optional ratio requirement via QBIT_FAILSAFE_MIN_RATIO
|
||||
#
|
||||
# ── HOST AWARENESS ────────────────────────────────────────────────────────────────────────────
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_SLSKD_*, HOST*_SABNZBD_*, HOST*_QBIT_* vars.
|
||||
# If a downloader URL is empty for this host — that section is skipped with a clear message.
|
||||
# HOST2 currently has no downloaders configured — all sections skip cleanly on HOST2.
|
||||
#
|
||||
# ── SAFEGUARDS ────────────────────────────────────────────────────────────────────────────────
|
||||
# slskd — skips users with InProgress or Queued transfers — never interrupts active downloads
|
||||
# SABnzbd — age check before deletion — only removes items past retention threshold
|
||||
# qBittorrent — age + optional ratio check — failsafe only removes old completed torrents
|
||||
# All sections — skip gracefully if downloader is unreachable, no fatal exit
|
||||
# acquire_lock "wait" — if previous run still active, waits briefly then exits cleanly
|
||||
#
|
||||
# ── CONFIGURATION (master_host*.conf) ─────────────────────────────────────────────────────────
|
||||
# HOST*_SLSKD_URL / HOST*_SLSKD_API_KEY / HOST*_SLSKD_FAILED_IMPORTS_DIR
|
||||
# HOST*_SABNZBD_URL / HOST*_SABNZBD_API_KEY
|
||||
# HOST*_QBIT_URL / HOST*_QBIT_USERNAME / HOST*_QBIT_PASSWORD
|
||||
#
|
||||
# ── CONFIGURATION (master.conf) ───────────────────────────────────────────────────────────────
|
||||
# DOWNLOADER_RETENTION_DAYS — days before history entries are purged
|
||||
# QBIT_FAILSAFE_MIN_DAYS — minimum torrent age before failsafe deletion
|
||||
# QBIT_FAILSAFE_MIN_RATIO — minimum ratio requirement (0 = age only)
|
||||
#
|
||||
# ── USAGE ─────────────────────────────────────────────────────────────────────────────────────
|
||||
# downloaders_reset.sh — normal reset
|
||||
# downloaders_reset.sh --dry-run — preview without making changes
|
||||
# downloaders_reset.sh --log — verbose output
|
||||
# downloaders_reset.sh --status — show config and exit
|
||||
# ==============================================================================================
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
source "$SCRIPT_DIR/../load_config.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Setup ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Downloaders Reset ━━━"
|
||||
|
||||
@@ -43,45 +73,54 @@ if [[ "$EUID" -ne 0 ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
detect_hosts
|
||||
|
||||
# Lock first — wait mode since this runs every 15min and previous may still be finishing
|
||||
acquire_lock "wait"
|
||||
|
||||
# detect_hosts() sets MY_ID and aliases all HOST*_SLSKD_*, HOST*_SABNZBD_*, HOST*_QBIT_* vars
|
||||
detect_hosts
|
||||
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
# Select host-specific config
|
||||
if [[ "$LOCAL_SERVER_NAME" == "$HOST1" ]]; then
|
||||
SLSKD_URL="$HOST1_SLSKD_URL"
|
||||
SLSKD_API_KEY="$HOST1_SLSKD_API_KEY"
|
||||
SLSKD_FAILED_IMPORTS_DIR="$HOST1_SLSKD_FAILED_IMPORTS_DIR"
|
||||
SABNZBD_URL="$HOST1_SABNZBD_URL"
|
||||
SABNZBD_API_KEY="$HOST1_SABNZBD_API_KEY"
|
||||
QBIT_URL="$HOST1_QBIT_URL"
|
||||
QBIT_USERNAME="$HOST1_QBIT_USERNAME"
|
||||
QBIT_PASSWORD="$HOST1_QBIT_PASSWORD"
|
||||
else
|
||||
# HOST2 placeholders — fill in when HOST2 is back online
|
||||
SLSKD_URL="${HOST2_SLSKD_URL:-}"
|
||||
SLSKD_API_KEY="${HOST2_SLSKD_API_KEY:-}"
|
||||
SLSKD_FAILED_IMPORTS_DIR="${HOST2_SLSKD_FAILED_IMPORTS_DIR:-}"
|
||||
SABNZBD_URL="${HOST2_SABNZBD_URL:-}"
|
||||
SABNZBD_API_KEY="${HOST2_SABNZBD_API_KEY:-}"
|
||||
QBIT_URL="${HOST2_QBIT_URL:-}"
|
||||
QBIT_USERNAME="${HOST2_QBIT_USERNAME:-}"
|
||||
QBIT_PASSWORD="${HOST2_QBIT_PASSWORD:-}"
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
CUTOFF=$(( $(date +%s) - (DOWNLOADER_RETENTION_DAYS * 86400) ))
|
||||
TOTAL_PASS=0
|
||||
TOTAL_FAIL=0
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Status ━━━
|
||||
# ==============================================================================================
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_GEAR slskd: ${SLSKD_URL:-not configured}"
|
||||
echo "$ICON_GEAR SABnzbd: ${SABNZBD_URL:-not configured}"
|
||||
echo "$ICON_GEAR qBittorrent: ${QBIT_URL:-not configured}"
|
||||
echo "$ICON_TIME Retention: ${DOWNLOADER_RETENTION_DAYS} days"
|
||||
echo "$ICON_GEAR qBit age: ${QBIT_FAILSAFE_MIN_DAYS} days"
|
||||
echo "$ICON_GEAR qBit ratio: ${QBIT_FAILSAFE_MIN_RATIO} (0=age only)"
|
||||
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${MY_DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# Log which downloaders are active on this host
|
||||
if [[ -z "$SLSKD_URL" ]] && [[ -z "$SABNZBD_URL" ]] && [[ -z "$QBIT_URL" ]]; then
|
||||
warn "No downloaders configured for $MY_ID — nothing to reset"
|
||||
exit 0
|
||||
fi
|
||||
[[ -n "$SLSKD_URL" ]] && log "slskd active on $MY_ID"
|
||||
[[ -n "$SABNZBD_URL" ]] && log "SABnzbd active on $MY_ID"
|
||||
[[ -n "$QBIT_URL" ]] && log "qBittorrent active on $MY_ID"
|
||||
|
||||
# ==============================================================================================
|
||||
# ━━━ slskd — Stuck Searches ━━━
|
||||
# Clears searches in Completed/Errored state left by Soularr crashes
|
||||
# Prevents 409 Conflict error on next Soularr startup
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# Clears searches in Completed/Errored state left by Soularr crashes.
|
||||
# Prevents 409 Conflict error on next Soularr startup when it tries to
|
||||
# create a search with the same ID that already exists in a terminal state.
|
||||
|
||||
if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 slskd — Stuck Searches ━━━"
|
||||
@@ -127,12 +166,13 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ slskd — Dead Transfer Records ━━━
|
||||
# Removes completed/errored/aborted transfer records per user
|
||||
# Prevents Soularr 404 loop when polling a user whose transfer no longer exists
|
||||
# NEVER removes transfers that are InProgress or Queued
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# Removes completed/errored/aborted transfer records per user.
|
||||
# Prevents Soularr 404 loop when polling a user whose transfer no longer exists.
|
||||
# Safety: NEVER removes transfers that are InProgress or Queued — active downloads protected.
|
||||
|
||||
if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 slskd — Dead Transfer Records ━━━"
|
||||
@@ -152,6 +192,7 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then
|
||||
SUCCESS=0; SKIPPED=0; FAIL=0
|
||||
while IFS= read -r USER; do
|
||||
[[ -z "$USER" ]] && continue
|
||||
# Skip users with any active or queued transfers — never interrupt downloads
|
||||
ACTIVE=$(echo "$TRANSFERS" | grep -o "\"username\":\"$USER\"[^}]*\"state\":\"[^\"]*\"" | \
|
||||
grep -c "InProgress\|Queued")
|
||||
if [[ "$ACTIVE" -gt 0 ]]; then
|
||||
@@ -182,11 +223,13 @@ if [[ -n "$SLSKD_URL" ]] && [[ -n "$SLSKD_API_KEY" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ slskd — Purge Expired Failed Imports ━━━
|
||||
# Removes albums Soularr downloaded but Lidarr rejected
|
||||
# Soularr moves these to failed_imports/ and never cleans them up
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# Removes albums Soularr downloaded but Lidarr rejected.
|
||||
# Soularr moves rejected albums to failed_imports/ and never cleans them up.
|
||||
# Purges directories older than DOWNLOADER_RETENTION_DAYS to prevent unbounded growth.
|
||||
|
||||
if [[ -n "$SLSKD_FAILED_IMPORTS_DIR" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 slskd — Failed Imports (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━"
|
||||
@@ -217,10 +260,12 @@ if [[ -n "$SLSKD_FAILED_IMPORTS_DIR" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ SABnzbd — Clear Completed History ━━━
|
||||
# Removes completed download history older than DOWNLOADER_RETENTION_DAYS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# Removes completed download history older than DOWNLOADER_RETENTION_DAYS.
|
||||
# Keeps recent history for reference — only purges what's past the retention window.
|
||||
|
||||
if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 SABnzbd — Completed History (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━"
|
||||
@@ -261,10 +306,12 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ SABnzbd — Clear Failed History ━━━
|
||||
# Removes failed download history older than DOWNLOADER_RETENTION_DAYS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# Removes failed download history older than DOWNLOADER_RETENTION_DAYS.
|
||||
# Failed history is kept briefly for diagnosis but purged after the retention window.
|
||||
|
||||
if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 SABnzbd — Failed History (older than ${DOWNLOADER_RETENTION_DAYS} days) ━━━"
|
||||
@@ -305,11 +352,14 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ SABnzbd — Remove Stalled Queue Items ━━━
|
||||
# Removes paused queue items no longer progressing
|
||||
# Active downloading items are never touched
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# Removes queue items in Paused or Stuck state that are no longer progressing.
|
||||
# Active downloading items (Downloading, Grabbing) are never touched.
|
||||
# Paused items may be intentional pauses — but in an automated environment
|
||||
# a Paused item sitting in the queue indefinitely is effectively stalled.
|
||||
|
||||
if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 SABnzbd — Stalled Queue Items ━━━"
|
||||
@@ -331,15 +381,19 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
[[ -z "$NZO_ID" ]] && continue
|
||||
STATUS=$(echo "$QUEUE" | grep -A10 "$NZO_ID" | \
|
||||
grep -o '"status":"[^"]*"' | sed 's/"status":"//;s/"//')
|
||||
[[ "$STATUS" != "Paused" ]] && ((SKIPPED++)) && continue
|
||||
# Only remove Paused or Stuck items — Downloading/Grabbing are active
|
||||
if [[ "$STATUS" != "Paused" ]] && [[ "$STATUS" != "Stuck" ]]; then
|
||||
((SKIPPED++))
|
||||
continue
|
||||
fi
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would remove stalled item: $NZO_ID"
|
||||
warn "DRY RUN — would remove stalled item: $NZO_ID ($STATUS)"
|
||||
((DELETED++))
|
||||
else
|
||||
curl -sf --max-time 10 \
|
||||
"$SABNZBD_URL/api?mode=queue&name=delete&value=$NZO_ID&apikey=$SABNZBD_API_KEY" \
|
||||
>/dev/null
|
||||
info "$ICON_TRASH Removed stalled: $NZO_ID"
|
||||
info "$ICON_TRASH Removed stalled ($STATUS): $NZO_ID"
|
||||
((DELETED++))
|
||||
fi
|
||||
done <<< "$STALLED_IDS"
|
||||
@@ -349,11 +403,17 @@ if [[ -n "$SABNZBD_URL" ]] && [[ -n "$SABNZBD_API_KEY" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ qBittorrent — Last Chance Failsafe Cleanup ━━━
|
||||
# Deletes torrents older than QBIT_FAILSAFE_MIN_DAYS
|
||||
# deleteFiles=false — removes from qBit, leaves files for arrs to manage
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ qBittorrent — Age Failsafe Cleanup ━━━
|
||||
# ==============================================================================================
|
||||
# Last-chance cleanup for torrents that have been sitting in qBit past their useful life.
|
||||
# deleteFiles=false — removes the torrent record from qBit but leaves files on disk.
|
||||
# Radarr/Sonarr manage actual files independently — this only cleans up the qBit entry.
|
||||
#
|
||||
# Safety checks before deletion:
|
||||
# Age must exceed QBIT_FAILSAFE_MIN_DAYS
|
||||
# Ratio must meet QBIT_FAILSAFE_MIN_RATIO (0 = age only, no ratio requirement)
|
||||
|
||||
if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
|
||||
echo ""
|
||||
echo "━━━ 🔍 qBittorrent — Failsafe (older than ${QBIT_FAILSAFE_MIN_DAYS} days) ━━━"
|
||||
@@ -366,7 +426,8 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
|
||||
grep SID | awk '{print "SID="$NF}')
|
||||
|
||||
if [[ -z "$QBIT_COOKIE" ]]; then
|
||||
error "Failed to authenticate with qBittorrent"
|
||||
error "Failed to authenticate with qBittorrent — check QBIT_USERNAME/PASSWORD"
|
||||
notify "qBittorrent auth failed on $(hostname) — check credentials in master_host*.conf" "Downloaders Reset" "warning"
|
||||
((TOTAL_FAIL++))
|
||||
else
|
||||
TORRENTS=$(curl -sf --max-time 15 \
|
||||
@@ -385,8 +446,11 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
|
||||
[[ -z "$HASH" || -z "$ADDED" ]] && continue
|
||||
|
||||
AGE_DAYS=$(( (NOW - ADDED) / 86400 ))
|
||||
|
||||
# Age check — must be old enough
|
||||
[[ "$AGE_DAYS" -lt "$QBIT_FAILSAFE_MIN_DAYS" ]] && ((SKIPPED++)) && continue
|
||||
|
||||
# Ratio check — if configured
|
||||
if [[ "$QBIT_FAILSAFE_MIN_RATIO" != "0" ]]; then
|
||||
RATIO_INT="${RATIO%.*}"
|
||||
MIN_RATIO_INT="${QBIT_FAILSAFE_MIN_RATIO%.*}"
|
||||
@@ -411,14 +475,15 @@ if [[ -n "$QBIT_URL" ]] && [[ -n "$QBIT_USERNAME" ]]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ==============================================================================================
|
||||
# ━━━ Summary ━━━
|
||||
# ==============================================================================================
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY DOWNLOADERS RESET SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( $(date +%s) - START_TIME )))"
|
||||
echo "$ICON_SUCCESS Actions: $TOTAL_PASS"
|
||||
echo "$ICON_ERROR Failures: $TOTAL_FAIL"
|
||||
echo "$ICON_HOST Identity: $MY_ID ($LOCAL_SERVER_NAME)"
|
||||
echo "$ICON_TIME Duration: $(format_duration $(( $(date +%s) - START_TIME )))"
|
||||
echo "$ICON_SUCCESS Actions: $TOTAL_PASS"
|
||||
echo "$ICON_ERROR Failures: $TOTAL_FAIL"
|
||||
echo ""
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
|
||||
Reference in New Issue
Block a user