Added system watchdog. and way to many other changes
This commit is contained in:
@@ -22,7 +22,6 @@ parse_args "$@"
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
# ROOT CHECK
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
@@ -30,10 +29,9 @@ fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
# Verify Docker is available
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found — check PATH or Docker installation"
|
||||
notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "alert"
|
||||
notify "Docker daily restart failed — Docker not found on $(hostname)" "Docker Daily Restart" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -60,9 +58,6 @@ fi
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Attempts a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
|
||||
# Returns 0 on success, 1 if all attempts fail.
|
||||
# Usage: retry_docker docker restart Emby
|
||||
retry_docker() {
|
||||
local attempt=1
|
||||
|
||||
@@ -84,7 +79,7 @@ retry_docker() {
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CONTAINERS $ICON_STOP $ICON_START Daily Restart ━━━
|
||||
# ━━━ $ICON_CONTAINERS Daily Restart ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Daily Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
@@ -100,7 +95,6 @@ STARTED=()
|
||||
for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
# Verify container exists
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
error "$container does not exist — skipping"
|
||||
FAILED+=("$container")
|
||||
@@ -121,7 +115,8 @@ for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
echo "$ICON_STARTED $container restarted"
|
||||
RESTARTED+=("$container")
|
||||
else
|
||||
error "Failed to restart $container"
|
||||
error "Failed to restart $container after $RETRY_COUNT attempts"
|
||||
notify "$container failed to restart on $(hostname)" "Docker Daily Restart" "warning"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
fi
|
||||
@@ -136,7 +131,8 @@ for container in "${DAILY_RESTART_CONTAINERS[@]}"; do
|
||||
echo "$ICON_STARTED $container started"
|
||||
STARTED+=("$container")
|
||||
else
|
||||
error "Failed to start $container"
|
||||
error "Failed to start $container after $RETRY_COUNT attempts"
|
||||
notify "$container failed to start on $(hostname)" "Docker Daily Restart" "warning"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
fi
|
||||
@@ -157,27 +153,19 @@ END=$(date +%s)
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo "━━━━━ $ICON_SUMMARY DAILY RESTART SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
|
||||
if [[ ${#RESTARTED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||
fi
|
||||
if [[ ${#STARTED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_STARTED Started: ${STARTED[*]}"
|
||||
fi
|
||||
if [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
fi
|
||||
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||
[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE"
|
||||
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started" "Docker Daily Restart" "normal"
|
||||
notify "Daily restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Daily Restart" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed"
|
||||
notify "Daily restart completed with errors — failed: ${FAILED[*]}" "Docker Daily Restart" "alert"
|
||||
notify "Daily restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Daily Restart" "warning"
|
||||
fi
|
||||
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
|
||||
@@ -2,24 +2,28 @@
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Docker Watchdog --------------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Self-healing watchdog for Docker containers — monitors memory, CPU and HTTP responsiveness.
|
||||
# Restarts containers that exceed configured thresholds using a strike system for CPU and
|
||||
# responsiveness checks to avoid restarting on brief spikes.
|
||||
# First line of defense — monitors Docker containers for memory, CPU, HTTP responsiveness,
|
||||
# and unexpected stops. Restarts containers that exceed thresholds or go offline.
|
||||
#
|
||||
# Works alongside system_watchdog.sh:
|
||||
# docker_watchdog.sh — container level, minimal disruption, tries to self-heal
|
||||
# system_watchdog.sh — system level, last resort, reboots when healing fails
|
||||
#
|
||||
# Behaviour:
|
||||
# Memory — immediate restart if hard limit is exceeded
|
||||
# CPU — strike system, restarts after CPU_FAIL_LIMIT consecutive over-threshold checks
|
||||
# HTTP — strike system, restarts after RESP_FAIL_LIMIT consecutive failed curl checks
|
||||
# Memory — immediate restart if hard limit exceeded
|
||||
# CPU — strike system, restarts after CPU_FAIL_LIMIT consecutive hits
|
||||
# HTTP — strike system, restarts after RESP_FAIL_LIMIT consecutive failures
|
||||
# Required — strike system, restarts stopped containers, persistent skip list
|
||||
# prevents reboot loops, auto-clears when container recovers
|
||||
# Daemon — immediate notify if Docker daemon is unresponsive
|
||||
#
|
||||
# Strike system:
|
||||
# Strikes persist between runs via WATCHDOG_STATE_FILE (/tmp — resets on reboot)
|
||||
# Strike cadence depends on cron schedule:
|
||||
# Every 15min + 2 strikes = 30min sustained abuse before restart
|
||||
# Every 10min + 2 strikes = 20min sustained abuse before restart
|
||||
# Every 5min + 2 strikes = 10min sustained abuse before restart
|
||||
# Strike cadence depends on cron schedule:
|
||||
# Every 15min + 2 strikes = 30min sustained before restart
|
||||
# Every 10min + 2 strikes = 20min sustained before restart
|
||||
# Every 5min + 2 strikes = 10min sustained before restart
|
||||
#
|
||||
# All configuration lives in Master.conf under the Docker Watchdog section.
|
||||
# Supports --dry-run to show what would be restarted without taking any action.
|
||||
# All configuration in Master.conf under Docker Watchdog section.
|
||||
# Supports --dry-run to show what would happen without acting.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
@@ -29,16 +33,18 @@ source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# Auto-detect total CPU cores for normalisation
|
||||
TOTAL_CORES=$(nproc)
|
||||
|
||||
# Persistent skip list — shared with system_watchdog.sh
|
||||
# Containers in this list are skipped until they recover
|
||||
SKIP_LIST_FILE="$SYS_WATCHDOG_FAILED_FILE"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
# ROOT CHECK
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
@@ -47,27 +53,32 @@ fi
|
||||
success "Running as root"
|
||||
info "$ICON_WATCHDOG Watchdog initialising — $TOTAL_CORES cores detected"
|
||||
|
||||
# Ensure state file exists
|
||||
touch "$WATCHDOG_STATE_FILE" 2>/dev/null || {
|
||||
error "Cannot create state file: $WATCHDOG_STATE_FILE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
touch "$SKIP_LIST_FILE" 2>/dev/null || {
|
||||
error "Cannot create skip list: $SKIP_LIST_FILE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_WATCHDOG Containers monitored: ${!WATCHDOG_CONTAINERS[*]}"
|
||||
echo "$ICON_MEM Soft mem threshold: ${SOFT_MEM_THRESHOLD}% of per-container limit"
|
||||
echo "$ICON_ZFS CPU soft threshold: ${SOFT_CPU_THRESHOLD}%"
|
||||
echo "$ICON_ZFS CPU hard threshold: ${HARD_CPU_THRESHOLD}%"
|
||||
echo "$ICON_RETRY CPU fail limit: ${CPU_FAIL_LIMIT} strikes"
|
||||
echo "$ICON_PING Resp fail limit: ${RESP_FAIL_LIMIT} strikes"
|
||||
echo "$ICON_TIME Curl timeout: ${CURL_TIMEOUT}s"
|
||||
echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "$ICON_WATCHDOG Monitored: ${!WATCHDOG_CONTAINERS[*]}"
|
||||
echo "$ICON_CONTAINERS Required: ${WATCHDOG_REQUIRED_CONTAINERS[*]}"
|
||||
echo "$ICON_MEM Soft mem: ${SOFT_MEM_THRESHOLD}% of limit"
|
||||
echo "$ICON_ZFS CPU soft: ${SOFT_CPU_THRESHOLD}%"
|
||||
echo "$ICON_ZFS CPU hard: ${HARD_CPU_THRESHOLD}%"
|
||||
echo "$ICON_RETRY CPU strikes: ${CPU_FAIL_LIMIT}"
|
||||
echo "$ICON_RETRY Resp strikes: ${RESP_FAIL_LIMIT}"
|
||||
echo "$ICON_TIME Curl timeout: ${CURL_TIMEOUT}s"
|
||||
echo "$ICON_NOTIFY Notify: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
@@ -76,19 +87,13 @@ fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# STATE HELPERS
|
||||
# Reads and writes per-container strike counts to the state file.
|
||||
# State file format: container:metric:count
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Returns current strike count for a container/metric pair.
|
||||
# Usage: get_strikes "Emby" "CPU"
|
||||
get_strikes() {
|
||||
local container="$1" metric="$2"
|
||||
grep -E "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null | cut -d':' -f3
|
||||
}
|
||||
|
||||
# Sets strike count for a container/metric pair.
|
||||
# Usage: set_strikes "Emby" "CPU" 2
|
||||
set_strikes() {
|
||||
local container="$1" metric="$2" count="$3"
|
||||
grep -vE "^${container}:${metric}:" "$WATCHDOG_STATE_FILE" 2>/dev/null > "${WATCHDOG_STATE_FILE}.tmp"
|
||||
@@ -96,19 +101,83 @@ set_strikes() {
|
||||
mv "${WATCHDOG_STATE_FILE}.tmp" "$WATCHDOG_STATE_FILE"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# SKIP LIST HELPERS
|
||||
# Container skip list — persistent across reboots via /boot/
|
||||
# Auto-clears entries when container is found running again.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
is_in_skip_list() {
|
||||
local container="$1"
|
||||
grep -qE "^${container}$" "$SKIP_LIST_FILE" 2>/dev/null
|
||||
}
|
||||
|
||||
add_to_skip_list() {
|
||||
local container="$1"
|
||||
if ! is_in_skip_list "$container"; then
|
||||
echo "$container" >> "$SKIP_LIST_FILE"
|
||||
warn "$ICON_WATCHDOG $container added to persistent skip list"
|
||||
notify "$container added to watchdog skip list on $(hostname) — manual check recommended" "Docker Watchdog" "warning"
|
||||
fi
|
||||
}
|
||||
|
||||
remove_from_skip_list() {
|
||||
local container="$1"
|
||||
grep -vE "^${container}$" "$SKIP_LIST_FILE" 2>/dev/null > "${SKIP_LIST_FILE}.tmp"
|
||||
mv "${SKIP_LIST_FILE}.tmp" "$SKIP_LIST_FILE"
|
||||
success "$ICON_WATCHDOG $container recovered — removed from skip list"
|
||||
notify "$container recovered and removed from watchdog skip list on $(hostname)" "Docker Watchdog" "normal"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# SKIP LIST AUTO-HEAL CHECK
|
||||
# On every run check if any skipped containers are now running.
|
||||
# If running remove from skip list — could have recovered after reboot or manual fix.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_skip_list_recovery() {
|
||||
[[ ! -s "$SKIP_LIST_FILE" ]] && return
|
||||
|
||||
info "$ICON_WATCHDOG Checking skip list for recovered containers..."
|
||||
|
||||
while IFS= read -r container; do
|
||||
[[ -z "$container" ]] && continue
|
||||
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
remove_from_skip_list "$container"
|
||||
else
|
||||
log "$container still not running — remains on skip list"
|
||||
fi
|
||||
done < "$SKIP_LIST_FILE"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# DOCKER DAEMON HEALTH CHECK
|
||||
# Verifies Docker daemon is responding before attempting any container operations.
|
||||
# A hung daemon means all checks will fail — notify immediately and exit.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_docker_daemon() {
|
||||
info "$ICON_CONTAINERS Checking Docker daemon..."
|
||||
|
||||
if ! timeout 10 docker ps >/dev/null 2>&1; then
|
||||
error "Docker daemon is not responding"
|
||||
notify "Docker daemon unresponsive on $(hostname) — immediate attention required" "Docker Watchdog" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Docker daemon is healthy"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# HELPERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Fetches memory and CPU stats for a container in a single docker stats call.
|
||||
# Returns: MEM_USAGE|CPU_PERCENT
|
||||
parse_stats() {
|
||||
local container="$1"
|
||||
docker stats --no-stream --format "{{.MemUsage}}|{{.CPUPerc}}" "$container"
|
||||
}
|
||||
|
||||
# Converts a memory value and unit to MB.
|
||||
# Supports KiB, MiB, GiB — returns UNKNOWN for unrecognised units.
|
||||
convert_to_mb() {
|
||||
local value="$1" unit="$2"
|
||||
case "$unit" in
|
||||
@@ -119,32 +188,32 @@ convert_to_mb() {
|
||||
esac
|
||||
}
|
||||
|
||||
# Restarts a container locally and sends a notification.
|
||||
# In dry run mode reports what would happen without acting.
|
||||
# Restarts a container and sends notification.
|
||||
# Failed restarts also notify — system_watchdog.sh is the next line of defense.
|
||||
restart_container() {
|
||||
local container="$1" reason="$2"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container ($reason)"
|
||||
return
|
||||
return 0
|
||||
fi
|
||||
|
||||
info "Restarting $container ($reason)..."
|
||||
if docker restart "$container" >/dev/null 2>&1; then
|
||||
echo "$ICON_STARTED $container restarted"
|
||||
notify "$container restarted — $reason" "Docker Watchdog" "warning"
|
||||
log "Restarted $container — reason: $reason"
|
||||
notify "$container restarted on $(hostname) — $reason" "Docker Watchdog" "warning"
|
||||
return 0
|
||||
else
|
||||
error "Failed to restart $container"
|
||||
notify "Failed to restart $container — $reason" "Docker Watchdog" "alert"
|
||||
notify "Failed to restart $container on $(hostname) — $reason" "Docker Watchdog" "warning"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# MEMORY CHECK
|
||||
# Compares current container memory usage against its configured hard limit.
|
||||
# Restarts immediately if at or above 100% of limit.
|
||||
# Warns if at or above SOFT_MEM_THRESHOLD % of limit.
|
||||
# Restarts immediately if container exceeds hard memory limit.
|
||||
# Warns if approaching soft threshold.
|
||||
# Usage: check_memory "Emby" 16384
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_memory() {
|
||||
@@ -174,9 +243,8 @@ check_memory() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# CPU CHECK
|
||||
# Normalises CPU usage against total core count and applies the strike system.
|
||||
# Warns at SOFT_CPU_THRESHOLD, strikes at HARD_CPU_THRESHOLD.
|
||||
# Restarts after CPU_FAIL_LIMIT consecutive strikes — resets strikes on restart or recovery.
|
||||
# Strike system — restarts after CPU_FAIL_LIMIT consecutive over-threshold checks.
|
||||
# Resets strikes on recovery or restart.
|
||||
# Usage: check_cpu "Emby"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_cpu() {
|
||||
@@ -193,24 +261,21 @@ check_cpu() {
|
||||
|
||||
if (( cpu_int >= HARD_CPU_THRESHOLD )); then
|
||||
((violations++))
|
||||
error "$ICON_ZFS $container CPU ${cpu_int}% — hard threshold hit ($violations/$CPU_FAIL_LIMIT strikes)"
|
||||
error "$ICON_ZFS $container CPU ${cpu_int}% — hard threshold ($violations/$CPU_FAIL_LIMIT strikes)"
|
||||
set_strikes "$container" "CPU" "$violations"
|
||||
elif (( cpu_int >= SOFT_CPU_THRESHOLD )); then
|
||||
((violations++))
|
||||
warn "$ICON_ZFS $container CPU ${cpu_int}% — soft threshold hit ($violations/$CPU_FAIL_LIMIT strikes)"
|
||||
warn "$ICON_ZFS $container CPU ${cpu_int}% — soft threshold ($violations/$CPU_FAIL_LIMIT strikes)"
|
||||
set_strikes "$container" "CPU" "$violations"
|
||||
else
|
||||
if (( violations > 0 )); then
|
||||
info "$ICON_ZFS $container CPU ${cpu_int}% — recovered, resetting strikes"
|
||||
else
|
||||
success "$ICON_ZFS $container CPU ${cpu_int}%"
|
||||
fi
|
||||
[[ $violations -gt 0 ]] && info "$ICON_ZFS $container CPU ${cpu_int}% — recovered, resetting strikes"
|
||||
[[ $violations -eq 0 ]] && success "$ICON_ZFS $container CPU ${cpu_int}%"
|
||||
set_strikes "$container" "CPU" 0
|
||||
violations=0
|
||||
fi
|
||||
|
||||
if (( violations >= CPU_FAIL_LIMIT )); then
|
||||
error "$ICON_ZFS $container hit CPU limit for $CPU_FAIL_LIMIT consecutive checks"
|
||||
error "$ICON_ZFS $container CPU limit hit for $CPU_FAIL_LIMIT consecutive checks"
|
||||
restart_container "$container" "sustained CPU abuse"
|
||||
set_strikes "$container" "CPU" 0
|
||||
fi
|
||||
@@ -218,9 +283,8 @@ check_cpu() {
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# RESPONSIVENESS CHECK
|
||||
# Sends an HTTP request to the container's configured URL.
|
||||
# Strike system — restarts after RESP_FAIL_LIMIT consecutive failed HTTP checks.
|
||||
# Skips containers with no URL defined in WATCHDOG_CONTAINER_URLS.
|
||||
# Applies the same strike system as CPU — restarts after RESP_FAIL_LIMIT consecutive failures.
|
||||
# Usage: check_responsiveness "Emby"
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_responsiveness() {
|
||||
@@ -238,11 +302,8 @@ check_responsiveness() {
|
||||
warn "$ICON_PING $container unresponsive at $url ($fails/$RESP_FAIL_LIMIT strikes)"
|
||||
set_strikes "$container" "RESP" "$fails"
|
||||
else
|
||||
if (( fails > 0 )); then
|
||||
info "$ICON_PING $container responsive again — resetting strikes"
|
||||
else
|
||||
success "$ICON_PING $container responsive at $url"
|
||||
fi
|
||||
[[ $fails -gt 0 ]] && info "$ICON_PING $container responsive again — resetting strikes"
|
||||
[[ $fails -eq 0 ]] && success "$ICON_PING $container responsive at $url"
|
||||
set_strikes "$container" "RESP" 0
|
||||
fails=0
|
||||
fi
|
||||
@@ -255,54 +316,131 @@ check_responsiveness() {
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_WATCHDOG Watchdog Check ━━━
|
||||
# REQUIRED CONTAINER CHECK
|
||||
# Monitors WATCHDOG_REQUIRED_CONTAINERS for unexpected stops.
|
||||
# Strike system — attempts restart on each strike.
|
||||
# After strike limit hit — adds to persistent skip list and notifies system_watchdog handoff.
|
||||
# Skip list auto-clears at start of each run if container has recovered.
|
||||
# Usage: check_required_containers
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
check_required_containers() {
|
||||
[[ ${#WATCHDOG_REQUIRED_CONTAINERS[@]} -eq 0 ]] && return
|
||||
|
||||
info "$ICON_CONTAINERS Checking required containers..."
|
||||
|
||||
for container in "${WATCHDOG_REQUIRED_CONTAINERS[@]}"; do
|
||||
[[ -z "$container" ]] && continue
|
||||
|
||||
# Skip if on persistent skip list
|
||||
if is_in_skip_list "$container"; then
|
||||
warn "$ICON_NOT_RUNNING $container is on skip list — skipping until recovered"
|
||||
continue
|
||||
fi
|
||||
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "unknown")
|
||||
|
||||
if [[ "$STATUS" == "true" ]]; then
|
||||
success "$ICON_RUNNING $container is running"
|
||||
set_strikes "$container" "STOP" 0
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$STATUS" == "unknown" ]]; then
|
||||
warn "$container not found on this host — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Container is stopped — apply strike
|
||||
local strikes
|
||||
strikes=$(get_strikes "$container" "STOP")
|
||||
[[ -z "$strikes" ]] && strikes=0
|
||||
((strikes++))
|
||||
|
||||
warn "$ICON_NOT_RUNNING $container is stopped ($strikes/$SYS_WATCHDOG_STRIKE_LIMIT strikes)"
|
||||
set_strikes "$container" "STOP" "$strikes"
|
||||
|
||||
# Attempt restart on each strike
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would attempt restart of $container"
|
||||
else
|
||||
if restart_container "$container" "unexpected stop"; then
|
||||
set_strikes "$container" "STOP" 0
|
||||
else
|
||||
# Restart failed
|
||||
if (( strikes >= SYS_WATCHDOG_STRIKE_LIMIT )); then
|
||||
error "$container failed to restart after $SYS_WATCHDOG_STRIKE_LIMIT attempts"
|
||||
add_to_skip_list "$container"
|
||||
set_strikes "$container" "STOP" 0
|
||||
notify "$container handed off to system_watchdog on $(hostname) — added to skip list" "Docker Watchdog" "warning"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_WATCHDOG Watchdog Run ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_WATCHDOG Watchdog Check — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "━━━ $ICON_WATCHDOG Watchdog Run — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo ""
|
||||
|
||||
SKIPPED=()
|
||||
RESTARTED=0
|
||||
|
||||
START=$(date +%s)
|
||||
SKIPPED=()
|
||||
|
||||
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
|
||||
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
# Daemon check first — if daemon is down nothing else works
|
||||
check_docker_daemon
|
||||
|
||||
# Verify container exists
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
warn "$container not found on this host — skipping"
|
||||
SKIPPED+=("$container")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
# Verify container is running
|
||||
if ! docker ps --filter "name=^/${container}$" --format "{{.Names}}" | grep -qw "$container"; then
|
||||
warn "$ICON_NOT_RUNNING $container is not running — skipping"
|
||||
SKIPPED+=("$container")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
check_memory "$container" "${WATCHDOG_CONTAINERS[$container]}"
|
||||
check_cpu "$container"
|
||||
check_responsiveness "$container"
|
||||
# Auto-heal skip list before processing
|
||||
check_skip_list_recovery
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Resource monitoring — WATCHDOG_CONTAINERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ ${#WATCHDOG_CONTAINERS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
done
|
||||
echo "━━━ $ICON_MEM Resource Monitoring ━━━"
|
||||
|
||||
for container in "${!WATCHDOG_CONTAINERS[@]}"; do
|
||||
echo ""
|
||||
info "$ICON_CONTAINERS $container"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
warn "$container not found — skipping"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! docker ps --filter "name=^/${container}$" --format "{{.Names}}" | grep -qw "$container"; then
|
||||
warn "$ICON_NOT_RUNNING $container is not running — skipping resource checks"
|
||||
SKIPPED+=("$container")
|
||||
continue
|
||||
fi
|
||||
|
||||
check_memory "$container" "${WATCHDOG_CONTAINERS[$container]}"
|
||||
check_cpu "$container"
|
||||
check_responsiveness "$container"
|
||||
done
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Required container monitoring — WATCHDOG_REQUIRED_CONTAINERS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Required Container Check ━━━"
|
||||
check_required_containers
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY WATCHDOG SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_WATCHDOG Monitored: ${#WATCHDOG_CONTAINERS[@]} containers"
|
||||
echo "$ICON_NOT_RUNNING Skipped: ${#SKIPPED[@]} containers"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Dry Run: no restarts executed"
|
||||
fi
|
||||
echo "$ICON_TIME $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo "$ICON_WATCHDOG Monitored: ${#WATCHDOG_CONTAINERS[@]} containers"
|
||||
echo "$ICON_CONTAINERS Required: ${#WATCHDOG_REQUIRED_CONTAINERS[@]} containers"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_NOT_RUNNING Skipped: ${SKIPPED[*]}"
|
||||
[[ "$DRY_RUN" == true ]] && echo "$ICON_WARN Dry Run: no actions taken"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -0,0 +1,174 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Docker Weekly Restart --------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Restarts or starts specified Docker containers with retry logic.
|
||||
# Containers are configured in Master.conf under WEEKLY_RESTART_CONTAINERS.
|
||||
# Uses global RETRY_COUNT and SLEEP from Master.conf for retry behaviour.
|
||||
# Sends notifications on completion or failure via common.sh notify().
|
||||
# Supports --dry-run to preview what would be restarted without taking action.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
if ! command -v docker &>/dev/null; then
|
||||
error "Docker command not found — check PATH or Docker installation"
|
||||
notify "Docker weekly restart failed — Docker not found on $(hostname)" "Docker Weekly Restart" "warning"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Docker found"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
|
||||
echo "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||
echo "$ICON_TIME Sleep: ${SLEEP}s between retries"
|
||||
echo "$ICON_NOTIFY Notifications: unRAID=${NOTIFY_UNRAID:-false} Discord=$([[ -n "${DISCORD_WEBHOOK:-}" ]] && echo enabled || echo disabled)"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no containers will be restarted"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# FUNCTIONS
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
# Attempts a docker command up to RETRY_COUNT times with SLEEP seconds between attempts.
|
||||
# Returns 0 on success, 1 if all attempts fail.
|
||||
retry_docker() {
|
||||
local attempt=1
|
||||
|
||||
while [[ "$attempt" -le "$RETRY_COUNT" ]]; do
|
||||
info "$ICON_RETRY Attempt $attempt of $RETRY_COUNT: $*"
|
||||
|
||||
if "$@"; then
|
||||
success "Succeeded on attempt $attempt"
|
||||
return 0
|
||||
else
|
||||
warn "Attempt $attempt failed"
|
||||
(( attempt++ ))
|
||||
[[ "$attempt" -le "$RETRY_COUNT" ]] && sleep "$SLEEP"
|
||||
fi
|
||||
done
|
||||
|
||||
error "Command failed after $RETRY_COUNT attempts: $*"
|
||||
return 1
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CONTAINERS Weekly Restart ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CONTAINERS Weekly Restart — $(date '+%Y-%m-%d %H:%M:%S') ━━━"
|
||||
echo "$ICON_CONTAINERS Containers: ${WEEKLY_RESTART_CONTAINERS[*]}"
|
||||
echo "$ICON_RETRY Retries: $RETRY_COUNT"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
FAILED=()
|
||||
RESTARTED=()
|
||||
STARTED=()
|
||||
|
||||
for container in "${WEEKLY_RESTART_CONTAINERS[@]}"; do
|
||||
echo "━━━ $ICON_CONTAINERS $container ━━━"
|
||||
|
||||
if ! docker inspect "$container" &>/dev/null; then
|
||||
error "$container does not exist — skipping"
|
||||
FAILED+=("$container")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
STATUS=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)
|
||||
|
||||
case "$STATUS" in
|
||||
true)
|
||||
echo "$ICON_RUNNING $container is running — restarting..."
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would restart $container"
|
||||
else
|
||||
if retry_docker docker restart "$container"; then
|
||||
echo "$ICON_STARTED $container restarted"
|
||||
RESTARTED+=("$container")
|
||||
else
|
||||
error "Failed to restart $container after $RETRY_COUNT attempts"
|
||||
notify "$container failed to restart on $(hostname)" "Docker Weekly Restart" "warning"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
false)
|
||||
echo "$ICON_NOT_RUNNING $container is stopped — starting..."
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would start $container"
|
||||
else
|
||||
if retry_docker docker start "$container"; then
|
||||
echo "$ICON_STARTED $container started"
|
||||
STARTED+=("$container")
|
||||
else
|
||||
error "Failed to start $container after $RETRY_COUNT attempts"
|
||||
notify "$container failed to start on $(hostname)" "Docker Weekly Restart" "warning"
|
||||
FAILED+=("$container")
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
error "Unknown status for $container: $STATUS"
|
||||
FAILED+=("$container")
|
||||
;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo "━━━━━ $ICON_SUMMARY WEEKLY RESTART SUMMARY ━━━━━"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
[[ ${#RESTARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Restarted: ${RESTARTED[*]}"
|
||||
[[ ${#STARTED[@]} -gt 0 ]] && echo "$ICON_STARTED Started: ${STARTED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -eq 0 ]]; then
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS ALL DONE"
|
||||
notify "Weekly restart complete — ${#RESTARTED[@]} restarted, ${#STARTED[@]} started on $(hostname)" "Docker Weekly Restart" "normal"
|
||||
else
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR ${#FAILED[@]} container(s) failed"
|
||||
notify "Weekly restart completed with errors on $(hostname) — failed: ${FAILED[*]}" "Docker Weekly Restart" "warning"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Media Permissions Script -----------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Applies permissions and ownership to all configured media shares.
|
||||
# Shares, permissions mode and owner are configured in Master.conf.
|
||||
# Supports --dry-run to preview what would be changed without making changes.
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
parse_args "$@"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_PERMS Mode: $PERMISSIONS_MODE"
|
||||
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||
echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no changes will be made"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_PERMS Media Permissions ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_PERMS Media Permissions ━━━"
|
||||
echo "$ICON_PERMS Mode: $PERMISSIONS_MODE"
|
||||
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||
echo "$ICON_PERMS Shares: ${#MEDIA_PERMISSION_SHARES[@]}"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
FAILED=()
|
||||
UPDATED=()
|
||||
SKIPPED=()
|
||||
|
||||
for SHARE in "${MEDIA_PERMISSION_SHARES[@]}"; do
|
||||
SHARE_NAME=$(basename "$SHARE")
|
||||
|
||||
if [[ ! -d "$SHARE" ]]; then
|
||||
warn "$ICON_PERMS $SHARE_NAME not found — skipping"
|
||||
SKIPPED+=("$SHARE_NAME")
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — would apply $PERMISSIONS_MODE $PERMISSIONS_OWNER to $SHARE"
|
||||
continue
|
||||
fi
|
||||
|
||||
info "$ICON_PERMS Updating $SHARE_NAME..."
|
||||
|
||||
CHMOD_OK=true
|
||||
CHOWN_OK=true
|
||||
|
||||
chmod -R "$PERMISSIONS_MODE" "$SHARE" 2>/dev/null || CHMOD_OK=false
|
||||
chown -R "$PERMISSIONS_OWNER" "$SHARE" 2>/dev/null || CHOWN_OK=false
|
||||
|
||||
if [[ "$CHMOD_OK" == true && "$CHOWN_OK" == true ]]; then
|
||||
echo "$ICON_UNLOCKED $SHARE_NAME — permissions applied"
|
||||
UPDATED+=("$SHARE_NAME")
|
||||
else
|
||||
error "$SHARE_NAME — permissions failed (chmod=$CHMOD_OK chown=$CHOWN_OK)"
|
||||
FAILED+=("$SHARE_NAME")
|
||||
fi
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY MEDIA PERMISSIONS SUMMARY ━━━━━"
|
||||
echo "$ICON_PERMS Mode: $PERMISSIONS_MODE"
|
||||
echo "$ICON_PERMS Owner: $PERMISSIONS_OWNER"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
echo ""
|
||||
[[ ${#UPDATED[@]} -gt 0 ]] && echo " $ICON_UNLOCKED Updated: ${#UPDATED[@]}"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && echo " $ICON_WARN Skipped: ${#SKIPPED[@]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo " $ICON_ERROR Failed: ${#FAILED[@]} — ${FAILED[*]}"
|
||||
echo ""
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no changes made"
|
||||
elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR SOME SHARES FAILED"
|
||||
notify "Media permissions failed on $(hostname) — ${FAILED[*]}" "Media Permissions" "warning"
|
||||
else
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
notify "Media permissions applied on $(hostname) — ${#UPDATED[@]} shares updated" "Media Permissions" "normal"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
@@ -0,0 +1,183 @@
|
||||
#!/bin/bash
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# --------------------------------- Media Cleaner Script ---------------------------------------
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Removes unwanted files from media share folders using configurable file patterns.
|
||||
# Supports two profiles: anime and media — each with their own folder list and file patterns.
|
||||
# Profiles and patterns are configured in Master.conf.
|
||||
# Supports --dry-run to preview what would be deleted without making changes.
|
||||
#
|
||||
# Usage:
|
||||
# media_cleaner.sh anime — clean anime shares
|
||||
# media_cleaner.sh media — clean media shares
|
||||
# media_cleaner.sh anime --dry-run — preview anime clean
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
source "$SCRIPT_DIR/../Master.conf"
|
||||
source "$SCRIPT_DIR/../common.sh"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# Separate profile argument from flags
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
PROFILE=""
|
||||
RAW_ARGS=()
|
||||
|
||||
for ARG in "$@"; do
|
||||
case "$ARG" in
|
||||
--*|*=*) RAW_ARGS+=("$ARG") ;;
|
||||
anime|media) PROFILE="$ARG" ;;
|
||||
*) RAW_ARGS+=("$ARG") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
parse_args "${RAW_ARGS[@]}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_GEAR Setup ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_GEAR Setup ━━━"
|
||||
|
||||
if [[ "$EUID" -ne 0 ]]; then
|
||||
error "Must be run as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Running as root"
|
||||
|
||||
if ! command -v find >/dev/null 2>&1; then
|
||||
error "find command not found — check findutils installation"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$PROFILE" ]]; then
|
||||
error "No profile specified. Usage: media_cleaner.sh <anime|media> [--dry-run]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Resolve profile folders and patterns
|
||||
case "$PROFILE" in
|
||||
anime)
|
||||
CLEAN_FOLDERS=("${ANIME_CLEAN_FOLDERS[@]}")
|
||||
FILE_PATTERNS=("${ANIME_FILE_PATTERNS[@]}")
|
||||
;;
|
||||
media)
|
||||
CLEAN_FOLDERS=("${MEDIA_CLEAN_FOLDERS[@]}")
|
||||
FILE_PATTERNS=("${MEDIA_FILE_PATTERNS[@]}")
|
||||
;;
|
||||
*)
|
||||
error "Unknown profile: $PROFILE — must be anime or media"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
info "$ICON_GEAR Profile: $PROFILE"
|
||||
info "$ICON_CLEAN Folders: ${#CLEAN_FOLDERS[@]}"
|
||||
info "$ICON_TRASH Patterns: ${#FILE_PATTERNS[@]}"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Status ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
if [[ "$SHOW_STATUS" == true ]]; then
|
||||
echo ""
|
||||
echo "━━━━━ $ICON_SUMMARY STATUS ━━━━━"
|
||||
echo "$ICON_GEAR Profile: $PROFILE"
|
||||
echo "$ICON_CLEAN Folders: ${CLEAN_FOLDERS[*]}"
|
||||
echo "$ICON_TRASH Patterns: ${FILE_PATTERNS[*]}"
|
||||
echo "$ICON_GEAR Dry Run: $DRY_RUN"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[[ "$DRY_RUN" == true ]] && warn "DRY RUN — no files will be deleted"
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_CLEAN Media Cleaner ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "━━━ $ICON_CLEAN Media Cleaner — $PROFILE ━━━"
|
||||
echo ""
|
||||
|
||||
START=$(date +%s)
|
||||
TOTAL_REMOVED=0
|
||||
FAILED=()
|
||||
SKIPPED=()
|
||||
|
||||
for FOLDER in "${CLEAN_FOLDERS[@]}"; do
|
||||
FOLDER_NAME=$(basename "$FOLDER")
|
||||
echo "━━━ $ICON_CLEAN $FOLDER_NAME ━━━"
|
||||
|
||||
if [[ ! -d "$FOLDER" ]]; then
|
||||
warn "$FOLDER_NAME not found — skipping"
|
||||
SKIPPED+=("$FOLDER_NAME")
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
# Build find command dynamically from FILE_PATTERNS array
|
||||
CMD=(find "$FOLDER" -type f \()
|
||||
for ((i = 0; i < ${#FILE_PATTERNS[@]}; i++)); do
|
||||
CMD+=(-iname "${FILE_PATTERNS[i]}")
|
||||
if [[ $i -lt $(( ${#FILE_PATTERNS[@]} - 1 )) ]]; then
|
||||
CMD+=(-o)
|
||||
fi
|
||||
done
|
||||
CMD+=(\))
|
||||
|
||||
# Count matching files before acting
|
||||
FILE_COUNT=$("${CMD[@]}" 2>/dev/null | wc -l)
|
||||
|
||||
if [[ "$FILE_COUNT" -eq 0 ]]; then
|
||||
success "$FOLDER_NAME — no matching files found"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
info "$ICON_TRASH $FILE_COUNT file(s) found in $FOLDER_NAME"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
warn "DRY RUN — files that would be deleted:"
|
||||
"${CMD[@]}" 2>/dev/null | while IFS= read -r f; do
|
||||
echo " $ICON_TRASH $f"
|
||||
done
|
||||
else
|
||||
CLEAN_CMD=("${CMD[@]}" -exec rm -f {} +)
|
||||
if "${CLEAN_CMD[@]}" 2>/dev/null; then
|
||||
success "$FOLDER_NAME — $FILE_COUNT file(s) removed"
|
||||
TOTAL_REMOVED=$((TOTAL_REMOVED + FILE_COUNT))
|
||||
else
|
||||
error "$FOLDER_NAME — cleanup failed"
|
||||
FAILED+=("$FOLDER_NAME")
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
END=$(date +%s)
|
||||
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
# ━━━ $ICON_SUMMARY Summary ━━━
|
||||
# -----------------------------------------------------------------------------------------------
|
||||
echo "━━━━━ $ICON_SUMMARY MEDIA CLEANER SUMMARY ━━━━━"
|
||||
echo "$ICON_GEAR Profile: $PROFILE"
|
||||
echo "$ICON_TIME Duration: $(format_duration $((END - START)))"
|
||||
[[ ${#SKIPPED[@]} -gt 0 ]] && echo "$ICON_WARN Skipped: ${SKIPPED[*]}"
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && echo "$ICON_ERROR Failed: ${FAILED[*]}"
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "$ICON_WARN Status: DRY RUN — no files deleted"
|
||||
elif [[ ${#FAILED[@]} -gt 0 ]]; then
|
||||
echo "$ICON_ERROR Status: $ICON_ERROR SOME FOLDERS FAILED"
|
||||
notify "Media cleaner ($PROFILE) failed on $(hostname) — ${FAILED[*]}" "Media Cleaner" "warning"
|
||||
else
|
||||
echo "$ICON_TRASH Removed: $TOTAL_REMOVED file(s)"
|
||||
echo "$ICON_DONE Status: $ICON_SUCCESS DONE"
|
||||
notify "Media cleaner ($PROFILE) complete on $(hostname) — $TOTAL_REMOVED file(s) removed" "Media Cleaner" "normal"
|
||||
fi
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
[[ ${#FAILED[@]} -gt 0 ]] && exit 1
|
||||
exit 0
|
||||
Reference in New Issue
Block a user